0.2 - docker
This commit is contained in:
@@ -0,0 +1,6 @@
|
||||
.venv
|
||||
__pycache__
|
||||
.git
|
||||
*.pyc
|
||||
res/
|
||||
done/
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY parse.py .
|
||||
COPY lists/ lists/
|
||||
COPY docker/app-entrypoint.sh /entrypoint.sh
|
||||
RUN chmod +x /entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
CMD ["python", "parse.py"]
|
||||
@@ -0,0 +1,27 @@
|
||||
# Папки
|
||||
- `lists/` — необработанные .txt файлы
|
||||
- `done/` — обработанные файлы (переносятся из lists/)
|
||||
- `res/` — результат в JSON
|
||||
|
||||
# Запуск
|
||||
|
||||
## Docker
|
||||
```bash
|
||||
chmod +x run.sh && ./run.sh
|
||||
```
|
||||
|
||||
```bash
|
||||
# Статус загрузки и работы модели в отдельном терминале
|
||||
docker compose logs -f ollama
|
||||
```
|
||||
|
||||
При первом запуске Ollama скачает модель `gemma3n:e4b` (7.5 GB) (может занять несколько минут).
|
||||
|
||||
## Локально
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
python3 parse.py
|
||||
```
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
services:
|
||||
ollama:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: docker/Dockerfile.ollama
|
||||
container_name: ollama
|
||||
volumes:
|
||||
- ollama_data:/root/.ollama
|
||||
ports:
|
||||
- "11435:11434" # 11435 — если локальный Ollama уже на 11434
|
||||
environment:
|
||||
OLLAMA_MODEL: gemma3n:e4b
|
||||
entrypoint: ["/bin/sh", "-c"]
|
||||
command:
|
||||
- |
|
||||
ollama serve &
|
||||
sleep 5
|
||||
until curl -sf http://localhost:11434/api/tags > /dev/null 2>&1; do sleep 2; done
|
||||
ollama pull $${OLLAMA_MODEL:-gemma3n:e4b}
|
||||
wait
|
||||
|
||||
app:
|
||||
build: .
|
||||
container_name: cart-to-json
|
||||
volumes:
|
||||
- ./lists:/app/lists
|
||||
- ./res:/app/res
|
||||
- ./done:/app/done
|
||||
environment:
|
||||
OLLAMA_HOST: http://ollama:11434
|
||||
MODEL: gemma3n:e4b
|
||||
depends_on:
|
||||
ollama:
|
||||
condition: service_started
|
||||
|
||||
volumes:
|
||||
ollama_data:
|
||||
@@ -0,0 +1,4 @@
|
||||
FROM ollama/ollama
|
||||
|
||||
USER root
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/*
|
||||
@@ -0,0 +1,22 @@
|
||||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
OLLAMA_HOST="${OLLAMA_HOST:-http://localhost:11434}"
|
||||
MODEL="${MODEL:-gemma3n:e4b}"
|
||||
MODEL_NAME="${MODEL%%:*}"
|
||||
|
||||
echo "Waiting for Ollama..."
|
||||
until curl -sf "${OLLAMA_HOST}/api/tags" > /dev/null 2>&1; do
|
||||
sleep 5
|
||||
done
|
||||
|
||||
echo "Waiting for model ${MODEL}..."
|
||||
elapsed=0
|
||||
until curl -sf "${OLLAMA_HOST}/api/tags" | grep -q "${MODEL_NAME}"; do
|
||||
echo " ${elapsed}s — загрузка..."
|
||||
sleep 10
|
||||
elapsed=$((elapsed + 10))
|
||||
done
|
||||
|
||||
echo "Ollama ready (модель загружена за ${elapsed}s)."
|
||||
exec "$@"
|
||||
@@ -1,6 +1,4 @@
|
||||
CHEVROLET Tacuma 1.6 [A16DMS] 09.2000-
|
||||
DAEWOO Rezzo/Tacuma 1.6 [A16DMS] 09.2000-
|
||||
DAEWOO Espero 1.5 16V [A15MF] 02.1995-
|
||||
CHEVROLET Tacuma 1.6 [A16DMS] 09.2000- DAEWOO Rezzo/Tacuma 1.6 [A16DMS] 09.2000- \n DAEWOO Espero 1.5 16V [A15MF] 02.1995-
|
||||
DAEWOO Kalos 1.4 [F14S3] 09.2002-
|
||||
DAEWOO Kalos 1.4 16V [F14D3] 04.2003-
|
||||
DAEWOO Lacetti 1.4 03.2004-01.2005
|
||||
|
||||
@@ -6,8 +6,10 @@
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
|
||||
from ollama import Client
|
||||
@@ -16,10 +18,11 @@ from ollama import Client
|
||||
LISTS_DIR = pathlib.Path(__file__).parent / "lists"
|
||||
RES_DIR = pathlib.Path(__file__).parent / "res"
|
||||
DONE_DIR = pathlib.Path(__file__).parent / "done"
|
||||
OLLAMA_HOST = "http://localhost:11434"
|
||||
MODEL = "mithunrajm06/gemma-3n-e4b:latest"
|
||||
OLLAMA_HOST = os.environ.get("OLLAMA_HOST", "http://localhost:11434")
|
||||
MODEL = os.environ.get("MODEL", "gemma3n:e4b")
|
||||
RETRY_COUNT = 3
|
||||
RETRY_DELAY = 5
|
||||
LOG_STREAM = os.environ.get("LOG_STREAM", "").lower() in ("1", "true", "yes")
|
||||
|
||||
SYSTEM_PROMPT = """You are a data processing assistant.
|
||||
The input text is a list of tab-separated lines: brand, model, engine_code, date_range.
|
||||
@@ -37,23 +40,36 @@ def process_file(client: Client, filepath: pathlib.Path) -> bool:
|
||||
for attempt in range(1, RETRY_COUNT + 1):
|
||||
try:
|
||||
t0 = time.perf_counter()
|
||||
response = client.chat(
|
||||
print(f" {filepath.name} — ", end="", flush=True)
|
||||
|
||||
result = ""
|
||||
stream = client.chat(
|
||||
model=MODEL,
|
||||
messages=[
|
||||
{"role": "system", "content": SYSTEM_PROMPT},
|
||||
{"role": "user", "content": f"Convert to JSON. Output only the JSON array:\n\n{content}"},
|
||||
],
|
||||
stream=True,
|
||||
)
|
||||
elapsed = time.perf_counter() - t0
|
||||
for chunk in stream:
|
||||
part = chunk.get("message", {}).get("content", "") or ""
|
||||
result += part
|
||||
if part:
|
||||
if LOG_STREAM:
|
||||
sys.stdout.write(part)
|
||||
else:
|
||||
sys.stdout.write(".")
|
||||
sys.stdout.flush()
|
||||
|
||||
result = response["message"]["content"]
|
||||
elapsed = time.perf_counter() - t0
|
||||
try:
|
||||
data = json.loads(result)
|
||||
out_path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
except json.JSONDecodeError:
|
||||
out_path.write_text(result, encoding="utf-8")
|
||||
|
||||
print(f" {filepath.name} — {elapsed:.1f}s — OK")
|
||||
print(f"\n" if LOG_STREAM else " ", end="")
|
||||
print(f" {len(result)} chars, {elapsed:.1f}s — OK")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
[{"brand": "HYUNDAI", "model": "Accent 1.3", "engine_code": "[G4E-A]", "date_range": "08.1999-"}, {"brand": "HYUNDAI", "model": "Accent 1.3", "engine_code": "[G4EH]", "date_range": "08.2002-"}, {"brand": "HYUNDAI", "model": "Accent 1.4 GL", "engine_code": "", "date_range": "11.2005-"}, {"brand": "HYUNDAI", "model": "Accent 1.5", "engine_code": "[G4EB]", "date_range": "08.2002-"}, {"brand": "HYUNDAI", "model": "Accent 1.6", "engine_code": "[G4ED-G]", "date_range": "12.2002-"}, {"brand": "HYUNDAI", "model": "Accent 1.6 GLS", "engine_code": "", "date_range": "11.2005-"}, {"brand": "HYUNDAI", "model": "Coupe 1.6 16V", "engine_code": "[G4ED-G]", "date_range": "03.2002-"}, {"brand": "HYUNDAI", "model": "Coupe 2.0 16V", "engine_code": "[G4GF]", "date_range": "08.2002-"}, {"brand": "HYUNDAI", "model": "Coupe 2.0 GLS", "engine_code": "", "date_range": "08.2002-"}, {"brand": "KIA", "model": "Rio 1.6 16V", "engine_code": "[G4ED]", "date_range": "03.2005-}]
|
||||
@@ -0,0 +1,13 @@
|
||||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
docker compose up -d --build ollama
|
||||
echo "Ожидание загрузки модели..."
|
||||
elapsed=0
|
||||
until docker exec ollama ollama list 2>/dev/null | grep -q gemma3n; do
|
||||
echo " ${elapsed}s — загрузка..."
|
||||
sleep 10
|
||||
elapsed=$((elapsed + 10))
|
||||
done
|
||||
echo "Модель загружена (${elapsed}s)."
|
||||
docker compose run --rm app
|
||||
Reference in New Issue
Block a user