From 5defeaefd45efe6cbd79afa7bf46a6c8d69af00c Mon Sep 17 00:00:00 2001 From: atarasov Date: Thu, 12 Mar 2026 14:52:36 +0300 Subject: [PATCH] 0.2 - docker --- .dockerignore | 6 ++++ Dockerfile | 16 ++++++++++ README.md | 27 +++++++++++++++++ docker-compose.yml | 37 +++++++++++++++++++++++ docker/Dockerfile.ollama | 4 +++ docker/app-entrypoint.sh | 22 ++++++++++++++ done/example-long.txt | 4 +-- done/short.txt => lists/example-short.txt | 0 parse.py | 28 +++++++++++++---- res/short.json | 1 - run.sh | 13 ++++++++ 11 files changed, 148 insertions(+), 10 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 docker-compose.yml create mode 100644 docker/Dockerfile.ollama create mode 100644 docker/app-entrypoint.sh rename done/short.txt => lists/example-short.txt (100%) delete mode 100644 res/short.json create mode 100755 run.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..a32a3d8 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +.venv +__pycache__ +.git +*.pyc +res/ +done/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c3c6d11 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.12-slim + +RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY parse.py . +COPY lists/ lists/ +COPY docker/app-entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] +CMD ["python", "parse.py"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..6e9f8f3 --- /dev/null +++ b/README.md @@ -0,0 +1,27 @@ +# Папки +- `lists/` — необработанные .txt файлы +- `done/` — обработанные файлы (переносятся из lists/) +- `res/` — результат в JSON + +# Запуск + +## Docker +```bash +chmod +x run.sh && ./run.sh +``` + +```bash +# Статус загрузки и работы модели в отдельном терминале +docker compose logs -f ollama +``` + +При первом запуске Ollama скачает модель `gemma3n:e4b` (7.5 GB) (может занять несколько минут). + +## Локально +```bash +pip install -r requirements.txt +python3 parse.py +``` + + + diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..992ce67 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,37 @@ +services: + ollama: + build: + context: . + dockerfile: docker/Dockerfile.ollama + container_name: ollama + volumes: + - ollama_data:/root/.ollama + ports: + - "11435:11434" # 11435 — если локальный Ollama уже на 11434 + environment: + OLLAMA_MODEL: gemma3n:e4b + entrypoint: ["/bin/sh", "-c"] + command: + - | + ollama serve & + sleep 5 + until curl -sf http://localhost:11434/api/tags > /dev/null 2>&1; do sleep 2; done + ollama pull $${OLLAMA_MODEL:-gemma3n:e4b} + wait + + app: + build: . + container_name: cart-to-json + volumes: + - ./lists:/app/lists + - ./res:/app/res + - ./done:/app/done + environment: + OLLAMA_HOST: http://ollama:11434 + MODEL: gemma3n:e4b + depends_on: + ollama: + condition: service_started + +volumes: + ollama_data: diff --git a/docker/Dockerfile.ollama b/docker/Dockerfile.ollama new file mode 100644 index 0000000..287f2ee --- /dev/null +++ b/docker/Dockerfile.ollama @@ -0,0 +1,4 @@ +FROM ollama/ollama + +USER root +RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/* diff --git a/docker/app-entrypoint.sh b/docker/app-entrypoint.sh new file mode 100644 index 0000000..0b85a68 --- /dev/null +++ b/docker/app-entrypoint.sh @@ -0,0 +1,22 @@ +#!/bin/sh +set -e + +OLLAMA_HOST="${OLLAMA_HOST:-http://localhost:11434}" +MODEL="${MODEL:-gemma3n:e4b}" +MODEL_NAME="${MODEL%%:*}" + +echo "Waiting for Ollama..." +until curl -sf "${OLLAMA_HOST}/api/tags" > /dev/null 2>&1; do + sleep 5 +done + +echo "Waiting for model ${MODEL}..." +elapsed=0 +until curl -sf "${OLLAMA_HOST}/api/tags" | grep -q "${MODEL_NAME}"; do + echo " ${elapsed}s — загрузка..." + sleep 10 + elapsed=$((elapsed + 10)) +done + +echo "Ollama ready (модель загружена за ${elapsed}s)." +exec "$@" diff --git a/done/example-long.txt b/done/example-long.txt index 3595687..ce346fe 100644 --- a/done/example-long.txt +++ b/done/example-long.txt @@ -1,6 +1,4 @@ -CHEVROLET Tacuma 1.6 [A16DMS] 09.2000- -DAEWOO Rezzo/Tacuma 1.6 [A16DMS] 09.2000- -DAEWOO Espero 1.5 16V [A15MF] 02.1995- +CHEVROLET Tacuma 1.6 [A16DMS] 09.2000- DAEWOO Rezzo/Tacuma 1.6 [A16DMS] 09.2000- \n DAEWOO Espero 1.5 16V [A15MF] 02.1995- DAEWOO Kalos 1.4 [F14S3] 09.2002- DAEWOO Kalos 1.4 16V [F14D3] 04.2003- DAEWOO Lacetti 1.4 03.2004-01.2005 diff --git a/done/short.txt b/lists/example-short.txt similarity index 100% rename from done/short.txt rename to lists/example-short.txt diff --git a/parse.py b/parse.py index f087418..089127e 100644 --- a/parse.py +++ b/parse.py @@ -6,8 +6,10 @@ """ import json +import os import pathlib import shutil +import sys import time from ollama import Client @@ -16,10 +18,11 @@ from ollama import Client LISTS_DIR = pathlib.Path(__file__).parent / "lists" RES_DIR = pathlib.Path(__file__).parent / "res" DONE_DIR = pathlib.Path(__file__).parent / "done" -OLLAMA_HOST = "http://localhost:11434" -MODEL = "mithunrajm06/gemma-3n-e4b:latest" +OLLAMA_HOST = os.environ.get("OLLAMA_HOST", "http://localhost:11434") +MODEL = os.environ.get("MODEL", "gemma3n:e4b") RETRY_COUNT = 3 RETRY_DELAY = 5 +LOG_STREAM = os.environ.get("LOG_STREAM", "").lower() in ("1", "true", "yes") SYSTEM_PROMPT = """You are a data processing assistant. The input text is a list of tab-separated lines: brand, model, engine_code, date_range. @@ -37,23 +40,36 @@ def process_file(client: Client, filepath: pathlib.Path) -> bool: for attempt in range(1, RETRY_COUNT + 1): try: t0 = time.perf_counter() - response = client.chat( + print(f" {filepath.name} — ", end="", flush=True) + + result = "" + stream = client.chat( model=MODEL, messages=[ {"role": "system", "content": SYSTEM_PROMPT}, {"role": "user", "content": f"Convert to JSON. Output only the JSON array:\n\n{content}"}, ], + stream=True, ) - elapsed = time.perf_counter() - t0 + for chunk in stream: + part = chunk.get("message", {}).get("content", "") or "" + result += part + if part: + if LOG_STREAM: + sys.stdout.write(part) + else: + sys.stdout.write(".") + sys.stdout.flush() - result = response["message"]["content"] + elapsed = time.perf_counter() - t0 try: data = json.loads(result) out_path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") except json.JSONDecodeError: out_path.write_text(result, encoding="utf-8") - print(f" {filepath.name} — {elapsed:.1f}s — OK") + print(f"\n" if LOG_STREAM else " ", end="") + print(f" {len(result)} chars, {elapsed:.1f}s — OK") return True except Exception as e: diff --git a/res/short.json b/res/short.json deleted file mode 100644 index a318a7e..0000000 --- a/res/short.json +++ /dev/null @@ -1 +0,0 @@ -[{"brand": "HYUNDAI", "model": "Accent 1.3", "engine_code": "[G4E-A]", "date_range": "08.1999-"}, {"brand": "HYUNDAI", "model": "Accent 1.3", "engine_code": "[G4EH]", "date_range": "08.2002-"}, {"brand": "HYUNDAI", "model": "Accent 1.4 GL", "engine_code": "", "date_range": "11.2005-"}, {"brand": "HYUNDAI", "model": "Accent 1.5", "engine_code": "[G4EB]", "date_range": "08.2002-"}, {"brand": "HYUNDAI", "model": "Accent 1.6", "engine_code": "[G4ED-G]", "date_range": "12.2002-"}, {"brand": "HYUNDAI", "model": "Accent 1.6 GLS", "engine_code": "", "date_range": "11.2005-"}, {"brand": "HYUNDAI", "model": "Coupe 1.6 16V", "engine_code": "[G4ED-G]", "date_range": "03.2002-"}, {"brand": "HYUNDAI", "model": "Coupe 2.0 16V", "engine_code": "[G4GF]", "date_range": "08.2002-"}, {"brand": "HYUNDAI", "model": "Coupe 2.0 GLS", "engine_code": "", "date_range": "08.2002-"}, {"brand": "KIA", "model": "Rio 1.6 16V", "engine_code": "[G4ED]", "date_range": "03.2005-}] \ No newline at end of file diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..2866bfd --- /dev/null +++ b/run.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -e + +docker compose up -d --build ollama +echo "Ожидание загрузки модели..." +elapsed=0 +until docker exec ollama ollama list 2>/dev/null | grep -q gemma3n; do + echo " ${elapsed}s — загрузка..." + sleep 10 + elapsed=$((elapsed + 10)) +done +echo "Модель загружена (${elapsed}s)." +docker compose run --rm app