0.2 - docker

.dockerignore

@@ -0,0 +1,6 @@
+.venv
+__pycache__
+.git
+*.pyc
+res/
+done/

Dockerfile

@@ -0,0 +1,16 @@
+FROM python:3.12-slim
+
+RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY parse.py .
+COPY lists/ lists/
+COPY docker/app-entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["python", "parse.py"]

README.md

@@ -0,0 +1,27 @@
+# Папки
+- `lists/` — необработанные .txt файлы
+- `done/` — обработанные файлы (переносятся из lists/)
+- `res/` — результат в JSON
+
+# Запуск
+
+## Docker
+```bash
+chmod +x run.sh && ./run.sh
+```
+
+```bash
+# Статус загрузки и работы модели в отдельном терминале
+docker compose logs -f ollama
+``` 
+
+При первом запуске Ollama скачает модель `gemma3n:e4b` (7.5 GB) (может занять несколько минут).
+
+## Локально
+```bash
+pip install -r requirements.txt
+python3 parse.py
+```
+
+
+

docker-compose.yml

@@ -0,0 +1,37 @@
+services:
+  ollama:
+    build:
+      context: .
+      dockerfile: docker/Dockerfile.ollama
+    container_name: ollama
+    volumes:
+      - ollama_data:/root/.ollama
+    ports:
+      - "11435:11434"  # 11435 — если локальный Ollama уже на 11434
+    environment:
+      OLLAMA_MODEL: gemma3n:e4b
+    entrypoint: ["/bin/sh", "-c"]
+    command:
+      - |
+        ollama serve &
+        sleep 5
+        until curl -sf http://localhost:11434/api/tags > /dev/null 2>&1; do sleep 2; done
+        ollama pull $${OLLAMA_MODEL:-gemma3n:e4b}
+        wait
+
+  app:
+    build: .
+    container_name: cart-to-json
+    volumes:
+      - ./lists:/app/lists
+      - ./res:/app/res
+      - ./done:/app/done
+    environment:
+      OLLAMA_HOST: http://ollama:11434
+      MODEL: gemma3n:e4b
+    depends_on:
+      ollama:
+        condition: service_started
+
+volumes:
+  ollama_data:

docker/Dockerfile.ollama

@@ -0,0 +1,4 @@
+FROM ollama/ollama
+
+USER root
+RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/*

docker/app-entrypoint.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+set -e
+
+OLLAMA_HOST="${OLLAMA_HOST:-http://localhost:11434}"
+MODEL="${MODEL:-gemma3n:e4b}"
+MODEL_NAME="${MODEL%%:*}"
+
+echo "Waiting for Ollama..."
+until curl -sf "${OLLAMA_HOST}/api/tags" > /dev/null 2>&1; do
+  sleep 5
+done
+
+echo "Waiting for model ${MODEL}..."
+elapsed=0
+until curl -sf "${OLLAMA_HOST}/api/tags" | grep -q "${MODEL_NAME}"; do
+  echo "  ${elapsed}s — загрузка..."
+  sleep 10
+  elapsed=$((elapsed + 10))
+done
+
+echo "Ollama ready (модель загружена за ${elapsed}s)."
+exec "$@"

done/example-long.txt

@@ -1,6 +1,4 @@
-CHEVROLET	Tacuma 1.6	[A16DMS]	09.2000-
+CHEVROLET	Tacuma 1.6	[A16DMS]	09.2000- DAEWOO	Rezzo/Tacuma 1.6	[A16DMS]	09.2000- \n DAEWOO	Espero 1.5 16V	[A15MF]	02.1995-	
-DAEWOO	Rezzo/Tacuma 1.6	[A16DMS]	09.2000-
-DAEWOO	Espero 1.5 16V	[A15MF]	02.1995-	
 DAEWOO	Kalos 1.4	[F14S3]	09.2002-	
 DAEWOO	Kalos 1.4 16V	[F14D3]	04.2003-	
 DAEWOO	Lacetti 1.4		03.2004-01.2005	

parse.py

@@ -6,8 +6,10 @@
 """
 
 import json
+import os
 import pathlib
 import shutil
+import sys
 import time
 
 from ollama import Client
@@ -16,10 +18,11 @@ from ollama import Client
 LISTS_DIR = pathlib.Path(__file__).parent / "lists"
 RES_DIR = pathlib.Path(__file__).parent / "res"
 DONE_DIR = pathlib.Path(__file__).parent / "done"
-OLLAMA_HOST = "http://localhost:11434"
+OLLAMA_HOST = os.environ.get("OLLAMA_HOST", "http://localhost:11434")
-MODEL = "mithunrajm06/gemma-3n-e4b:latest"
+MODEL = os.environ.get("MODEL", "gemma3n:e4b")
 RETRY_COUNT = 3
 RETRY_DELAY = 5
+LOG_STREAM = os.environ.get("LOG_STREAM", "").lower() in ("1", "true", "yes")
 
 SYSTEM_PROMPT = """You are a data processing assistant.
 The input text is a list of tab-separated lines: brand, model, engine_code, date_range.
@@ -37,23 +40,36 @@ def process_file(client: Client, filepath: pathlib.Path) -> bool:
     for attempt in range(1, RETRY_COUNT + 1):
         try:
             t0 = time.perf_counter()
-            response = client.chat(
+            print(f"  {filepath.name} — ", end="", flush=True)
+
+            result = ""
+            stream = client.chat(
                 model=MODEL,
                 messages=[
                     {"role": "system", "content": SYSTEM_PROMPT},
                     {"role": "user", "content": f"Convert to JSON. Output only the JSON array:\n\n{content}"},
                 ],
+                stream=True,
             )
-            elapsed = time.perf_counter() - t0
+            for chunk in stream:
+                part = chunk.get("message", {}).get("content", "") or ""
+                result += part
+                if part:
+                    if LOG_STREAM:
+                        sys.stdout.write(part)
+                    else:
+                        sys.stdout.write(".")
+                    sys.stdout.flush()
 
-            result = response["message"]["content"]
+            elapsed = time.perf_counter() - t0
             try:
                 data = json.loads(result)
                 out_path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
             except json.JSONDecodeError:
                 out_path.write_text(result, encoding="utf-8")
 
-            print(f"  {filepath.name} — {elapsed:.1f}s — OK")
+            print(f"\n" if LOG_STREAM else " ", end="")
+            print(f" {len(result)} chars, {elapsed:.1f}s — OK")
             return True
 
         except Exception as e:

res/short.json

@@ -1 +0,0 @@
-[{"brand": "HYUNDAI", "model": "Accent 1.3", "engine_code": "[G4E-A]", "date_range": "08.1999-"}, {"brand": "HYUNDAI", "model": "Accent 1.3", "engine_code": "[G4EH]", "date_range": "08.2002-"}, {"brand": "HYUNDAI", "model": "Accent 1.4 GL", "engine_code": "", "date_range": "11.2005-"}, {"brand": "HYUNDAI", "model": "Accent 1.5", "engine_code": "[G4EB]", "date_range": "08.2002-"}, {"brand": "HYUNDAI", "model": "Accent 1.6", "engine_code": "[G4ED-G]", "date_range": "12.2002-"}, {"brand": "HYUNDAI", "model": "Accent 1.6 GLS", "engine_code": "", "date_range": "11.2005-"}, {"brand": "HYUNDAI", "model": "Coupe 1.6 16V", "engine_code": "[G4ED-G]", "date_range": "03.2002-"}, {"brand": "HYUNDAI", "model": "Coupe 2.0 16V", "engine_code": "[G4GF]", "date_range": "08.2002-"}, {"brand": "HYUNDAI", "model": "Coupe 2.0 GLS", "engine_code": "", "date_range": "08.2002-"}, {"brand": "KIA", "model": "Rio 1.6 16V", "engine_code": "[G4ED]", "date_range": "03.2005-}]

run.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+set -e
+
+docker compose up -d --build ollama
+echo "Ожидание загрузки модели..."
+elapsed=0
+until docker exec ollama ollama list 2>/dev/null | grep -q gemma3n; do
+  echo "  ${elapsed}s — загрузка..."
+  sleep 10
+  elapsed=$((elapsed + 10))
+done
+echo "Модель загружена (${elapsed}s)."
+docker compose run --rm app