From 53c429c5bca9ccb6192a324d4033269b257690da Mon Sep 17 00:00:00 2001 From: atarasov Date: Thu, 12 Mar 2026 11:56:58 +0300 Subject: [PATCH] 0.1 - init --- .gitignore | 1 + lists/cars-list-1.txt | 176 ++++++++++++++++++++++++++++++++++++++++++ lists/cars-list-2.txt | 25 ++++++ parse.py | 95 +++++++++++++++++++++++ parse_local.py | 42 ++++++++++ requirements.txt | 1 + 6 files changed, 340 insertions(+) create mode 100644 .gitignore create mode 100644 lists/cars-list-1.txt create mode 100644 lists/cars-list-2.txt create mode 100644 parse.py create mode 100644 parse_local.py create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b694934 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.venv \ No newline at end of file diff --git a/lists/cars-list-1.txt b/lists/cars-list-1.txt new file mode 100644 index 0000000..3595687 --- /dev/null +++ b/lists/cars-list-1.txt @@ -0,0 +1,176 @@ +CHEVROLET Tacuma 1.6 [A16DMS] 09.2000- +DAEWOO Rezzo/Tacuma 1.6 [A16DMS] 09.2000- +DAEWOO Espero 1.5 16V [A15MF] 02.1995- +DAEWOO Kalos 1.4 [F14S3] 09.2002- +DAEWOO Kalos 1.4 16V [F14D3] 04.2003- +DAEWOO Lacetti 1.4 03.2004-01.2005 +DAEWOO Lacetti 1.4 16V [F14D3] 03.2005- +DAEWOO Lacetti 1.6 [F16D3] 02.2004- +DAEWOO Lacetti 1.8 [T18SED] 08.2005- +DAEWOO Lanos 1.4 [A14SMS] 05.1997- +DAEWOO Lanos 1.5 [A15SMS] 05.1997- +DAEWOO Lanos 1.6 16V [A16DMS] 05.1997- +DAEWOO Nexia 1.5 [G15MF] 02.1995-08.1997 +DAEWOO Nexia 1.5 16V [A15MF] 02.1995-08.1997 +DAEWOO Nubira 1.6 [F16D3] 07.2003- +DAEWOO Nubira 1.6 16V [A16DMS] 05.1997- +DAEWOO Nubira 1.8 [T18SED] 07.2003- +DAEWOO Nubira 2.0 16V [T20SED] 12.2000- +DAEWOO Rezzo 1.6 08.2005- +DAEWOO Rezzo/Tacuma 1.8 [A18DMS] 09.2000-01.2005 +OPEL Agila 1.0 i 12V [Z10XE] 09.2000-08.2003 +OPEL Agila 1.2 16V Twinport [Z12XEP] 07.2004- +OPEL Agila 1.2 i 16V [Z12XE] 09.2000-06.2004 +OPEL Ascona C 1.6 [16SV] 09.1986-08.1988 +OPEL Ascona C 1.6 i [C16LZ] 09.1986-08.1987 +OPEL Ascona C 1.6 i [C16NZ] 09.1987-08.1988 +OPEL Astra F 1.4 i [C14NZ] 09.1991-01.1998 +OPEL Astra F 1.4 i [C14NZ][C14SE] 03.1992-09.1998 +OPEL Astra F 1.4 i [X14NZ] 09.1997-08.2000 +OPEL Astra F 1.4 i 16V [X14NZ] 09.1997-08.2000 +OPEL Astra F 1.4 i 16V [X14XE] 04.1996-02.1998 +OPEL Astra F 1.4 Si [C14SE] 03.1992-01.1998 +OPEL Astra F 1.6 i [16NZR] 09.1995-01.1998 +OPEL Astra F 1.6 i [C16NZ] 09.1991-08.1994 +OPEL Astra F 1.6 i [C16NZ][C16SE] 03.1992-02.1998 +OPEL Astra F 1.6 i [C16NZ][C16SE][X16SZ] 02.1993-07.1996 +OPEL Astra F 1.6 i [C16NZ][C16SE][X16SZ][X16SZR] 08.1996-01.1998 +OPEL Astra F 1.6 i 16V [X16XEL] 09.1994-08.2000 +OPEL Astra G 1.2 16V [X12XE] 09.1997-09.2003 +OPEL Astra G 1.2 i 16V [X12XE] 09.1997-09.2003 +OPEL Astra G 1.4 i 16V [X14XE] 09.1997-09.2000 +OPEL Astra G 1.4 i 16V [X14XE][Z14XE] 09.2000-01.2004 +OPEL Astra G 1.6 [Z16XEP] 03.2003-08.2005 +OPEL Astra G 1.6 16V [X16XEL] 09.1997-08.2000 +OPEL Astra G 1.6 16V [X16XEL][Z16XEP] 03.2000-02.2005 +OPEL Astra G 1.6 i [X16SZR] 02.1998-09.2000 +OPEL Astra G 1.6 i 16V [X16XEL] 09.1997-08.2000 +OPEL Astra G 1.6 i 16V [X16XEL][Z16XE] 09.1998-01.2005 +OPEL Astra G 1.6 Twinport [Z16XEP] 03.2003-01.2004 +OPEL Astra H 1.4 [Z14XEP] 03.2004- +OPEL Astra H 1.4 Twinport GTC [Z14XEP] 03.2005- +OPEL Combo 1.2 i [C12NZ] 07.1994-09.1997 +OPEL Combo 1.2 i [X12SZ] 07.1994-08.1997 +OPEL Combo 1.4 [C14NZ] 07.1994-03.1996 +OPEL Combo 1.4 16V [X14XE] 08.1999-10.2001 +OPEL Combo 1.4 16V [Z14XEP] 08.2005- +OPEL Combo 1.4 i [C14SE] 07.1994-08.1997 +OPEL Combo 1.4 Tour [Z14XEP] 10.2004- +OPEL Corsa A 1.2 [12NV] 09.1986-03.1993 +OPEL Corsa A 1.2 [12ST] 02.1986-04.1989 +OPEL Corsa A 1.2 i [C12NZ] 09.1989-03.1993 +OPEL Corsa A 1.2 S [12ST] 09.1982-03.1993 +OPEL Corsa A 1.2 S [12S] 09.1982-08.1994 +OPEL Corsa A 1.3 [13NB] 03.1986-10.1999 +OPEL Corsa A 1.3 [13SB] 09.1982-03.1993 +OPEL Corsa A 1.3 i [13NB] 09.1985-03.1993 +OPEL Corsa A 1.3 i [C13N] 09.1985-12.1992 +OPEL Corsa A 1.3 S [13SB] 09.1982-09.1994 +OPEL Corsa A 1.4 i [C14NZ] 01.1990-03.1993 +OPEL Corsa A 1.4 i [C14SE] 09.1991-03.1993 +OPEL Corsa A 1.4 S [14NV] 09.1989-03.1993 +OPEL Corsa A 1.6 GSi [C16NZ] 08.1988-09.1992 +OPEL Corsa A 1.6 GSi [C16SEI] 09.1989-03.1993 +OPEL Corsa A 1.6 GSi [C16SE] 05.1988-03.1993 +OPEL Corsa B 1.0 i 12V [X10XE] 09.1996-08.1999 +OPEL Corsa B 1.2 i [12NZ] 03.1993-09.2000 +OPEL Corsa B 1.2 i 16V [X12XE] 09.1997-09.2000 +OPEL Corsa B 1.4 i [C14NZ] 03.1993-09.2000 +OPEL Corsa B 1.4 i [X14SZ] 08.1999-09.2000 +OPEL Corsa B 1.4 i 16V [C14SEL] 08.1997-09.2000 +OPEL Corsa B 1.4 i 16V [X14XE] 09.1994-08.1999 +OPEL Corsa B 1.4 Si [C14SE] 03.1993-09.2000 +OPEL Corsa B 1.6 GSi 16V [C16XE] 03.1993-09.2000 +OPEL Corsa B 1.6 i 16V [C16SEL] 08.1997-09.2000 +OPEL Corsa B 1.6 i 16V [X16XE] 03.1993-09.2000 +OPEL Corsa C 1.0 i 12V [Z10XE] 09.2000- +OPEL Corsa C 1.0 Twinport [Z10XEP] 06.2003- +OPEL Corsa C 1.2 i 16V [Z12XE] 09.2000- +OPEL Corsa C 1.2 Twinport [Z12XEP] 07.2004- +OPEL Corsa C 1.4 i 16V [Z14XEP] 09.2003- +OPEL Corsa C 1.4 i 16V [Z14XE] 09.2000- +OPEL Corsa C 1.4 Twinport [Z14XEP] 06.2003- +OPEL Corsa D 1.0 [Z10XEP] 07.2006- +OPEL Corsa D 1.2 [Z12XEP] 07.2006- +OPEL Corsa D 1.4 [Z14XEP] 07.2006- +OPEL Kadett E 1.4 i [C14NZ] 01.1990-08.1991 +OPEL Kadett E 1.6 [16SV] 09.1986-08.1991 +OPEL Kadett E 1.6 Combo [16SV] 09.1986-12.1991 +OPEL Kadett E 1.6 i [C16LZ] 09.1988-08.1991 +OPEL Kadett E 1.6 S [16SV] 09.1986-08.1991 +OPEL Meriva 1.4 16V Twinport [Z14XEP] 07.2004- +OPEL Meriva 1.6 [Z16XEP] 01.2006- +OPEL Tigra 1.4 i [Z14XEP] 06.2004- +OPEL Tigra 1.4 i 16V [X14XE] 07.1994-08.1999 +OPEL Tigra 1.6 i 16V [C16SEL] 08.1997- +OPEL Tigra 1.6 i 16V [X16XE] 07.1994-09.1997 +OPEL Vectra A 1.4 S [14NV] 04.1988-06.1992 +OPEL Vectra A 1.6 [16SV] 09.1988-08.1995 +OPEL Vectra A 1.6 i [16LZ2] 09.1993-11.1995 +OPEL Vectra A 1.6 i [C16NZ2] 09.1993-08.1995 +OPEL Vectra A 1.6 i [C16NZ] 09.1988-08.1995 +OPEL Vectra A 1.6 S [16SV] 09.1988-08.1995 +OPEL Vectra B 1.6 i [X16SZR] 09.1996-07.2003 +OPEL Vectra B 1.6 i 16V [X16XEL] 09.1995-07.2002 +OPEL Vectra B 1.6 i 16V [Y16XE] 09.1995-09.2002 +OPEL Vectra B 1.6 i 16V [Z16XE] 09.2000-09.2002 +OPEL Vectra C 1.6 [Z16XEP] 01.2006- +OPEL Vectra C 1.6 GTS [Z16XEP] 01.2006- +OPEL Zafira A 1.6 16V [X16XEL] 04.1999-07.2005 +OPEL Zafira A 1.6 i [Z16YNG] 09.2001- +OPEL Zafira B 1.6 [Z16XEP] 07.2005- +VAUXHALL Agila 1.0 12V [Z10XE] 09.1999-08.2003 +VAUXHALL Agila 1.2 16V Twinport [Z12XEP] 07.2004- +VAUXHALL Astra E 1.4 i [C14NZ] 01.1990-08.1991 +VAUXHALL Astra E 1.4 S [14NV] 01.1990-08.1991 +VAUXHALL Astra F 1.4 i [C14NZ] 09.1991-09.1998 +VAUXHALL Astra F 1.4 i [C14SE] 03.1991-02.1998 +VAUXHALL Astra F 1.4 i [X14XE] 01.1996-03.2001 +VAUXHALL Astra F 1.4 i 16V [X14XE] 02.1996-09.1998 +VAUXHALL Astra F 1.4 S [14NV] 09.1991-02.1998 +VAUXHALL Astra F 1.6 i [C16NZ] 09.1991-02.1998 +VAUXHALL Astra F 1.6 i [C16SE] 03.1992-02.1998 +VAUXHALL Astra F 1.6 i [X16SZR] 03.1992-08.1997 +VAUXHALL Astra F 1.6 i [X16SZ] 07.1994-02.1998 +VAUXHALL Astra G 1.2 16V [X12XE] 09.1997-09.2000 +VAUXHALL Astra G 1.4 16V [X14XE] 09.1997-08.2000 +VAUXHALL Astra G 1.4 16V [Z14XE] 09.2000-08.2005 +VAUXHALL Astra G 1.6 [X16SZR] 09.1997-08.2005 +VAUXHALL Astra G 1.6 16V [X16XEL] 09.1997-08.2000 +VAUXHALL Astra G 1.6 16V [Z16XE] 09.2000-08.2005 +VAUXHALL Astra MK III 1.4 i 16V [X14XE] 02.1996-02.1998 +VAUXHALL Astra MK III 1.6 i 16V [X16XEL] 01.1994-02.1998 +VAUXHALL Astra MK IV 1.2 16V [Z12XE] 09.2000-08.2005 +VAUXHALL Astra MK IV 1.6 [Z16SE] 09.2000-08.2005 +VAUXHALL Astra MK IV 1.6 16V [Z16XE] 09.2000- +VAUXHALL Astra MK V 1.4 [Z14XEP] 08.2004- +VAUXHALL Astravan MK IV 1.6 i [X16SZR] 08.1998-09.2000 +VAUXHALL Astravan MK IV 1.6 i [Z16SE] 09.2000- +VAUXHALL Belmont 1.6 [16SH] 00.1988-09.1993 +VAUXHALL Cavalier 1.4 S [14NV] 09.1988-11.1995 +VAUXHALL Cavalier 1.6 [16SV] 09.1988-05.1993 +VAUXHALL Cavalier 1.6 i [X16SZ] 09.1994-11.1995 +VAUXHALL Combo 1.2 [C12NZ] 07.1994-08.1997 +VAUXHALL Combo 1.4 [C14NZ] 07.1994-10.2001 +VAUXHALL Combo 1.4 [Z14XEP] 01.2004- +VAUXHALL Combo 1.6 [Z16SE] 10.2001- +VAUXHALL Corsa 1.4 i 16V [X14XE] 09.1994-08.1999 +VAUXHALL Corsa B 1.0 i 12V [X10XE] 09.1996-09.2000 +VAUXHALL Corsa B 1.2 i 16V [X12XE] 03.1998-09.2000 +VAUXHALL Corsa B 1.4 i [C14NZ] 03.1993-09.2000 +VAUXHALL Corsa B 1.4 i [C14SE] 04.1994-09.2000 +VAUXHALL Corsa B 1.4 i [X14SZ] 09.1996-08.1998 +VAUXHALL Corsa B 1.4 i 16V [X14XE] 09.1994-09.2000 +VAUXHALL Corsa B 1.6 GSi 16V [C16XE] 09.1994-09.2000 +VAUXHALL Corsa B 1.6 i 16V [X16XE] 09.1994-09.2000 +VAUXHALL Corsa MK II 1.0 12V [Z10XE] 09.2000-08.2003 +VAUXHALL Corsa MK II 1.2 16V [Z12XE] 09.2000- +VAUXHALL Corsa MK II 1.2 Dualfuel 16V [Z12XE] 08.2001-08.2004 +VAUXHALL Corsa MK II 1.4 16V [Z14XE] 09.2000- +VAUXHALL Corsa Mk III 1.2 [Z12XEP] 07.2006- +VAUXHALL Corsa Mk III 1.4 [Z14XEP] 07.2006- +VAUXHALL Corsavan MK II 1.2 Dualfuel 16V [Z12XE] 08.2001- +VAUXHALL Meriva 1.4 16V Twinport [Z14XEP] 07.2004- +VAUXHALL Meriva 1.6 i 16V Twinport [Z16XEP] 01.2006- +VAUXHALL Nova 1.4 S [14NV] 01.1990-02.1993 +VAUXHALL Nova A 1.2 i [C12NZ] 0 \ No newline at end of file diff --git a/lists/cars-list-2.txt b/lists/cars-list-2.txt new file mode 100644 index 0000000..0c1299f --- /dev/null +++ b/lists/cars-list-2.txt @@ -0,0 +1,25 @@ +HYUNDAI Accent 1.3 [G4E-A] 08.1999- +HYUNDAI Accent 1.3 [G4EH] 08.2002- +HYUNDAI Accent 1.4 GL 11.2005- +HYUNDAI Accent 1.5 [G4EB] 08.2002- +HYUNDAI Accent 1.6 [G4ED-G] 12.2002- +HYUNDAI Accent 1.6 GLS 11.2005- +HYUNDAI Coupe 1.6 16V [G4ED-G] 03.2002- +HYUNDAI Coupe 2.0 16V [G4GF] 08.2002- +HYUNDAI Coupe 2.0 GLS 08.2002- +HYUNDAI Elantra 1.6 [G4ED-G] 06.2000- +HYUNDAI Elantra 1.6 [G4GA] 09.2003- +HYUNDAI Elantra 1.8 [G4GB] 11.2002- +HYUNDAI Elantra 2.0 [G4GC-G] 10.2003- AC +HYUNDAI Getz 1.3 [G4E-A] 09.2002- +HYUNDAI Getz 1.4 [G4EE-G] 08.2005- +HYUNDAI Getz 1.6 [G4ED-G] 09.2002- +HYUNDAI Matrix 1.6 [G4ED-G] 06.2001- +HYUNDAI Matrix 1.8 [G4GB-G] 06.2001- +HYUNDAI Tucson 2.0 04.2004- +HYUNDAI Tucson 2.0 4WD 04.2004- +KIA Carens 2.0 08.2004- +KIA Cerato 1.6 [G4ED] 04.2004- +KIA Cerato 2.0 [G4GC] 04.2004- +KIA Rio 1.4 16V [G4EE] 03.2005- +KIA Rio 1.6 16V [G4ED] 03.2005- \ No newline at end of file diff --git a/parse.py b/parse.py new file mode 100644 index 0000000..156adb7 --- /dev/null +++ b/parse.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +""" +Скрипт загружает .txt файлы из lists/, отправляет каждый в локальную Ollama +с системным промптом и сохраняет ответ в res/. Успешные файлы переносятся в done/. +При падении можно перезапустить — необработанные остаются в lists/. +""" + +import json +import pathlib +import shutil +import time + +from ollama import Client + +# --- Конфигурация --- +LISTS_DIR = pathlib.Path(__file__).parent / "lists" +RES_DIR = pathlib.Path(__file__).parent / "res" +DONE_DIR = pathlib.Path(__file__).parent / "done" +OLLAMA_HOST = "http://localhost:11434" +MODEL = "llama3.1:8b" +RETRY_COUNT = 3 +RETRY_DELAY = 5 + +SYSTEM_PROMPT = """You are a data processing assistant. +The input text is a list of tab-separated lines: brand, model, engine_code, date_range. +Convert to a JSON array of objects with fields: brand, model, engine_code, date_range. +The number of elements in JSON MUST match the number of lines in the list. +Keep empty fields as empty string "". +Output ONLY the JSON array. No explanations, no markdown, no other text before or after.""" + + +def process_file(client: Client, filepath: pathlib.Path) -> bool: + """Обрабатывает один файл. Возвращает True при успехе.""" + content = filepath.read_text(encoding="utf-8") + out_path = RES_DIR / filepath.with_suffix(".json").name + + for attempt in range(1, RETRY_COUNT + 1): + try: + t0 = time.perf_counter() + response = client.chat( + model=MODEL, + messages=[ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": f"Convert to JSON. Output only the JSON array:\n\n{content}"}, + ], + ) + elapsed = time.perf_counter() - t0 + + result = response["message"]["content"] + try: + data = json.loads(result) + out_path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") + except json.JSONDecodeError: + out_path.write_text(result, encoding="utf-8") + + print(f" {filepath.name} — {elapsed:.1f}s — OK") + return True + + except Exception as e: + print(f" {filepath.name} — попытка {attempt}/{RETRY_COUNT} — ошибка: {e}") + if attempt < RETRY_COUNT: + time.sleep(RETRY_DELAY) + else: + print(f" {filepath.name} — пропущен после {RETRY_COUNT} попыток") + return False + + return False + + +def main() -> None: + RES_DIR.mkdir(exist_ok=True) + DONE_DIR.mkdir(exist_ok=True) + + client = Client(host=OLLAMA_HOST) + txt_files = sorted(LISTS_DIR.glob("*.txt")) + + if not txt_files: + print(f"Нет .txt файлов в {LISTS_DIR}") + return + + print(f"Файлов к обработке: {len(txt_files)}") + + # Последовательная обработка: следующий файл только после завершения текущего + for filepath in txt_files: + out_path = RES_DIR / filepath.with_suffix(".json").name + if out_path.exists(): + print(f" {filepath.name} — уже в res/, перенос в done/") + shutil.move(str(filepath), DONE_DIR / filepath.name) + continue + if process_file(client, filepath): + shutil.move(str(filepath), DONE_DIR / filepath.name) + + +if __name__ == "__main__": + main() diff --git a/parse_local.py b/parse_local.py new file mode 100644 index 0000000..3c462c0 --- /dev/null +++ b/parse_local.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +""" +Локальный парсер без Ollama: читает .txt из lists/, парсит табуляцию, +сохраняет JSON в res/, переносит успешные в done/. +""" + +import json +import pathlib +import shutil + +LISTS_DIR = pathlib.Path(__file__).parent / "lists" +RES_DIR = pathlib.Path(__file__).parent / "res" +DONE_DIR = pathlib.Path(__file__).parent / "done" + + +def parse_line(line: str) -> dict: + parts = line.strip().split("\t") + return { + "brand": parts[0] if len(parts) > 0 else "", + "model": parts[1] if len(parts) > 1 else "", + "engine_code": parts[2] if len(parts) > 2 else "", + "date_range": parts[3] if len(parts) > 3 else "", + } + + +def main() -> None: + RES_DIR.mkdir(exist_ok=True) + DONE_DIR.mkdir(exist_ok=True) + + for filepath in sorted(LISTS_DIR.glob("*.txt")): + content = filepath.read_text(encoding="utf-8") + lines = [ln for ln in content.splitlines() if ln.strip()] + data = [parse_line(ln) for ln in lines] + + out_path = RES_DIR / filepath.with_suffix(".json").name + out_path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") + shutil.move(str(filepath), DONE_DIR / filepath.name) + print(f" {filepath.name} — {len(data)} записей — OK") + + +if __name__ == "__main__": + main() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1111ae2 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +ollama>=0.3.0