From 91784923a7cb622fb4a656f9773d5ba62e9576a1 Mon Sep 17 00:00:00 2001 From: fUS1ONd Date: Sun, 11 Oct 2026 17:10:27 +0000 Subject: [PATCH 1/4] =?UTF-8?q?fix(structurize):=20=D0=B4=D0=BE=D1=81?= =?UTF-8?q?=D1=82=D1=80=D0=B0=D0=B8=D0=B2=D0=B0=D1=82=D1=8C=20end=20=D0=BF?= =?UTF-8?q?=D0=BE=D0=B4=D1=82=D0=B5=D0=BC=D1=8B=20=D0=B8=20=D1=82=D0=B5?= =?UTF-8?q?=D0=BC=D1=8B=20=D0=BF=D0=BE=20=D1=81=D0=BE=D1=81=D0=B5=D0=B4?= =?UTF-8?q?=D1=8F=D0=BC,=20=D0=B5=D1=81=D0=BB=D0=B8=20=D1=81=D0=BF=D0=BB?= =?UTF-8?q?=D0=B8=D1=82=D1=82=D0=B5=D1=80=20=D0=B5=D0=B3=D0=BE=20=D0=BD?= =?UTF-8?q?=D0=B5=20=D0=B2=D1=8B=D0=B4=D0=B0=D0=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../structurize/gemini_structurizer.py | 29 ++++-- tests/unit/test_gemini_structurizer.py | 99 +++++++++++++++++++ 2 files changed, 119 insertions(+), 9 deletions(-) diff --git a/lecturelog/infrastructure/structurize/gemini_structurizer.py b/lecturelog/infrastructure/structurize/gemini_structurizer.py index b094b0c..f5acf35 100644 --- a/lecturelog/infrastructure/structurize/gemini_structurizer.py +++ b/lecturelog/infrastructure/structurize/gemini_structurizer.py @@ -44,7 +44,7 @@ def _repair_section_timeline(topics_sections: list[list[dict[str, Any]]], srt_content: str) -> None: - """Чинит секции, у которых сплиттер выдал end <= start. + """Чинит секции, у которых сплиттер выдал end <= start или не выдал end вовсе. Битая граница обходится дорого: ffmpeg падает на нарезке («-to value smaller than -ss») уже после того, как потрачены транскрипция и все LLM-стадии, а @@ -52,6 +52,8 @@ def _repair_section_timeline(topics_sections: list[list[dict[str, Any]]], srt_co «invalid section timeline»). Чиним по соседям — конец битой секции берём из начала следующей, а для последней секции из конца транскрипта, — потому что содержание секции при этом уже отрендерено и терять его незачем. + + Тот же путь чинит и темы: им достаточно передать список тем одним «разделом». """ flat = [section for sections in topics_sections for section in sections] @@ -61,8 +63,8 @@ def _repair_section_timeline(topics_sections: list[list[dict[str, Any]]], srt_co transcript_end = max((block.end_s for block in parse_srt_blocks(srt_content)), default=0.0) for index, section in enumerate(flat): start = parse_srt_time(str(section["start"])) - end = parse_srt_time(str(section["end"])) - if end > start: + raw_end = section.get("end") + if raw_end not in (None, "") and parse_srt_time(str(raw_end)) > start: continue repaired = next( @@ -76,12 +78,19 @@ def _repair_section_timeline(topics_sections: list[list[dict[str, Any]]], srt_co ), start + 1.0, ) - logger.warning( - "секция %r имеет end <= start (%s → %s); конец исправлен по соседям", - section.get("title"), - section["start"], - section["end"], - ) + if raw_end in (None, ""): + logger.warning( + "секция %r пришла без end (start %s); конец достроен по соседям", + section.get("title"), + section["start"], + ) + else: + logger.warning( + "секция %r имеет end <= start (%s → %s); конец исправлен по соседям", + section.get("title"), + section["start"], + raw_end, + ) section["end"] = format_srt_seconds(repaired) @@ -388,6 +397,8 @@ async def structurize( topics_data = _parse_json(split_raw) if not isinstance(topics_data, list): raise ValueError("Ответ split_topics должен быть JSON-массивом") + # Конец темы нужен уже для нарезки фрагмента под subsplit. + _repair_section_timeline([topics_data], srt_content) await _emit_progress(on_progress, 10) # ── Этап 2: Subsplit (10% → 30%) ─────────────────────────── diff --git a/tests/unit/test_gemini_structurizer.py b/tests/unit/test_gemini_structurizer.py index 6517a98..be89ad2 100644 --- a/tests/unit/test_gemini_structurizer.py +++ b/tests/unit/test_gemini_structurizer.py @@ -1,4 +1,5 @@ import json +import logging import pytest @@ -555,6 +556,104 @@ async def test_structurize_repairs_inverted_range_in_last_section(tmp_path, prom ] +@pytest.mark.asyncio +async def test_structurize_completes_section_without_end_in_the_middle( + tmp_path, prompts_dir, caplog +): + """Сплиттер иногда теряет поле end у подтемы, и задача падала с KeyError: 'end'. + + Конец достраиваем тем же путём, что и при end <= start: начало следующей подтемы. + """ + srt = tmp_path / "t.srt" + srt.write_text(_THREE_CUES_SRT, encoding="utf-8") + + topics_json = json.dumps([{"title": "Тема 1", "start": "00:00:00", "end": "00:10:00"}]) + sections_json = json.dumps( + [ + {"title": "Подтема 1", "start": "00:00:00", "end": "00:04:00"}, + {"title": "Подтема 2", "start": "00:04:00"}, + {"title": "Подтема 3", "start": "00:07:00", "end": "00:10:00"}, + ] + ) + gemini = ScriptedGemini([topics_json, sections_json, "к1", "к2", "к3"]) + + structurizer = _make_structurizer(gemini, prompts_dir) + with caplog.at_level(logging.WARNING): + topics = await structurizer.structurize( + srt_path=srt, slide_images=[], output_dir=tmp_path / "out" + ) + + assert [(s.start, s.end) for s in topics[0].sections] == [ + ("00:00:00", "00:04:00"), + ("00:04:00", "00:07:00"), + ("00:07:00", "00:10:00"), + ] + assert any("Подтема 2" in record.getMessage() for record in caplog.records) + + +@pytest.mark.asyncio +async def test_structurize_completes_last_section_without_end(tmp_path, prompts_dir, caplog): + """У последней подтемы без end соседа справа нет — конец берём из конца транскрипта.""" + srt = tmp_path / "t.srt" + srt.write_text(_SIX_MIN_CUES_SRT, encoding="utf-8") + + topics_json = json.dumps([{"title": "Тема 1", "start": "00:00:00", "end": "00:10:00"}]) + sections_json = json.dumps( + [ + {"title": "Подтема 1", "start": "00:00:00", "end": "00:06:00"}, + {"title": "Подтема 2", "start": "00:06:00"}, + ] + ) + gemini = ScriptedGemini([topics_json, sections_json, "к1", "к2"]) + + structurizer = _make_structurizer(gemini, prompts_dir) + with caplog.at_level(logging.WARNING): + topics = await structurizer.structurize( + srt_path=srt, slide_images=[], output_dir=tmp_path / "out" + ) + + assert [(s.start, s.end) for s in topics[0].sections] == [ + ("00:00:00", "00:06:00"), + ("00:06:00", "00:10:00"), + ] + assert any("Подтема 2" in record.getMessage() for record in caplog.records) + + +@pytest.mark.asyncio +async def test_structurize_completes_topic_without_end(tmp_path, prompts_dir, caplog): + """Тема без end ломала уже нарезку фрагмента для subsplit, до всяких подтем. + + Путь общий с подтемами: конец первой темы — начало второй, последней — конец + транскрипта. + """ + srt = tmp_path / "t.srt" + srt.write_text(_SIX_MIN_CUES_SRT, encoding="utf-8") + + topics_json = json.dumps( + [ + {"title": "Тема 1", "start": "00:00:00"}, + {"title": "Тема 2", "start": "00:06:00"}, + ] + ) + sections_1 = json.dumps([{"title": "Подтема 1", "start": "00:00:00", "end": "00:06:00"}]) + sections_2 = json.dumps([{"title": "Подтема 2", "start": "00:06:00", "end": "00:10:00"}]) + gemini = ScriptedGemini([topics_json, sections_1, sections_2, "к1", "к2"]) + + structurizer = _make_structurizer(gemini, prompts_dir) + with caplog.at_level(logging.WARNING): + topics = await structurizer.structurize( + srt_path=srt, slide_images=[], output_dir=tmp_path / "out" + ) + + assert [(t.start, t.end) for t in topics] == [ + ("00:00:00", "00:06:00"), + ("00:06:00", "00:10:00"), + ] + messages = [record.getMessage() for record in caplog.records] + assert any("Тема 1" in message for message in messages) + assert any("Тема 2" in message for message in messages) + + @pytest.mark.asyncio async def test_structurize_subsplit_fallback_on_bad_json(tmp_path, prompts_dir): srt = tmp_path / "t.srt" From 9d182a840ad34d4e56d133c005503651d836c2c1 Mon Sep 17 00:00:00 2001 From: fUS1ONd Date: Sun, 11 Oct 2026 17:12:50 +0000 Subject: [PATCH 2/4] =?UTF-8?q?fix(errors):=20error=5Fcode=20=D0=BF=D0=BE?= =?UTF-8?q?=20=D1=81=D1=82=D0=B0=D1=82=D1=83=D1=81=D1=83=20=D0=BF=D1=80?= =?UTF-8?q?=D0=BE=D0=B2=D0=B0=D0=B9=D0=B4=D0=B5=D1=80=D0=B0=20=D0=B8=20?= =?UTF-8?q?=D0=B1=D0=B5=D0=B7=20bad=5Finput=20=D0=B4=D0=BB=D1=8F=20=D0=BE?= =?UTF-8?q?=D0=B1=D1=80=D1=8B=D0=B2=D0=B0=20JSON=20=D0=BE=D1=82=20LLM?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lecturelog/application/error_classifier.py | 25 +++++++++-- lecturelog/infrastructure/llm/llm_client.py | 7 +++- .../openrouter_401_with_previous_429.txt | 1 + .../openrouter_503_high_demand.txt | 1 + tests/unit/test_error_classifier.py | 42 +++++++++++++++++++ tests/unit/test_llm_client.py | 34 +++++++++++++++ 6 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 tests/fixtures/provider_errors/openrouter_401_with_previous_429.txt create mode 100644 tests/fixtures/provider_errors/openrouter_503_high_demand.txt diff --git a/lecturelog/application/error_classifier.py b/lecturelog/application/error_classifier.py index 6007e50..ae8820e 100644 --- a/lecturelog/application/error_classifier.py +++ b/lecturelog/application/error_classifier.py @@ -1,5 +1,8 @@ from __future__ import annotations +import json +import re + import httpx from lecturelog.domain.enums import ErrorCode @@ -11,6 +14,13 @@ # Ядро классифицирует по тексту, а не по типам SDK — это позволяет распознавать # лимит независимо от конкретного провайдера/клиента. _RATE_LIMIT_TOKENS = ("429", "RESOURCE_EXHAUSTED", "503", "UNAVAILABLE") +_RATE_LIMIT_STATUSES = (429, 503) + +# Статус ответа провайдера в тексте ошибки openai SDK: «Error code: 401 - {...}». +# Берём первое вхождение: дальше в теле лежит metadata.previous_errors — история +# ретраев самого OpenRouter, где встречаются чужие 429/503. По подстрокам такой +# 401 (отозванный ключ) раньше превращался в rate_limit. +_PROVIDER_STATUS_RE = re.compile(r"ERROR CODE: (\d{3})") # Подстроки-сигналы протухших/непринятых YouTube-cookies (текст yt-dlp). _COOKIES_INVALID_TOKENS = ("SIGN IN TO CONFIRM", "CONFIRM YOU'RE NOT A BOT") @@ -37,15 +47,24 @@ def classify_error(exc: BaseException) -> ErrorCode: }: return ErrorCode.BAD_INPUT return ErrorCode.INTERNAL - # 2) Типовые сигналы битого/нераспознанного входа. + # 2) Обрыв JSON в ответе LLM — подкласс ValueError, но вход пользователя + # тут ни при чём. + if isinstance(exc, json.JSONDecodeError): + return ErrorCode.INTERNAL + # 3) Типовые сигналы битого/нераспознанного входа. if isinstance(exc, (FileNotFoundError, ValueError, InvalidSlidesDocument)): return ErrorCode.BAD_INPUT - # 3) Текстовый сигнал лимита (LlmClient оборачивает last_error в RuntimeError). + # 4) Текстовый сигнал лимита (LlmClient оборачивает last_error в RuntimeError). message = str(exc).upper() # Сигнал протухших cookies от yt-dlp. if any(token in message for token in _COOKIES_INVALID_TOKENS): return ErrorCode.COOKIES_INVALID + status = _PROVIDER_STATUS_RE.search(message) + if status is not None: + if int(status.group(1)) in _RATE_LIMIT_STATUSES: + return ErrorCode.RATE_LIMIT + return ErrorCode.INTERNAL if any(token in message for token in _RATE_LIMIT_TOKENS): return ErrorCode.RATE_LIMIT - # 4) Дефолт. + # 5) Дефолт. return ErrorCode.INTERNAL diff --git a/lecturelog/infrastructure/llm/llm_client.py b/lecturelog/infrastructure/llm/llm_client.py index be77a43..b2682d3 100644 --- a/lecturelog/infrastructure/llm/llm_client.py +++ b/lecturelog/infrastructure/llm/llm_client.py @@ -149,7 +149,9 @@ async def call( extra_body["reasoning"] = {"effort": effort, "exclude": True} last_error: Exception | None = None - last_reason = "429/RESOURCE_EXHAUSTED" + # Причина попадает в текст итоговой ошибки, а по нему error_classifier + # выбирает error_code задачи — поэтому она всегда от последней ошибки. + last_reason = "ни одной попытки" for attempt in range(retries): model = await self._cooldown.acquire(models) kwargs: dict[str, Any] = { @@ -177,6 +179,7 @@ async def call( resp = await self._client.chat.completions.create(**kwargs) except openai.RateLimitError as error: last_error = error + last_reason = "429/RESOURCE_EXHAUSTED" raw = _extract_rate_limit_raw(error) ttl, _kind = parse_cooldown_ttl( raw, seconds_to_midnight=self._cooldown.seconds_to_midnight() @@ -189,6 +192,7 @@ async def call( if not _is_google_byok_auth_error(error): raise last_error = error + last_reason = "401: BYOK-ключ отклонён" logger.warning( "Google BYOK credential rejected for %s; trying next model", model, @@ -216,6 +220,7 @@ async def call( # Сетевой флап (не 429): модель не виновата, cooldown не трогаем — # ждём с нарастающим бэк-оффом и пробуем снова. last_error = error + last_reason = "сетевые ошибки" logger.warning( "Сетевая ошибка OpenRouter (%s), попытка %d/%d: %s", model, diff --git a/tests/fixtures/provider_errors/openrouter_401_with_previous_429.txt b/tests/fixtures/provider_errors/openrouter_401_with_previous_429.txt new file mode 100644 index 0000000..0f883e7 --- /dev/null +++ b/tests/fixtures/provider_errors/openrouter_401_with_previous_429.txt @@ -0,0 +1 @@ +Error code: 401 - {'error': {'message': 'Provider returned error', 'code': 401, 'metadata': {'raw': '{\n "error": {\n "code": 401,\n "message": "The bound service account is deleted or disabled. The service account bound to the API key must be active.",\n "status": "UNAUTHENTICATED",\n "details": [\n {\n "@type": "type.googleapis.com/google.rpc.ErrorInfo",\n "reason": "ACCOUNT_STATE_INVALID",\n "metadata": {\n "method": "google.ai.generativelanguage.v1alpha.GenerativeService.StreamGenerateContent",\n "service": "generativelanguage.googleapis.com"\n }\n }\n ]\n }\n}\n', 'provider_name': 'Google AI Studio', 'is_byok': True, 'provider_error_code': '401', 'previous_errors': [{'code': 429, 'message': 'Provider returned error', 'provider_name': 'Google AI Studio', 'raw': '{\n "error": {\n "code": 429,\n "message": "You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20, model: gemini-3.6-flash\\nPlease retry in 31.989211488s.",\n "status": "RESOURCE_EXHAUSTED",\n "details": [\n {\n "@type": "type.googleapis.com/google.rpc.Help",\n "links": [\n {\n "description": "Learn more about Gemini API quotas",\n "url": "https://ai.google.dev/gemini-api/docs/rate-limits"\n }\n ]\n },\n {\n "@type": "type.googleapis.com/google.rpc.QuotaFailure",\n "violations": [\n {\n "quotaMetric": "generativelanguage.googleapis.com/generate_content_free_tier_requests",\n "quotaId": "GenerateRequestsPerDayPerProjectPerModel-FreeTier",\n "quotaDimensions": {\n "location": "global",\n "model": "gemini-3.6-flash"\n },\n "quotaValue": "20"\n }\n ]\n },\n {\n "@type": "type.googleapis.com/google.rpc.RetryInfo",\n "retryDelay": "31s"\n }\n ]\n }\n}\n'}, {'code': 429, 'message': 'Provider returned error', 'provider_name': 'Google AI Studio', 'raw': '{\n "error": {\n "code": 429,\n "message": "You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20, model: gemini-3.6-flash\\nPlease retry in 31.687644476s.",\n "status": "RESOURCE_EXHAUSTED",\n "details": [\n {\n "@type": "type.googleapis.com/google.rpc.Help",\n "links": [\n {\n "description": "Learn more about Gemini API quotas",\n "url": "https://ai.google.dev/gemini-api/docs/rate-limits"\n }\n ]\n },\n {\n "@type": "type.googleapis.com/google.rpc.QuotaFailure",\n "violations": [\n {\n "quotaMetric": "generativelanguage.googleapis.com/generate_content_free_tier_requests",\n "quotaId": "GenerateRequestsPerDayPerProjectPerModel-FreeTier",\n "quotaDimensions": {\n "location": "global",\n "model": "gemini-3.6-flash"\n },\n "quotaValue": "20"\n }\n ]\n },\n {\n "@type": "type.googleapis.com/google.rpc.RetryInfo",\n "retryDelay": "31s"\n }\n ]\n }\n}\n'}, {'code': 429, 'message': 'Provider returned error', 'provider_name': 'Google AI Studio', 'raw': '{\n "error": {\n "code": 429,\n "message": "You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20, model: gemini-3.6-flash\\nPlease retry in 31.381486872s.",\n "status": "RESOURCE_EXHAUSTED",\n "details": [\n {\n "@type": "type.googleapis.com/google.rpc.Help",\n "links": [\n {\n "description": "Learn more about Gemini API quotas",\n "url": "https://ai.google.dev/gemini-api/docs/rate-limits"\n }\n ]\n },\n {\n "@type": "type.googleapis.com/google.rpc.QuotaFailure",\n "violations": [\n {\n "quotaMetric": "generativelanguage.googleapis.com/generate_content_free_tier_requests",\n "quotaId": "GenerateRequestsPerDayPerProjectPerModel-FreeTier",\n "quotaDimensions": {\n "model": "gemini-3.6-flash",\n "location": "global"\n },\n "quotaValue": "20"\n }\n ]\n },\n {\n "@type": "type.googleapis.com/google.rpc.RetryInfo",\n "retryDelay": "31s"\n }\n ]\n }\n}\n'}]}}, 'user_REDACTED': 'user_REDACTED'} \ No newline at end of file diff --git a/tests/fixtures/provider_errors/openrouter_503_high_demand.txt b/tests/fixtures/provider_errors/openrouter_503_high_demand.txt new file mode 100644 index 0000000..a2e141d --- /dev/null +++ b/tests/fixtures/provider_errors/openrouter_503_high_demand.txt @@ -0,0 +1 @@ +Error code: 503 - {'error': {'message': 'Provider returned error', 'code': 503, 'metadata': {'raw': '{\n "error": {\n "code": 503,\n "message": "This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.",\n "status": "UNAVAILABLE"\n }\n}\n', 'provider_name': 'Google AI Studio', 'is_byok': True, 'provider_error_code': '503', 'previous_errors': [{'code': 503, 'message': 'Provider returned error', 'provider_name': 'Google AI Studio', 'raw': '{\n "error": {\n "code": 503,\n "message": "This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.",\n "status": "UNAVAILABLE"\n }\n}\n'}, {'code': 503, 'message': 'Provider returned error', 'provider_name': 'Google AI Studio', 'raw': '{\n "error": {\n "code": 503,\n "message": "This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.",\n "status": "UNAVAILABLE"\n }\n}\n'}]}}, 'user_REDACTED': 'user_REDACTED'} \ No newline at end of file diff --git a/tests/unit/test_error_classifier.py b/tests/unit/test_error_classifier.py index b3fdb79..d442efa 100644 --- a/tests/unit/test_error_classifier.py +++ b/tests/unit/test_error_classifier.py @@ -1,8 +1,12 @@ +import json +from pathlib import Path + import httpx import pytest from lecturelog.application.error_classifier import classify_error from lecturelog.domain.enums import ErrorCode +from lecturelog.infrastructure.structurize.gemini_structurizer import _parse_json def _http_status_error(code: int) -> httpx.HTTPStatusError: @@ -56,3 +60,41 @@ def test_value_error_is_bad_input(): def test_generic_runtime_is_internal(): assert classify_error(RuntimeError("ffmpeg завершился с ошибкой: ...")) is ErrorCode.INTERNAL + + +# Реальные тексты ошибок с прода (tasks.error, user_id вымаран). +_PROVIDER_ERRORS = Path(__file__).parent.parent / "fixtures" / "provider_errors" +_PROD_401 = (_PROVIDER_ERRORS / "openrouter_401_with_previous_429.txt").read_text(encoding="utf-8") +_PROD_503 = (_PROVIDER_ERRORS / "openrouter_503_high_demand.txt").read_text(encoding="utf-8") + + +def test_prod_401_with_previous_429_is_not_rate_limit(): + # Подстроки 429/RESOURCE_EXHAUSTED приносит metadata.previous_errors — + # история ретраев самого OpenRouter, а настоящая причина — отозванный ключ. + assert "429" in _PROD_401 + assert classify_error(RuntimeError(_PROD_401)) is not ErrorCode.RATE_LIMIT + + +def test_llm_client_wrapper_over_prod_401_is_not_rate_limit(): + exc = RuntimeError( + f"OpenRouter не дал ответ за 5 попыток (401: BYOK-ключ отклонён): {_PROD_401}" + ) + assert classify_error(exc) is not ErrorCode.RATE_LIMIT + + +def test_prod_503_high_demand_is_rate_limit(): + assert classify_error(RuntimeError(_PROD_503)) is ErrorCode.RATE_LIMIT + + +def test_llm_client_wrapper_over_prod_503_is_rate_limit(): + exc = RuntimeError(f"OpenRouter не дал ответ за 5 попыток (HTTP 503 апстрима): {_PROD_503}") + assert classify_error(exc) is ErrorCode.RATE_LIMIT + + +def test_truncated_llm_json_is_not_bad_input(): + # Прод: «Unterminated string starting at: line 20 column 12» из разбора ответа + # split_topics. Вход пользователя тут ни при чём. + with pytest.raises(json.JSONDecodeError) as exc_info: + _parse_json('```json\n[{"title": "Тема 1", "start": "00:00:00", "end": "00:0') + assert "Unterminated string" in str(exc_info.value) + assert classify_error(exc_info.value) is ErrorCode.INTERNAL diff --git a/tests/unit/test_llm_client.py b/tests/unit/test_llm_client.py index 74f0c8f..4f0e6d4 100644 --- a/tests/unit/test_llm_client.py +++ b/tests/unit/test_llm_client.py @@ -6,6 +6,8 @@ import pytest import lecturelog.infrastructure.llm.llm_client as mod +from lecturelog.application.error_classifier import classify_error +from lecturelog.domain.enums import ErrorCode from lecturelog.infrastructure.llm.llm_client import ( _NETWORK_BACKOFF_S as _NETWORK_BACKOFF_S_EXPECTED, ) @@ -363,6 +365,38 @@ async def test_google_byok_auth_error_with_sdk_unwrapped_body_falls_back(monkeyp assert await client.call("q", models=["m1", "m2"]) == "fallback ok" +@pytest.mark.asyncio +async def test_byok_auth_errors_exhausted_do_not_claim_rate_limit(): + """Причина в тексте обёртки — по ней классифицируется error_code задачи. + + Раньше причина по умолчанию была «429/RESOURCE_EXHAUSTED», и мёртвый ключ + превращался у клиента в rate_limit. + """ + fake = FakeAsyncOpenAI([_authentication_error(byok=True) for _ in range(3)]) + client = LlmClient(fake, ModelCooldown()) + + with pytest.raises(RuntimeError) as exc_info: + await client.call("q", models=["m1"], retries=3) + + message = str(exc_info.value) + assert "429" not in message + assert "RESOURCE_EXHAUSTED" not in message + assert classify_error(exc_info.value) is not ErrorCode.RATE_LIMIT + + +@pytest.mark.asyncio +async def test_last_reason_follows_last_error(monkeypatch): + """Причина в тексте — от последней ошибки, а не от первой нестандартной.""" + monkeypatch.setattr(mod.asyncio, "sleep", _no_sleep) + fake = FakeAsyncOpenAI([_upstream_unavailable_error(), _rate_limit_error(_RPM_RAW)]) + client = LlmClient(fake, ModelCooldown()) + + with pytest.raises(RuntimeError) as exc_info: + await client.call("q", models=["m1", "m2"], retries=2) + + assert "(429/RESOURCE_EXHAUSTED)" in str(exc_info.value) + + @pytest.mark.asyncio async def test_openrouter_auth_error_does_not_fallback(): client = LlmClient( From c414653191aab38b3ed284663a11b7773f98b090 Mon Sep 17 00:00:00 2001 From: fUS1ONd Date: Sun, 11 Oct 2026 17:17:18 +0000 Subject: [PATCH 3/4] =?UTF-8?q?fix(errors):=20=D1=81=D1=82=D0=B0=D1=82?= =?UTF-8?q?=D1=83=D1=81=20=D0=BF=D1=80=D0=BE=D0=B2=D0=B0=D0=B9=D0=B4=D0=B5?= =?UTF-8?q?=D1=80=D0=B0=20=D1=80=D0=B5=D1=88=D0=B0=D0=B5=D1=82=20=D1=82?= =?UTF-8?q?=D0=BE=D0=BB=D1=8C=D0=BA=D0=BE=20=D0=B4=D0=BB=D1=8F=204xx,=205x?= =?UTF-8?q?x=20=D1=81=20UNAVAILABLE=20=D0=BE=D1=81=D1=82=D0=B0=D1=91=D1=82?= =?UTF-8?q?=D1=81=D1=8F=20rate=5Flimit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lecturelog/application/error_classifier.py | 10 ++++----- .../structurize/gemini_structurizer.py | 5 +++-- tests/unit/test_error_classifier.py | 22 ++++++++++++++++--- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/lecturelog/application/error_classifier.py b/lecturelog/application/error_classifier.py index ae8820e..57d161d 100644 --- a/lecturelog/application/error_classifier.py +++ b/lecturelog/application/error_classifier.py @@ -14,12 +14,14 @@ # Ядро классифицирует по тексту, а не по типам SDK — это позволяет распознавать # лимит независимо от конкретного провайдера/клиента. _RATE_LIMIT_TOKENS = ("429", "RESOURCE_EXHAUSTED", "503", "UNAVAILABLE") -_RATE_LIMIT_STATUSES = (429, 503) # Статус ответа провайдера в тексте ошибки openai SDK: «Error code: 401 - {...}». # Берём первое вхождение: дальше в теле лежит metadata.previous_errors — история # ретраев самого OpenRouter, где встречаются чужие 429/503. По подстрокам такой -# 401 (отозванный ключ) раньше превращался в rate_limit. +# 401 (отозванный ключ) раньше превращался в rate_limit. Решающим статус считаем +# только для 4xx, кроме 429: это отказ по самому запросу или ключу, а не +# перегрузка. 5xx Gemini бывает и 500/504 с UNAVAILABLE в теле — их оставляем +# подстрокам. _PROVIDER_STATUS_RE = re.compile(r"ERROR CODE: (\d{3})") # Подстроки-сигналы протухших/непринятых YouTube-cookies (текст yt-dlp). @@ -60,9 +62,7 @@ def classify_error(exc: BaseException) -> ErrorCode: if any(token in message for token in _COOKIES_INVALID_TOKENS): return ErrorCode.COOKIES_INVALID status = _PROVIDER_STATUS_RE.search(message) - if status is not None: - if int(status.group(1)) in _RATE_LIMIT_STATUSES: - return ErrorCode.RATE_LIMIT + if status is not None and 400 <= int(status.group(1)) < 500 and status.group(1) != "429": return ErrorCode.INTERNAL if any(token in message for token in _RATE_LIMIT_TOKENS): return ErrorCode.RATE_LIMIT diff --git a/lecturelog/infrastructure/structurize/gemini_structurizer.py b/lecturelog/infrastructure/structurize/gemini_structurizer.py index f5acf35..b143927 100644 --- a/lecturelog/infrastructure/structurize/gemini_structurizer.py +++ b/lecturelog/infrastructure/structurize/gemini_structurizer.py @@ -64,7 +64,8 @@ def _repair_section_timeline(topics_sections: list[list[dict[str, Any]]], srt_co for index, section in enumerate(flat): start = parse_srt_time(str(section["start"])) raw_end = section.get("end") - if raw_end not in (None, "") and parse_srt_time(str(raw_end)) > start: + end_missing = raw_end in (None, "") + if not end_missing and parse_srt_time(str(raw_end)) > start: continue repaired = next( @@ -78,7 +79,7 @@ def _repair_section_timeline(topics_sections: list[list[dict[str, Any]]], srt_co ), start + 1.0, ) - if raw_end in (None, ""): + if end_missing: logger.warning( "секция %r пришла без end (start %s); конец достроен по соседям", section.get("title"), diff --git a/tests/unit/test_error_classifier.py b/tests/unit/test_error_classifier.py index d442efa..8fd7df0 100644 --- a/tests/unit/test_error_classifier.py +++ b/tests/unit/test_error_classifier.py @@ -6,7 +6,6 @@ from lecturelog.application.error_classifier import classify_error from lecturelog.domain.enums import ErrorCode -from lecturelog.infrastructure.structurize.gemini_structurizer import _parse_json def _http_status_error(code: int) -> httpx.HTTPStatusError: @@ -72,7 +71,7 @@ def test_prod_401_with_previous_429_is_not_rate_limit(): # Подстроки 429/RESOURCE_EXHAUSTED приносит metadata.previous_errors — # история ретраев самого OpenRouter, а настоящая причина — отозванный ключ. assert "429" in _PROD_401 - assert classify_error(RuntimeError(_PROD_401)) is not ErrorCode.RATE_LIMIT + assert classify_error(RuntimeError(_PROD_401)) is ErrorCode.INTERNAL def test_llm_client_wrapper_over_prod_401_is_not_rate_limit(): @@ -95,6 +94,23 @@ def test_truncated_llm_json_is_not_bad_input(): # Прод: «Unterminated string starting at: line 20 column 12» из разбора ответа # split_topics. Вход пользователя тут ни при чём. with pytest.raises(json.JSONDecodeError) as exc_info: - _parse_json('```json\n[{"title": "Тема 1", "start": "00:00:00", "end": "00:0') + json.loads('[{"title": "Тема 1", "start": "00:00:00", "end": "00:0') assert "Unterminated string" in str(exc_info.value) assert classify_error(exc_info.value) is ErrorCode.INTERNAL + + +@pytest.mark.parametrize( + "text", + [ + "Error code: 500 - {'error': {'message': 'model UNAVAILABLE'}}", + "Error code: 504 - {'error': {'status': 'UNAVAILABLE'}}", + ], +) +def test_upstream_5xx_with_unavailable_stays_rate_limit(text): + # Gemini отдаёт перегрузку и как 500/504 — статус не должен её перебивать. + assert classify_error(RuntimeError(text)) is ErrorCode.RATE_LIMIT + + +def test_provider_4xx_beats_rate_limit_tokens_in_body(): + exc = RuntimeError("Error code: 403 - {'previous_errors': [{'code': 429}]}") + assert classify_error(exc) is ErrorCode.INTERNAL From 14bedc240e4be2c4065c00e58b47f39dc9554bd2 Mon Sep 17 00:00:00 2001 From: fUS1ONd Date: Sun, 11 Oct 2026 17:21:55 +0000 Subject: [PATCH 4/4] =?UTF-8?q?fix(llm):=20=D0=BF=D0=B5=D1=80=D0=B5=D0=B6?= =?UTF-8?q?=D0=B8=D0=B4=D0=B0=D1=82=D1=8C=205xx=20=D0=B0=D0=BF=D1=81=D1=82?= =?UTF-8?q?=D1=80=D0=B8=D0=BC=D0=B0=20=D0=BF=D0=BE=20=D0=B1=D1=8E=D0=B4?= =?UTF-8?q?=D0=B6=D0=B5=D1=82=D1=83=20=D0=B2=D1=80=D0=B5=D0=BC=D0=B5=D0=BD?= =?UTF-8?q?=D0=B8=20=D0=B8=20=D0=B6=D0=B4=D0=B0=D1=82=D1=8C=20=D0=B1=D0=BB?= =?UTF-8?q?=D0=B8=D0=B6=D0=B0=D0=B9=D1=88=D1=83=D1=8E=20=D0=BC=D0=BE=D0=B4?= =?UTF-8?q?=D0=B5=D0=BB=D1=8C,=20=D0=B5=D1=81=D0=BB=D0=B8=20=D0=BE=D1=81?= =?UTF-8?q?=D1=82=D1=8B=D0=B2=D0=B0=D1=8E=D1=82=20=D0=B2=D1=81=D0=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lecturelog/infrastructure/llm/llm_client.py | 55 ++++++++++++++---- .../infrastructure/llm/model_cooldown.py | 6 ++ tests/unit/test_llm_client.py | 58 +++++++++++++++++-- 3 files changed, 102 insertions(+), 17 deletions(-) diff --git a/lecturelog/infrastructure/llm/llm_client.py b/lecturelog/infrastructure/llm/llm_client.py index b2682d3..d8457d2 100644 --- a/lecturelog/infrastructure/llm/llm_client.py +++ b/lecturelog/infrastructure/llm/llm_client.py @@ -14,6 +14,7 @@ import inspect import json import logging +import time from collections.abc import Awaitable, Callable from typing import Any @@ -36,6 +37,10 @@ # следующая попытка должна уйти на другую модель, но выводить эту из ротации # надолго незачем — отказ транзиентный. _UPSTREAM_ERROR_COOLDOWN_S = 60.0 +# Сколько один вызов готов пережидать 5xx апстрима. Перегрузка Gemini на проде +# длилась минутами, а пять попыток с бэк-оффом укладывались в ~30 с. Пока бюджет +# не исчерпан, попытки на 5xx не расходуют retries. +_UPSTREAM_BUDGET_S = 300.0 async def _emit_usage(on_usage: UsageCallback | None, payload: dict) -> None: @@ -123,10 +128,14 @@ def __init__( cooldown: ModelCooldown, *, max_tokens: int = _DEFAULT_MAX_TOKENS, + upstream_budget_s: float = _UPSTREAM_BUDGET_S, + clock: Callable[[], float] | None = None, ) -> None: self._client = async_openai_client self._cooldown = cooldown self._max_tokens = max_tokens + self._upstream_budget_s = upstream_budget_s + self._clock = clock or time.monotonic async def call( self, @@ -152,8 +161,21 @@ async def call( # Причина попадает в текст итоговой ошибки, а по нему error_classifier # выбирает error_code задачи — поэтому она всегда от последней ошибки. last_reason = "ни одной попытки" - for attempt in range(retries): + tried: list[str] = [] + upstream_deadline: float | None = None + upstream_retry = False + attempt = 0 + while True: + if upstream_retry: + if self._clock() >= upstream_deadline: + break + elif attempt >= retries: + break + upstream_retry = False + attempt += 1 model = await self._cooldown.acquire(models) + if model not in tried: + tried.append(model) kwargs: dict[str, Any] = { "model": model, "messages": messages, @@ -204,17 +226,27 @@ async def call( # с запросом: пока есть другие модели, задача падать не должна. last_error = error last_reason = f"HTTP {getattr(error, 'status_code', '5xx')} апстрима" + if upstream_deadline is None: + upstream_deadline = self._clock() + self._upstream_budget_s + upstream_retry = True + await self._cooldown.mark_rate_limited(model, _UPSTREAM_ERROR_COOLDOWN_S) + # Апстрим сам просит подождать. Если остывают все модели, ждём ту, + # что освободится раньше, а не бьём в горячую; иначе — короткий + # нарастающий бэк-офф перед следующей моделью. Пауза не выходит + # за бюджет. + wait = await self._cooldown.seconds_until_free(models) + if wait <= 0: + wait = _NETWORK_BACKOFF_S * attempt + wait = max(0.0, min(wait, upstream_deadline - self._clock())) logger.warning( - "Апстрим недоступен (%s), попытка %d/%d: %s", + "Апстрим недоступен (%s), попытка %d, ждём %.0f с (бюджет до %.0f с): %s", model, - attempt + 1, - retries, + attempt, + wait, + upstream_deadline - self._clock(), error, ) - await self._cooldown.mark_rate_limited(model, _UPSTREAM_ERROR_COOLDOWN_S) - # Апстрим сам просит подождать, поэтому ретрай без паузы сжёг бы - # все попытки за доли секунды. - await asyncio.sleep(_NETWORK_BACKOFF_S * (attempt + 1)) + await asyncio.sleep(wait) continue except (openai.APITimeoutError, openai.APIConnectionError) as error: # Сетевой флап (не 429): модель не виновата, cooldown не трогаем — @@ -224,11 +256,11 @@ async def call( logger.warning( "Сетевая ошибка OpenRouter (%s), попытка %d/%d: %s", model, - attempt + 1, + attempt, retries, error, ) - await asyncio.sleep(_NETWORK_BACKOFF_S * (attempt + 1)) + await asyncio.sleep(_NETWORK_BACKOFF_S * attempt) continue # Провайдер может отвалиться до начала генерации: тогда OpenRouter @@ -280,5 +312,6 @@ async def call( return text raise RuntimeError( - f"OpenRouter не дал ответ за {retries} попыток ({last_reason}): {last_error}" + f"OpenRouter не дал ответ за {attempt} попыток ({last_reason}), " + f"модели: {', '.join(tried)}: {last_error}" ) diff --git a/lecturelog/infrastructure/llm/model_cooldown.py b/lecturelog/infrastructure/llm/model_cooldown.py index bfa8a89..044bd66 100644 --- a/lecturelog/infrastructure/llm/model_cooldown.py +++ b/lecturelog/infrastructure/llm/model_cooldown.py @@ -55,6 +55,12 @@ async def acquire(self, models: list[str]) -> str: logger.warning("все модели остывают, беру %s (раньше всех освободится)", soonest) return soonest + async def seconds_until_free(self, models: list[str]) -> float: + """Сколько ждать, пока освободится хотя бы одна модель списка (0 — уже есть).""" + async with self._lock: + now = self._time() + return max(0.0, min(self._blocked_until[m] for m in models) - now) + async def mark_rate_limited(self, model: str, ttl: float) -> None: async with self._lock: self._blocked_until[model] = self._time() + ttl diff --git a/tests/unit/test_llm_client.py b/tests/unit/test_llm_client.py index 4f0e6d4..0d9b826 100644 --- a/tests/unit/test_llm_client.py +++ b/tests/unit/test_llm_client.py @@ -616,16 +616,62 @@ async def test_provider_unavailable_falls_back_to_next_model(monkeypatch): assert [model for model, _ttl in cooldown.marked] == ["m1"] +class FakeClock: + """Часы, которые двигает только подменённый asyncio.sleep.""" + + def __init__(self) -> None: + self.now = 0.0 + self.slept: list[float] = [] + + def __call__(self) -> float: + return self.now + + async def sleep(self, delay: float) -> None: + self.slept.append(delay) + self.now += delay + + @pytest.mark.asyncio -async def test_provider_unavailable_exhausts_retries_with_clear_message(monkeypatch): - monkeypatch.setattr(mod.asyncio, "sleep", _no_sleep) - fake = FakeAsyncOpenAI([_upstream_unavailable_error() for _ in range(3)]) - client = LlmClient(fake, ModelCooldown()) +async def test_provider_unavailable_waits_for_soonest_model_within_budget(monkeypatch): + """Перегрузка Gemini на проде длится минуты: пять быстрых попыток за ~30 с её + не переживали. Когда остывают все модели, ждём ближайшую, а не бьём в горячую, + и число попыток на 5xx ограничено бюджетом времени, а не retries.""" + clock = FakeClock() + monkeypatch.setattr(mod.asyncio, "sleep", clock.sleep) + monkeypatch.setattr(mod, "_UPSTREAM_ERROR_COOLDOWN_S", 60.0) + fake = FakeAsyncOpenAI([_upstream_unavailable_error() for _ in range(3)] + [_resp("ok")]) + client = LlmClient(fake, ModelCooldown(time_func=clock), clock=clock) + + assert await client.call("q", models=["m1", "m2"], retries=2) == "ok" + + assert [item["model"] for item in fake.chat.completions.kwargs_history] == [ + "m1", + "m2", + "m1", + "m2", + ] + # m2 свободна — обычная пауза; обе остывают — ждём до освобождения m1 (t=60), + # затем до освобождения m2 (t=62). + assert clock.slept == [_NETWORK_BACKOFF_S_EXPECTED, 58.0, 2.0] + + +@pytest.mark.asyncio +async def test_provider_unavailable_longer_than_budget_fails_with_models(monkeypatch): + clock = FakeClock() + monkeypatch.setattr(mod.asyncio, "sleep", clock.sleep) + fake = FakeAsyncOpenAI([_upstream_unavailable_error() for _ in range(100)]) + client = LlmClient(fake, ModelCooldown(time_func=clock), clock=clock, upstream_budget_s=200.0) with pytest.raises(RuntimeError) as exc_info: - await client.call("q", models=["m1"], retries=3) + await client.call("q", models=["m1", "m2"], retries=2) - assert "503" in str(exc_info.value) + message = str(exc_info.value) + assert "HTTP 503 апстрима" in message + assert "m1" in message and "m2" in message + assert classify_error(exc_info.value) is ErrorCode.RATE_LIMIT + # пауза не вылезает за бюджет + assert clock.now <= 200.0 + assert fake.chat.completions.calls > 2 @pytest.mark.asyncio