diff --git a/projekt-matching/projektmatch/ingest.py b/projekt-matching/projektmatch/ingest.py index a3d2215..e018885 100644 --- a/projekt-matching/projektmatch/ingest.py +++ b/projekt-matching/projektmatch/ingest.py @@ -119,6 +119,10 @@ def run_ingest(cfg, mailbox=None, dispatch=None, espo=None, resolve=None): or f"unbekannter Status: {result.get('status')!r}") results.append(result) summary[result["status"]] += 1 + try: + box.ensure_alive() # Flow-2-Dispatches können >30 min dauern — Server-Idle-Timeout + except Exception: # noqa: BLE001 + pass # Alert/Trash unten haben eigene Fallbacks failed = [r for r in results if r["status"] == "failed"] if failed: lines = "\n\n".join( @@ -134,7 +138,14 @@ def run_ingest(cfg, mailbox=None, dispatch=None, espo=None, resolve=None): "werden:\n\n" + lines + "\n") except Exception: # noqa: BLE001 summary["alertFailed"] = summary.get("alertFailed", 0) + 1 - box.move_to_trash(uid) + try: + box.move_to_trash(uid) + except Exception: # noqa: BLE001 + summary["trashFailed"] = summary.get("trashFailed", 0) + 1 + try: + box.flag_checked(uid) # nie erneut verarbeiten — Terminal-Zustand + except Exception: # noqa: BLE001 + pass # nächster Lauf räumt auf return summary finally: box.close() diff --git a/projekt-matching/projektmatch/llm.py b/projekt-matching/projektmatch/llm.py index b03415a..e920ebf 100644 --- a/projekt-matching/projektmatch/llm.py +++ b/projekt-matching/projektmatch/llm.py @@ -1,7 +1,9 @@ """vLLM (OpenAI-compatible) calls with response_format json_schema + German prompts. -max_tokens stays UNSET so nothing can truncate the final answer (65k -context window). Thinking is DISABLED via chat_template_kwargs: with the +max_tokens is capped at 8192: every valid answer fits comfortably, and an +uncapped run lets a degenerate generation grind for ~10 min before failing +(observed live: 100+-requirement JSON, broken at ~char 1725, 3 retries = +~35 min per cycle). Thinking is DISABLED via chat_template_kwargs: with the vLLM qwen3 reasoning parser active, json_schema guided decoding plus thinking degenerates — the run burns the entire context window (finish_reason "length", ~63k completion tokens) and returns empty or @@ -34,6 +36,7 @@ EXTRACT_SCHEMA = { "contactPerson": {"type": ["string", "null"]}, "requirements": { "type": "array", + "maxItems": 40, "items": { "type": "object", "properties": { @@ -139,6 +142,7 @@ z. B. vLLM, TGI, Triton" ab). def _post(base, model, messages, schema): body = {"model": model, "messages": messages, "temperature": 0.1, + "max_tokens": 8192, "chat_template_kwargs": {"enable_thinking": False}, "response_format": {"type": "json_schema", "json_schema": { "name": "out", "schema": schema}}} diff --git a/projekt-matching/projektmatch/mailer.py b/projekt-matching/projektmatch/mailer.py index 4ed2a09..c05120f 100644 --- a/projekt-matching/projektmatch/mailer.py +++ b/projekt-matching/projektmatch/mailer.py @@ -16,12 +16,31 @@ KEYWORD = "$ProjektChecked" class MailBox: def __init__(self, user, password, conn=None): - self.conn = conn or imaplib.IMAP4_SSL(IMAP_HOST, IMAP_PORT) - if conn is None: - self.conn.login(user, password) + self.user, self.password = user, password + self.conn = conn or self._connect() self.conn.select("INBOX") self._trash = None + def _connect(self): + conn = imaplib.IMAP4_SSL(IMAP_HOST, IMAP_PORT) + conn.login(self.user, self.password) + return conn + + def ensure_alive(self): + """Reconnect if the server dropped the connection (idle timeout).""" + try: + typ, _ = self.conn.noop() + if typ == "OK": + return + except Exception: # noqa: BLE001 + pass + try: + self.conn.logout() + except Exception: # noqa: BLE001 + pass + self.conn = self._connect() + self.conn.select("INBOX") + def unchecked_uids(self): typ, data = self.conn.uid("SEARCH", None, f"UNKEYWORD {KEYWORD}") if typ != "OK" or not data or not data[0]: diff --git a/projekt-matching/projektmatch/stages.py b/projekt-matching/projektmatch/stages.py index dcfcfda..80ce6b3 100644 --- a/projekt-matching/projektmatch/stages.py +++ b/projekt-matching/projektmatch/stages.py @@ -20,6 +20,7 @@ TEAM_BY_OFFER = {"Projekt": "DesTEngS", "Arbeitnehmer-Angebot": "Arbeitnehmer", "ANÜ": "ANÜ"} MIN_PAGE_CHARS = 200 +QUOTES = str.maketrans({c: "'" for c in "„“”\"«»‹›"}) def run_stage(name, fn, ctx, cfg, **kw): @@ -32,6 +33,25 @@ def run_stage(name, fn, ctx, cfg, **kw): return ctx +def page_text(html): + """Project text for the LLM: header + description, WITHOUT the skill-tag + cloud (div.project-body-badges, ~100 entries on some pages — blows the + requirements list up until the model emits broken JSON) and without + site chrome. Falls back to whole-page text if div.content is missing.""" + soup = BeautifulSoup(html, "html.parser") + for tag in soup(["script", "style", "noscript"]): + tag.decompose() + root = soup.find("div", class_="content") or soup + for tag in root.find_all(class_="project-body-badges"): + tag.decompose() + # Anführungszeichen -> Apostroph: ein wörtlich übernommenes " beendet den + # JSON-String vorzeitig und treibt guided decoding in eine Whitespace- + # Endlosschleife (finish_reason "length", live verifiziert an nproj/3022414) + return "\n".join(line for line in + root.get_text("\n", strip=True).splitlines() + if line.strip()).translate(QUOTES) + + def stage_fetch(ctx, cfg): last = None for attempt in range(3): @@ -39,12 +59,7 @@ def stage_fetch(ctx, cfg): resp = requests.get(ctx["canonical"], timeout=30, headers={"User-Agent": BROWSER_UA}) resp.raise_for_status() - soup = BeautifulSoup(resp.text, "html.parser") - for tag in soup(["script", "style", "noscript"]): - tag.decompose() - text = "\n".join(line for line in - soup.get_text("\n", strip=True).splitlines() - if line.strip()) + text = page_text(resp.text) if len(text) < MIN_PAGE_CHARS: raise ValueError( f"Seitentext zu kurz ({len(text)} Zeichen) — " diff --git a/projekt-matching/tests/test_ingest.py b/projekt-matching/tests/test_ingest.py index 4cb5483..956eda8 100644 --- a/projekt-matching/tests/test_ingest.py +++ b/projekt-matching/tests/test_ingest.py @@ -156,6 +156,63 @@ def test_close_called_even_if_unchecked_uids_raises(tmp_path): box.close.assert_called_once() +def test_ensure_alive_called_before_finalize(tmp_path): + mails = {"1": mail("Mail", PROJECT_HTML)} + summary, box, sent = run(tmp_path, mails, + dispatch_result={"status": "failed", + "error": "Read timed out"}) + names = [c[0] for c in box.method_calls] + assert "ensure_alive" in names + assert names.index("ensure_alive") < names.index("move_to_trash") + box.move_to_trash.assert_called_once_with("1") + assert len(sent) == 1 + + +def test_trash_failure_falls_back_to_flag_and_continues(tmp_path): + mails = {"1": mail("Mail A", PROJECT_HTML), + "2": mail("Mail B", PROJECT_HTML)} + box = mock.Mock() + box.unchecked_uids.return_value = ["1", "2"] + box.fetch.side_effect = lambda uid: mails[uid] + box.move_to_trash.side_effect = [OSError("socket error: EOF"), None] + espo = mock.Mock() + espo.find_opportunity_by_link.return_value = None + with mock.patch("projektmatch.ingest.mailparse.bodies", + side_effect=lambda m: (m["html"], "")), \ + mock.patch("projektmatch.ingest.mailer.send_mail", lambda *a: None): + summary = ingest.run_ingest( + cfg(tmp_path), mailbox=box, espo=espo, + resolve=lambda url: "https://www.freelancermap.de/projekt/eins", + dispatch=lambda item: {"status": "created"}) + assert summary["trashFailed"] == 1 + assert summary["created"] == 2 + box.flag_checked.assert_called_once_with("1") + assert box.move_to_trash.call_count == 2 + + +def test_dead_connection_everywhere_still_finishes_run(tmp_path): + mails = {"1": mail("Mail A", PROJECT_HTML), + "2": mail("Mail B", PROJECT_HTML)} + box = mock.Mock() + box.unchecked_uids.return_value = ["1", "2"] + box.fetch.side_effect = lambda uid: mails[uid] + box.ensure_alive.side_effect = OSError("server unreachable") + box.move_to_trash.side_effect = OSError("socket error: EOF") + box.flag_checked.side_effect = OSError("socket error: EOF") + espo = mock.Mock() + espo.find_opportunity_by_link.return_value = None + with mock.patch("projektmatch.ingest.mailparse.bodies", + side_effect=lambda m: (m["html"], "")), \ + mock.patch("projektmatch.ingest.mailer.send_mail", lambda *a: None): + summary = ingest.run_ingest( + cfg(tmp_path), mailbox=box, espo=espo, + resolve=lambda url: "https://www.freelancermap.de/projekt/eins", + dispatch=lambda item: {"status": "created"}) + assert summary["trashFailed"] == 2 + assert summary["created"] == 2 + box.close.assert_called_once() + + def test_parse_run_result_finds_status_json(): inner = json.dumps({"status": "created", "mustMatch": 90}) payload = {"outputs": [{"outputs": [{"results": {"text": { diff --git a/projekt-matching/tests/test_llm.py b/projekt-matching/tests/test_llm.py index 5a5fd92..2358853 100644 --- a/projekt-matching/tests/test_llm.py +++ b/projekt-matching/tests/test_llm.py @@ -24,11 +24,16 @@ def test_chat_json_strips_think_and_parses(): {"type": "object"}) assert out == {"a": 1} body = p.call_args.kwargs["json"] - assert "max_tokens" not in body + assert body["max_tokens"] == 8192 assert body["chat_template_kwargs"] == {"enable_thinking": False} assert body["response_format"]["json_schema"]["schema"] == {"type": "object"} +def test_extract_schema_caps_requirements_list(): + reqs = llm.EXTRACT_SCHEMA["properties"]["requirements"] + assert reqs["maxItems"] == 40 + + def test_chat_json_strips_markdown_fences(): content = "```json\n{\"a\": 2}\n```" with mock.patch("projektmatch.llm.requests.post", fake_post([content])): diff --git a/projekt-matching/tests/test_mailer.py b/projekt-matching/tests/test_mailer.py index a1e4119..bb67187 100644 --- a/projekt-matching/tests/test_mailer.py +++ b/projekt-matching/tests/test_mailer.py @@ -83,6 +83,28 @@ def test_send_mail_survives_append_failure(): mailer.send_mail("u@x", "pw", "to@x", "Subj", "Body") +def test_ensure_alive_keeps_healthy_connection(): + conn = fake_conn() + conn.noop.return_value = ("OK", [b""]) + box = mailer.MailBox("u", "p", conn=conn) + box.ensure_alive() + assert box.conn is conn + conn.noop.assert_called_once() + + +def test_ensure_alive_reconnects_on_dead_connection(): + conn = fake_conn() + conn.noop.side_effect = OSError("connection closed") + box = mailer.MailBox("u", "p", conn=conn) + fresh = fake_conn() + with mock.patch("projektmatch.mailer.imaplib.IMAP4_SSL", + return_value=fresh): + box.ensure_alive() + assert box.conn is fresh + fresh.login.assert_called_once_with("u", "p") + fresh.select.assert_called_once_with("INBOX") + + def test_append_sent_uses_special_use_folder(): conn = mock.Mock() conn.list.return_value = ("OK", [ diff --git a/projekt-matching/tests/test_stages.py b/projekt-matching/tests/test_stages.py index ebb038f..bb9a48e 100644 --- a/projekt-matching/tests/test_stages.py +++ b/projekt-matching/tests/test_stages.py @@ -33,6 +33,68 @@ def ctx_after_match(ratings): return ctx +PROJECT_PAGE = """ + + +
+
+

ISTQB Testautomatisierung (m/w/d)

+ iSAX Consulting GmbH + Start 8/2026 +
+
+
+ ISTQBTestsuiteIBM Aix +
+
+

Beschreibung: Muss-Anforderungen: 3 Jahre Testautomatisierung. + Erstellung, Beratung und methodische Unterstuetzung der Projektteams + bei Testkonzept und Teststrategie. Organisation der Testkoordination + fuer einen uebergreifenden Test. Durchfuehrung von manuellen und + automatisierten Tests sowie Last- und Performancetests.

+
+
+
+ + +""" + + +def test_page_text_keeps_header_and_description_drops_tag_cloud(): + text = stages.page_text(PROJECT_PAGE) + assert "ISTQB Testautomatisierung (m/w/d)" in text + assert "Start 8/2026" in text + assert "Muss-Anforderungen: 3 Jahre Testautomatisierung" in text + assert "IBM Aix" not in text + assert "Testsuite" not in text + assert "Impressum" not in text + + +def test_page_text_replaces_quotes_that_break_json_strings(): + html = ('
Zertifizierung „ISTQB Certified Tester" ' + 'sowie "Scrum" und “Kanban”.
') + text = stages.page_text(html) + assert '"' not in text + assert "„" not in text and "“" not in text and "”" not in text + assert "'ISTQB Certified Tester'" in text + + +def test_page_text_falls_back_to_whole_page(): + html = "

Nur Text ohne content-Div.

" + assert "Nur Text ohne content-Div." in stages.page_text(html) + + +def test_stage_fetch_uses_project_section(): + resp = mock.Mock(status_code=200, text=PROJECT_PAGE) + resp.raise_for_status = lambda: None + ctx = {"canonical": "https://www.freelancermap.de/projekt/x", "title": "t"} + with mock.patch("projektmatch.stages.requests.get", return_value=resp): + out = stages.stage_fetch(dict(ctx), CFG) + assert out["status"] == "ok" + assert "IBM Aix" not in out["page_text"] + assert "Muss-Anforderungen" in out["page_text"] + + def test_run_stage_catches_and_skips(): def boom(ctx, cfg): raise ValueError("kaputt")