fix(projekt-matching): break infinite alert loop (issue 2)
Root cause chain (evidence: 25+ failed traces on nproj/3022414): straight quote in page text ends the guided-decoding JSON string early -> grammar deadlock -> whitespace loop until context/token limit -> chat_json fails 3x (~35 min) -> IMAP conn idles out -> move_to_trash raises -> mail never trashed -> reprocessed forever. - ingest/mailer: MailBox.ensure_alive() reconnect before finalize; move_to_trash failure falls back to flag_checked, run continues - stages: page_text() feeds only div.content minus skill-tag cloud to the LLM and maps quote chars to apostrophes (the actual degeneration trigger, live-verified: extract now OK in 21 s) - llm: requirements maxItems 40; max_tokens 8192 caps degenerate runs at ~100 s instead of ~10 min Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -20,6 +20,7 @@ TEAM_BY_OFFER = {"Projekt": "DesTEngS",
|
||||
"Arbeitnehmer-Angebot": "Arbeitnehmer",
|
||||
"ANÜ": "ANÜ"}
|
||||
MIN_PAGE_CHARS = 200
|
||||
QUOTES = str.maketrans({c: "'" for c in "„“”\"«»‹›"})
|
||||
|
||||
|
||||
def run_stage(name, fn, ctx, cfg, **kw):
|
||||
@@ -32,6 +33,25 @@ def run_stage(name, fn, ctx, cfg, **kw):
|
||||
return ctx
|
||||
|
||||
|
||||
def page_text(html):
|
||||
"""Project text for the LLM: header + description, WITHOUT the skill-tag
|
||||
cloud (div.project-body-badges, ~100 entries on some pages — blows the
|
||||
requirements list up until the model emits broken JSON) and without
|
||||
site chrome. Falls back to whole-page text if div.content is missing."""
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
for tag in soup(["script", "style", "noscript"]):
|
||||
tag.decompose()
|
||||
root = soup.find("div", class_="content") or soup
|
||||
for tag in root.find_all(class_="project-body-badges"):
|
||||
tag.decompose()
|
||||
# Anführungszeichen -> Apostroph: ein wörtlich übernommenes " beendet den
|
||||
# JSON-String vorzeitig und treibt guided decoding in eine Whitespace-
|
||||
# Endlosschleife (finish_reason "length", live verifiziert an nproj/3022414)
|
||||
return "\n".join(line for line in
|
||||
root.get_text("\n", strip=True).splitlines()
|
||||
if line.strip()).translate(QUOTES)
|
||||
|
||||
|
||||
def stage_fetch(ctx, cfg):
|
||||
last = None
|
||||
for attempt in range(3):
|
||||
@@ -39,12 +59,7 @@ def stage_fetch(ctx, cfg):
|
||||
resp = requests.get(ctx["canonical"], timeout=30,
|
||||
headers={"User-Agent": BROWSER_UA})
|
||||
resp.raise_for_status()
|
||||
soup = BeautifulSoup(resp.text, "html.parser")
|
||||
for tag in soup(["script", "style", "noscript"]):
|
||||
tag.decompose()
|
||||
text = "\n".join(line for line in
|
||||
soup.get_text("\n", strip=True).splitlines()
|
||||
if line.strip())
|
||||
text = page_text(resp.text)
|
||||
if len(text) < MIN_PAGE_CHARS:
|
||||
raise ValueError(
|
||||
f"Seitentext zu kurz ({len(text)} Zeichen) — "
|
||||
|
||||
Reference in New Issue
Block a user