tlg
36eac66abc
Root cause chain (evidence: 25+ failed traces on nproj/3022414): straight quote in page text ends the guided-decoding JSON string early -> grammar deadlock -> whitespace loop until context/token limit -> chat_json fails 3x (~35 min) -> IMAP conn idles out -> move_to_trash raises -> mail never trashed -> reprocessed forever. - ingest/mailer: MailBox.ensure_alive() reconnect before finalize; move_to_trash failure falls back to flag_checked, run continues - stages: page_text() feeds only div.content minus skill-tag cloud to the LLM and maps quote chars to apostrophes (the actual degeneration trigger, live-verified: extract now OK in 21 s) - llm: requirements maxItems 40; max_tokens 8192 caps degenerate runs at ~100 s instead of ~10 min Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
221 lines
8.6 KiB
Python
221 lines
8.6 KiB
Python
import json
|
|
|
|
import pytest
|
|
from unittest import mock
|
|
|
|
from projektmatch import ingest
|
|
from projektmatch.config import Cfg
|
|
|
|
|
|
PROJECT_HTML = ('<a href="https://www.freelancermap.de/nproj/1.html?x=1">'
|
|
'Projekt Eins</a>')
|
|
|
|
|
|
def cfg(tmp_path):
|
|
return Cfg(imap_password="pw", alert_to="a@x", flow2_id="F2",
|
|
langflow_api_key="key", data_dir=str(tmp_path))
|
|
|
|
|
|
def run(tmp_path, mails, dup=None, dispatch_result=None):
|
|
box = mock.Mock()
|
|
box.unchecked_uids.return_value = list(mails)
|
|
box.fetch.side_effect = lambda uid: mails[uid]
|
|
espo = mock.Mock()
|
|
espo.find_opportunity_by_link.return_value = dup
|
|
sent = []
|
|
with mock.patch("projektmatch.ingest.mailparse.bodies",
|
|
side_effect=lambda m: (m["html"], "")), \
|
|
mock.patch("projektmatch.ingest.mailer.send_mail",
|
|
lambda *a: sent.append(a)):
|
|
summary = ingest.run_ingest(
|
|
cfg(tmp_path), mailbox=box, espo=espo,
|
|
resolve=lambda url: "https://www.freelancermap.de/projekt/eins",
|
|
dispatch=lambda item: dispatch_result or {"status": "created"})
|
|
return summary, box, sent
|
|
|
|
|
|
def mail(subject, html=""):
|
|
return {"Subject": subject, "html": html}
|
|
|
|
|
|
def test_non_project_mail_flagged_not_trashed(tmp_path):
|
|
mails = {"1": mail("Newsletter", "<p>Hallo</p>")}
|
|
summary, box, sent = run(tmp_path, mails)
|
|
box.flag_checked.assert_called_once_with("1")
|
|
box.move_to_trash.assert_not_called()
|
|
assert summary["mails"] == 0 and not sent
|
|
|
|
|
|
def test_own_mail_skipped(tmp_path):
|
|
mails = {"1": mail("[Projekt-Match] Freelancermap — 90 % — X",
|
|
PROJECT_HTML)}
|
|
summary, box, sent = run(tmp_path, mails)
|
|
box.flag_checked.assert_called_once_with("1")
|
|
box.move_to_trash.assert_not_called()
|
|
|
|
|
|
def test_project_mail_created_and_trashed(tmp_path):
|
|
mails = {"1": mail("Example 1 for Langflow process", PROJECT_HTML)}
|
|
summary, box, sent = run(tmp_path, mails)
|
|
assert summary == {"mails": 1, "projects": 1, "created": 1,
|
|
"rejected": 0, "duplicate": 0, "failed": 0}
|
|
box.move_to_trash.assert_called_once_with("1")
|
|
assert not sent
|
|
|
|
|
|
def test_duplicate_skips_dispatch(tmp_path):
|
|
mails = {"1": mail("Mail", PROJECT_HTML)}
|
|
summary, box, sent = run(tmp_path, mails, dup={"id": "X"})
|
|
assert summary["duplicate"] == 1 and summary["created"] == 0
|
|
box.move_to_trash.assert_called_once()
|
|
|
|
|
|
def test_failed_dispatch_alerts_and_trashes(tmp_path):
|
|
mails = {"1": mail("Mail", PROJECT_HTML)}
|
|
summary, box, sent = run(tmp_path, mails,
|
|
dispatch_result={"status": "failed",
|
|
"error": "vLLM down"})
|
|
assert summary["failed"] == 1
|
|
assert len(sent) == 1
|
|
_, _, to, subject, body = sent[0]
|
|
assert to == "a@x"
|
|
assert subject == "[Projekt-Match-Fehler] Freelancermap — 1 Projekt(e)"
|
|
assert "vLLM down" in body
|
|
box.move_to_trash.assert_called_once()
|
|
|
|
|
|
def test_lock_prevents_second_run(tmp_path):
|
|
lock = ingest.acquire_lock(str(tmp_path))
|
|
assert lock is not None
|
|
summary = ingest.run_ingest(cfg(tmp_path), mailbox=mock.Mock())
|
|
assert summary == {"skipped": "locked"}
|
|
ingest.release_lock(lock)
|
|
|
|
|
|
def test_unknown_dispatch_status_normalized_to_failed(tmp_path):
|
|
mails = {"1": mail("Mail", PROJECT_HTML)}
|
|
summary, box, sent = run(tmp_path, mails,
|
|
dispatch_result={"weird": True})
|
|
assert summary["failed"] == 1
|
|
box.move_to_trash.assert_called_once()
|
|
|
|
|
|
def test_alert_send_failure_still_trashes(tmp_path):
|
|
mails = {"1": mail("Mail", PROJECT_HTML)}
|
|
box = mock.Mock()
|
|
box.unchecked_uids.return_value = list(mails)
|
|
box.fetch.side_effect = lambda uid: mails[uid]
|
|
espo = mock.Mock()
|
|
espo.find_opportunity_by_link.return_value = None
|
|
def boom_send(*a):
|
|
raise OSError("smtp down")
|
|
with mock.patch("projektmatch.ingest.mailparse.bodies",
|
|
side_effect=lambda m: (m["html"], "")), \
|
|
mock.patch("projektmatch.ingest.mailer.send_mail", boom_send):
|
|
summary = ingest.run_ingest(
|
|
cfg(tmp_path), mailbox=box, espo=espo,
|
|
resolve=lambda url: "https://www.freelancermap.de/projekt/eins",
|
|
dispatch=lambda item: {"status": "failed", "error": "x"})
|
|
assert summary["alertFailed"] == 1
|
|
box.move_to_trash.assert_called_once_with("1")
|
|
|
|
|
|
def test_unreadable_mail_flagged_pipeline_continues(tmp_path):
|
|
good = mail("Example 2 for Langflow process", PROJECT_HTML)
|
|
box = mock.Mock()
|
|
box.unchecked_uids.return_value = ["1", "2"]
|
|
|
|
def fetch_side_effect(uid):
|
|
if uid == "1":
|
|
raise RuntimeError("FETCH 1 fehlgeschlagen: NO")
|
|
return good
|
|
box.fetch.side_effect = fetch_side_effect
|
|
espo = mock.Mock()
|
|
espo.find_opportunity_by_link.return_value = None
|
|
with mock.patch("projektmatch.ingest.mailparse.bodies",
|
|
side_effect=lambda m: (m["html"], "")), \
|
|
mock.patch("projektmatch.ingest.mailer.send_mail",
|
|
lambda *a: None):
|
|
summary = ingest.run_ingest(
|
|
cfg(tmp_path), mailbox=box, espo=espo,
|
|
resolve=lambda url: "https://www.freelancermap.de/projekt/eins",
|
|
dispatch=lambda item: {"status": "created"})
|
|
assert summary["unreadable"] == 1
|
|
assert summary["created"] == 1
|
|
box.flag_checked.assert_called_once_with("1")
|
|
box.move_to_trash.assert_called_once_with("2")
|
|
box.close.assert_called_once()
|
|
|
|
|
|
def test_close_called_even_if_unchecked_uids_raises(tmp_path):
|
|
box = mock.Mock()
|
|
box.unchecked_uids.side_effect = RuntimeError("boom")
|
|
espo = mock.Mock()
|
|
with pytest.raises(RuntimeError):
|
|
ingest.run_ingest(cfg(tmp_path), mailbox=box, espo=espo)
|
|
box.close.assert_called_once()
|
|
|
|
|
|
def test_ensure_alive_called_before_finalize(tmp_path):
|
|
mails = {"1": mail("Mail", PROJECT_HTML)}
|
|
summary, box, sent = run(tmp_path, mails,
|
|
dispatch_result={"status": "failed",
|
|
"error": "Read timed out"})
|
|
names = [c[0] for c in box.method_calls]
|
|
assert "ensure_alive" in names
|
|
assert names.index("ensure_alive") < names.index("move_to_trash")
|
|
box.move_to_trash.assert_called_once_with("1")
|
|
assert len(sent) == 1
|
|
|
|
|
|
def test_trash_failure_falls_back_to_flag_and_continues(tmp_path):
|
|
mails = {"1": mail("Mail A", PROJECT_HTML),
|
|
"2": mail("Mail B", PROJECT_HTML)}
|
|
box = mock.Mock()
|
|
box.unchecked_uids.return_value = ["1", "2"]
|
|
box.fetch.side_effect = lambda uid: mails[uid]
|
|
box.move_to_trash.side_effect = [OSError("socket error: EOF"), None]
|
|
espo = mock.Mock()
|
|
espo.find_opportunity_by_link.return_value = None
|
|
with mock.patch("projektmatch.ingest.mailparse.bodies",
|
|
side_effect=lambda m: (m["html"], "")), \
|
|
mock.patch("projektmatch.ingest.mailer.send_mail", lambda *a: None):
|
|
summary = ingest.run_ingest(
|
|
cfg(tmp_path), mailbox=box, espo=espo,
|
|
resolve=lambda url: "https://www.freelancermap.de/projekt/eins",
|
|
dispatch=lambda item: {"status": "created"})
|
|
assert summary["trashFailed"] == 1
|
|
assert summary["created"] == 2
|
|
box.flag_checked.assert_called_once_with("1")
|
|
assert box.move_to_trash.call_count == 2
|
|
|
|
|
|
def test_dead_connection_everywhere_still_finishes_run(tmp_path):
|
|
mails = {"1": mail("Mail A", PROJECT_HTML),
|
|
"2": mail("Mail B", PROJECT_HTML)}
|
|
box = mock.Mock()
|
|
box.unchecked_uids.return_value = ["1", "2"]
|
|
box.fetch.side_effect = lambda uid: mails[uid]
|
|
box.ensure_alive.side_effect = OSError("server unreachable")
|
|
box.move_to_trash.side_effect = OSError("socket error: EOF")
|
|
box.flag_checked.side_effect = OSError("socket error: EOF")
|
|
espo = mock.Mock()
|
|
espo.find_opportunity_by_link.return_value = None
|
|
with mock.patch("projektmatch.ingest.mailparse.bodies",
|
|
side_effect=lambda m: (m["html"], "")), \
|
|
mock.patch("projektmatch.ingest.mailer.send_mail", lambda *a: None):
|
|
summary = ingest.run_ingest(
|
|
cfg(tmp_path), mailbox=box, espo=espo,
|
|
resolve=lambda url: "https://www.freelancermap.de/projekt/eins",
|
|
dispatch=lambda item: {"status": "created"})
|
|
assert summary["trashFailed"] == 2
|
|
assert summary["created"] == 2
|
|
box.close.assert_called_once()
|
|
|
|
|
|
def test_parse_run_result_finds_status_json():
|
|
inner = json.dumps({"status": "created", "mustMatch": 90})
|
|
payload = {"outputs": [{"outputs": [{"results": {"text": {
|
|
"data": {"text": inner}}}}]}]}
|
|
assert ingest.parse_run_result(payload)["status"] == "created"
|