From 9ca7767208cff09d36d18aed62c8fb291a4ffa583a69f60cd4c65c07737aed21 Mon Sep 17 00:00:00 2001 From: tlg Date: Thu, 9 Jul 2026 17:15:39 +0200 Subject: [PATCH] =?UTF-8?q?fix(projekt-matching):=20final-review=20fixes?= =?UTF-8?q?=20=E2=80=94=20safe=20redeploy,=20poison-mail=20hardening,=20pl?= =?UTF-8?q?an=20deviations=20note?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- .../2026-07-09-langflow-projekt-matching.md | 12 +++ projekt-matching/deploy/deploy_files.sh | 56 +++++----- projekt-matching/projektmatch/ingest.py | 101 ++++++++++-------- projekt-matching/projektmatch/mailer.py | 2 + projekt-matching/tests/test_ingest.py | 38 +++++++ projekt-matching/tests/test_mailer.py | 18 ++++ 6 files changed, 154 insertions(+), 73 deletions(-) diff --git a/docs/superpowers/plans/2026-07-09-langflow-projekt-matching.md b/docs/superpowers/plans/2026-07-09-langflow-projekt-matching.md index 5d46604..59b210d 100644 --- a/docs/superpowers/plans/2026-07-09-langflow-projekt-matching.md +++ b/docs/superpowers/plans/2026-07-09-langflow-projekt-matching.md @@ -25,6 +25,18 @@ - Langflow API auth (v1.10.0): flow CRUD + variables need Bearer JWT from `GET /api/v1/auto_login`; `POST /api/v1/run/...` and webhook need `x-api-key`. Responses are gzip → `curl --compressed`. - Langflow data dir: host `~/.local/share/langflow_pod/langflow-data` = container `/app/langflow`. Deployed files need group-read (dir is `g+rwxs`, uid-1000/gid-0 process). +## Deviations discovered during execution + +- `response_format json_schema` replaces `guided_json` (vLLM 0.22 ignores `guided_json`). +- `chat_template_kwargs` `enable_thinking: false` is required for structured calls. +- Component input `ctx` was renamed `ctx_in` (Langflow reserves `Component.ctx`). +- `PYTHONPATH=/app/langflow` env var replaces the `.pth`-file approach. +- Langfuse org/project provisioning goes through the session tRPC endpoint (the admin API is EE-gated). +- Flow 1 is a single `PMIngest` component (not the originally sketched multi-node graph). +- `PMNotify` calls `stage_notify` directly rather than going through an intermediate layer. + +Task-level code snippets further below in this document were NOT retro-edited to reflect these deviations; the `projektmatch` package source under `/home/lwc/bin/projekt-matching/` is authoritative. + ## File Structure ``` diff --git a/projekt-matching/deploy/deploy_files.sh b/projekt-matching/deploy/deploy_files.sh index 1db4985..dbd0429 100755 --- a/projekt-matching/deploy/deploy_files.sh +++ b/projekt-matching/deploy/deploy_files.sh @@ -26,42 +26,46 @@ done # Langflow runs as uid 1000 gid 0 -> needs group read chmod -R g+rX "$DEST/projektmatch" "$DEST/vorgaben" -# Make /app/langflow importable as a Python path root inside the container. -# The component files (Task 9) do `sys.path.insert(0, "/app/langflow")` as -# a *runtime* statement, but Langflow 1.10's custom-component loader -# (lfx/custom/validate.py: prepare_global_scope/create_class) statically -# scans the module's top-level `import`/`from ... import` AST nodes and -# resolves them via importlib BEFORE executing any other code -- an `if:` -# block containing the sys.path hack is a plain ast.If node, which that -# loader silently drops without ever executing it. So "import projektmatch" -# 404s both at design time (POST /custom_component, used by build_flows.py) -# and at real flow-run time, regardless of the in-file sys.path trick, -# unless /app/langflow is already on sys.path when the interpreter starts. -# Fix: drop a .pth file into the venv's site-packages (only takes effect -# for *new* Python processes, so the running langflow server needs a -# restart to pick it up -- the venv itself lives in the container's -# ephemeral overlay, not the bind-mounted data dir, so this must be -# redone after any container recreation). +# /app/langflow is already on sys.path inside the container: create_pod_langflow.sh +# runs the langflow container with `-e PYTHONPATH=/app/langflow` baked into the +# `podman run` invocation itself (see the comment block above that env var in +# create_pod_langflow.sh), so "import projektmatch" resolves from the very first +# boot without any venv-local .pth file. Do NOT reintroduce a .pth here: this +# container's systemd unit is generated with `podman generate systemd --new` +# (--rm + --replace in ExecStart), so the venv's site-packages lives in the +# container's ephemeral overlay and is wiped on every recreation anyway. +# +# The freshly copied code still needs a running-process restart to take effect +# (Python doesn't hot-reload modules), so restart via the systemd user unit -- +# never a bare `podman restart`: since the unit is `--new`-managed, a bare +# `podman restart` on a container systemd also manages can leave systemd's +# view of the unit out of sync with the container's actual state (documented +# pitfall: the unit can end up believing the container is REMOVED). Restarting +# through systemd keeps systemd and podman in agreement. CTR="langflow_ctr" +SVC="container-langflow_ctr.service" if podman inspect "$CTR" >/dev/null 2>&1; then - SITE_PKGS="$(podman exec "$CTR" python -c 'import site; print(site.getsitepackages()[0])')" - PTH_FILE="$SITE_PKGS/projektmatch.pth" - CURRENT="$(podman exec "$CTR" cat "$PTH_FILE" 2>/dev/null || true)" - if [ "$CURRENT" != "/app/langflow" ]; then - echo "Registering /app/langflow on container sys.path (restart required)..." - podman exec "$CTR" sh -c "echo /app/langflow > '$PTH_FILE'" - podman restart "$CTR" >/dev/null + echo "Restarting $CTR via $SVC to pick up new code..." + if systemctl --user restart "$SVC"; then + code="" for i in $(seq 1 40); do - code=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:8090/api/v1/auto_login || true) + code=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:8090/health || true) [ "$code" = "200" ] && break sleep 3 done if [ "$code" != "200" ]; then - echo "WARN: $CTR did not come back healthy after restart (last health code: $code)" >&2 + echo "WARN: $CTR did not come back healthy within 120s after restart" \ + "(last health code: ${code:-none}). Recover with:" \ + "'systemctl --user restart $SVC && curl -fsS http://127.0.0.1:8090/health'" >&2 fi + else + echo "WARN: 'systemctl --user restart $SVC' failed - $CTR may not have" \ + "picked up the new code. Recover with:" \ + "'systemctl --user restart $SVC'" >&2 fi else - echo "WARN: container $CTR not found - skipping sys.path fix" >&2 + echo "WARN: container $CTR not found - skipping restart" \ + "(recover with: 'systemctl --user start $SVC')" >&2 fi echo "Deployed to $DEST" diff --git a/projekt-matching/projektmatch/ingest.py b/projekt-matching/projektmatch/ingest.py index cb579a0..a3d2215 100644 --- a/projekt-matching/projektmatch/ingest.py +++ b/projekt-matching/projektmatch/ingest.py @@ -83,53 +83,60 @@ def run_ingest(cfg, mailbox=None, dispatch=None, espo=None, resolve=None): resolve = resolve or mailparse.canonical_url summary = {"mails": 0, "projects": 0, **{k: 0 for k in SUMMARY_KEYS}} - for uid in box.unchecked_uids(): - msg = box.fetch(uid) - subject = decode_subject(msg) - html, text = mailparse.bodies(msg) - items = [] if mailparse.is_own_mail(subject) else \ - mailparse.split_projects(html, text) - if not items: - box.flag_checked(uid) - continue - summary["mails"] += 1 - results = [] - for item in items: - summary["projects"] += 1 - result = dict(item) + try: + for uid in box.unchecked_uids(): try: - result["canonical"] = resolve(item["url"]) - dup = crm.find_opportunity_by_link(result["canonical"]) - if dup: - result["status"] = "duplicate" - else: - result.update(send(result)) - except Exception as exc: # noqa: BLE001 - result.update(status="failed", error=str(exc)) - if result.get("status") not in SUMMARY_KEYS: - result.update( - status="failed", - error=result.get("error") - or f"unbekannter Status: {result.get('status')!r}") - results.append(result) - summary[result["status"]] += 1 - failed = [r for r in results if r["status"] == "failed"] - if failed: - lines = "\n\n".join( - f"- {f.get('title', '?')}\n" - f" {f.get('canonical', f.get('url', '?'))}\n" - f" Fehler: {f.get('error', '?')}" for f in failed) - try: - mailer.send_mail( - cfg.imap_user, cfg.imap_password, cfg.alert_to, - f"[Projekt-Match-Fehler] Freelancermap — " - f"{len(failed)} Projekt(e)", - "Folgende Projekte konnten nicht verarbeitet " - "werden:\n\n" + lines + "\n") - except Exception: # noqa: BLE001 - summary["alertFailed"] = summary.get("alertFailed", 0) + 1 - box.move_to_trash(uid) - box.close() - return summary + msg = box.fetch(uid) + subject = decode_subject(msg) + html, text = mailparse.bodies(msg) + except Exception: # noqa: BLE001 + box.flag_checked(uid) # nie wieder anfassen, Pipeline läuft weiter + summary["unreadable"] = summary.get("unreadable", 0) + 1 + continue + items = [] if mailparse.is_own_mail(subject) else \ + mailparse.split_projects(html, text) + if not items: + box.flag_checked(uid) + continue + summary["mails"] += 1 + results = [] + for item in items: + summary["projects"] += 1 + result = dict(item) + try: + result["canonical"] = resolve(item["url"]) + dup = crm.find_opportunity_by_link(result["canonical"]) + if dup: + result["status"] = "duplicate" + else: + result.update(send(result)) + except Exception as exc: # noqa: BLE001 + result.update(status="failed", error=str(exc)) + if result.get("status") not in SUMMARY_KEYS: + result.update( + status="failed", + error=result.get("error") + or f"unbekannter Status: {result.get('status')!r}") + results.append(result) + summary[result["status"]] += 1 + failed = [r for r in results if r["status"] == "failed"] + if failed: + lines = "\n\n".join( + f"- {f.get('title', '?')}\n" + f" {f.get('canonical', f.get('url', '?'))}\n" + f" Fehler: {f.get('error', '?')}" for f in failed) + try: + mailer.send_mail( + cfg.imap_user, cfg.imap_password, cfg.alert_to, + f"[Projekt-Match-Fehler] Freelancermap — " + f"{len(failed)} Projekt(e)", + "Folgende Projekte konnten nicht verarbeitet " + "werden:\n\n" + lines + "\n") + except Exception: # noqa: BLE001 + summary["alertFailed"] = summary.get("alertFailed", 0) + 1 + box.move_to_trash(uid) + return summary + finally: + box.close() finally: release_lock(lock) diff --git a/projekt-matching/projektmatch/mailer.py b/projekt-matching/projektmatch/mailer.py index 66485e7..9dbdd0b 100644 --- a/projekt-matching/projektmatch/mailer.py +++ b/projekt-matching/projektmatch/mailer.py @@ -30,6 +30,8 @@ class MailBox: def fetch(self, uid): typ, data = self.conn.uid("FETCH", uid, "(BODY.PEEK[])") + if typ != "OK" or not data or not data[0] or not isinstance(data[0], tuple): + raise RuntimeError(f"FETCH {uid} fehlgeschlagen: {typ}") return email.message_from_bytes(data[0][1], policy=default_policy) def flag_checked(self, uid): diff --git a/projekt-matching/tests/test_ingest.py b/projekt-matching/tests/test_ingest.py index 5ad20aa..4cb5483 100644 --- a/projekt-matching/tests/test_ingest.py +++ b/projekt-matching/tests/test_ingest.py @@ -1,4 +1,6 @@ import json + +import pytest from unittest import mock from projektmatch import ingest @@ -118,6 +120,42 @@ def test_alert_send_failure_still_trashes(tmp_path): box.move_to_trash.assert_called_once_with("1") +def test_unreadable_mail_flagged_pipeline_continues(tmp_path): + good = mail("Example 2 for Langflow process", PROJECT_HTML) + box = mock.Mock() + box.unchecked_uids.return_value = ["1", "2"] + + def fetch_side_effect(uid): + if uid == "1": + raise RuntimeError("FETCH 1 fehlgeschlagen: NO") + return good + box.fetch.side_effect = fetch_side_effect + espo = mock.Mock() + espo.find_opportunity_by_link.return_value = None + with mock.patch("projektmatch.ingest.mailparse.bodies", + side_effect=lambda m: (m["html"], "")), \ + mock.patch("projektmatch.ingest.mailer.send_mail", + lambda *a: None): + summary = ingest.run_ingest( + cfg(tmp_path), mailbox=box, espo=espo, + resolve=lambda url: "https://www.freelancermap.de/projekt/eins", + dispatch=lambda item: {"status": "created"}) + assert summary["unreadable"] == 1 + assert summary["created"] == 1 + box.flag_checked.assert_called_once_with("1") + box.move_to_trash.assert_called_once_with("2") + box.close.assert_called_once() + + +def test_close_called_even_if_unchecked_uids_raises(tmp_path): + box = mock.Mock() + box.unchecked_uids.side_effect = RuntimeError("boom") + espo = mock.Mock() + with pytest.raises(RuntimeError): + ingest.run_ingest(cfg(tmp_path), mailbox=box, espo=espo) + box.close.assert_called_once() + + def test_parse_run_result_finds_status_json(): inner = json.dumps({"status": "created", "mustMatch": 90}) payload = {"outputs": [{"outputs": [{"results": {"text": { diff --git a/projekt-matching/tests/test_mailer.py b/projekt-matching/tests/test_mailer.py index 29385db..b29227a 100644 --- a/projekt-matching/tests/test_mailer.py +++ b/projekt-matching/tests/test_mailer.py @@ -1,4 +1,6 @@ import email + +import pytest from unittest import mock from projektmatch import mailer @@ -43,6 +45,22 @@ def test_move_uses_move_then_fallback(): assert conn.expunge.called +def test_fetch_raises_on_no_response(): + conn = fake_conn() + conn.uid.return_value = ("NO", [None]) + box = mailer.MailBox("u", "p", conn=conn) + with pytest.raises(RuntimeError): + box.fetch("3") + + +def test_fetch_raises_on_malformed_ok_response(): + conn = fake_conn() + conn.uid.return_value = ("OK", [None]) + box = mailer.MailBox("u", "p", conn=conn) + with pytest.raises(RuntimeError): + box.fetch("3") + + def test_send_mail_starttls(): with mock.patch("projektmatch.mailer.smtplib.SMTP") as smtp: server = smtp.return_value.__enter__.return_value