fix(projekt-matching): final-review fixes — safe redeploy, poison-mail hardening, plan deviations note

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-09 17:15:39 +02:00
parent 1863ce087c
commit 9ca7767208
6 changed files with 154 additions and 73 deletions
--- a/docs/superpowers/plans/2026-07-09-langflow-projekt-matching.md
+++ b/docs/superpowers/plans/2026-07-09-langflow-projekt-matching.md
@@ -25,6 +25,18 @@
 - Langflow API auth (v1.10.0): flow CRUD + variables need Bearer JWT from `GET /api/v1/auto_login`; `POST /api/v1/run/...` and webhook need `x-api-key`. Responses are gzip → `curl --compressed`.
 - Langflow data dir: host `~/.local/share/langflow_pod/langflow-data` = container `/app/langflow`. Deployed files need group-read (dir is `g+rwxs`, uid-1000/gid-0 process).

+## Deviations discovered during execution
+
+- `response_format json_schema` replaces `guided_json` (vLLM 0.22 ignores `guided_json`).
+- `chat_template_kwargs` `enable_thinking: false` is required for structured calls.
+- Component input `ctx` was renamed `ctx_in` (Langflow reserves `Component.ctx`).
+- `PYTHONPATH=/app/langflow` env var replaces the `.pth`-file approach.
+- Langfuse org/project provisioning goes through the session tRPC endpoint (the admin API is EE-gated).
+- Flow 1 is a single `PMIngest` component (not the originally sketched multi-node graph).
+- `PMNotify` calls `stage_notify` directly rather than going through an intermediate layer.
+
+Task-level code snippets further below in this document were NOT retro-edited to reflect these deviations; the `projektmatch` package source under `/home/lwc/bin/projekt-matching/` is authoritative.
+
 ## File Structure

 ```
--- a/projekt-matching/deploy/deploy_files.sh
+++ b/projekt-matching/deploy/deploy_files.sh
@@ -26,42 +26,46 @@ done
 # Langflow runs as uid 1000 gid 0 -> needs group read
 chmod -R g+rX "$DEST/projektmatch" "$DEST/vorgaben"

-# Make /app/langflow importable as a Python path root inside the container.
-# The component files (Task 9) do `sys.path.insert(0, "/app/langflow")` as
-# a *runtime* statement, but Langflow 1.10's custom-component loader
-# (lfx/custom/validate.py: prepare_global_scope/create_class) statically
-# scans the module's top-level `import`/`from ... import` AST nodes and
-# resolves them via importlib BEFORE executing any other code -- an `if:`
-# block containing the sys.path hack is a plain ast.If node, which that
-# loader silently drops without ever executing it. So "import projektmatch"
-# 404s both at design time (POST /custom_component, used by build_flows.py)
-# and at real flow-run time, regardless of the in-file sys.path trick,
-# unless /app/langflow is already on sys.path when the interpreter starts.
-# Fix: drop a .pth file into the venv's site-packages (only takes effect
-# for *new* Python processes, so the running langflow server needs a
-# restart to pick it up -- the venv itself lives in the container's
-# ephemeral overlay, not the bind-mounted data dir, so this must be
-# redone after any container recreation).
+# /app/langflow is already on sys.path inside the container: create_pod_langflow.sh
+# runs the langflow container with `-e PYTHONPATH=/app/langflow` baked into the
+# `podman run` invocation itself (see the comment block above that env var in
+# create_pod_langflow.sh), so "import projektmatch" resolves from the very first
+# boot without any venv-local .pth file. Do NOT reintroduce a .pth here: this
+# container's systemd unit is generated with `podman generate systemd --new`
+# (--rm + --replace in ExecStart), so the venv's site-packages lives in the
+# container's ephemeral overlay and is wiped on every recreation anyway.
+#
+# The freshly copied code still needs a running-process restart to take effect
+# (Python doesn't hot-reload modules), so restart via the systemd user unit --
+# never a bare `podman restart`: since the unit is `--new`-managed, a bare
+# `podman restart` on a container systemd also manages can leave systemd's
+# view of the unit out of sync with the container's actual state (documented
+# pitfall: the unit can end up believing the container is REMOVED). Restarting
+# through systemd keeps systemd and podman in agreement.
 CTR="langflow_ctr"
+SVC="container-langflow_ctr.service"
 if podman inspect "$CTR" >/dev/null 2>&1; then
-  SITE_PKGS="$(podman exec "$CTR" python -c 'import site; print(site.getsitepackages()[0])')"
-  PTH_FILE="$SITE_PKGS/projektmatch.pth"
-  CURRENT="$(podman exec "$CTR" cat "$PTH_FILE" 2>/dev/null || true)"
-  if [ "$CURRENT" != "/app/langflow" ]; then
-    echo "Registering /app/langflow on container sys.path (restart required)..."
-    podman exec "$CTR" sh -c "echo /app/langflow > '$PTH_FILE'"
-    podman restart "$CTR" >/dev/null
+  echo "Restarting $CTR via $SVC to pick up new code..."
+  if systemctl --user restart "$SVC"; then
+    code=""
    for i in $(seq 1 40); do
-      code=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:8090/api/v1/auto_login || true)
+      code=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:8090/health || true)
      [ "$code" = "200" ] && break
      sleep 3
    done
    if [ "$code" != "200" ]; then
-      echo "WARN: $CTR did not come back healthy after restart (last health code: $code)" >&2
+      echo "WARN: $CTR did not come back healthy within 120s after restart" \
+           "(last health code: ${code:-none}). Recover with:" \
+           "'systemctl --user restart $SVC && curl -fsS http://127.0.0.1:8090/health'" >&2
    fi
+  else
+    echo "WARN: 'systemctl --user restart $SVC' failed - $CTR may not have" \
+         "picked up the new code. Recover with:" \
+         "'systemctl --user restart $SVC'" >&2
  fi
 else
-  echo "WARN: container $CTR not found - skipping sys.path fix" >&2
+  echo "WARN: container $CTR not found - skipping restart" \
+       "(recover with: 'systemctl --user start $SVC')" >&2
 fi

 echo "Deployed to $DEST"
--- a/projekt-matching/projektmatch/ingest.py
+++ b/projekt-matching/projektmatch/ingest.py
@@ -83,53 +83,60 @@ def run_ingest(cfg, mailbox=None, dispatch=None, espo=None, resolve=None):
        resolve = resolve or mailparse.canonical_url
        summary = {"mails": 0, "projects": 0,
                   **{k: 0 for k in SUMMARY_KEYS}}
-        for uid in box.unchecked_uids():
-            msg = box.fetch(uid)
-            subject = decode_subject(msg)
-            html, text = mailparse.bodies(msg)
-            items = [] if mailparse.is_own_mail(subject) else \
-                mailparse.split_projects(html, text)
-            if not items:
-                box.flag_checked(uid)
-                continue
-            summary["mails"] += 1
-            results = []
-            for item in items:
-                summary["projects"] += 1
-                result = dict(item)
+        try:
+            for uid in box.unchecked_uids():
                try:
-                    result["canonical"] = resolve(item["url"])
-                    dup = crm.find_opportunity_by_link(result["canonical"])
-                    if dup:
-                        result["status"] = "duplicate"
-                    else:
-                        result.update(send(result))
-                except Exception as exc:          # noqa: BLE001
-                    result.update(status="failed", error=str(exc))
-                if result.get("status") not in SUMMARY_KEYS:
-                    result.update(
-                        status="failed",
-                        error=result.get("error")
-                        or f"unbekannter Status: {result.get('status')!r}")
-                results.append(result)
-                summary[result["status"]] += 1
-            failed = [r for r in results if r["status"] == "failed"]
-            if failed:
-                lines = "\n\n".join(
-                    f"- {f.get('title', '?')}\n"
-                    f"  {f.get('canonical', f.get('url', '?'))}\n"
-                    f"  Fehler: {f.get('error', '?')}" for f in failed)
-                try:
-                    mailer.send_mail(
-                        cfg.imap_user, cfg.imap_password, cfg.alert_to,
-                        f"[Projekt-Match-Fehler] Freelancermap — "
-                        f"{len(failed)} Projekt(e)",
-                        "Folgende Projekte konnten nicht verarbeitet "
-                        "werden:\n\n" + lines + "\n")
-                except Exception:          # noqa: BLE001
-                    summary["alertFailed"] = summary.get("alertFailed", 0) + 1
-            box.move_to_trash(uid)
-        box.close()
-        return summary
+                    msg = box.fetch(uid)
+                    subject = decode_subject(msg)
+                    html, text = mailparse.bodies(msg)
+                except Exception:              # noqa: BLE001
+                    box.flag_checked(uid)      # nie wieder anfassen, Pipeline läuft weiter
+                    summary["unreadable"] = summary.get("unreadable", 0) + 1
+                    continue
+                items = [] if mailparse.is_own_mail(subject) else \
+                    mailparse.split_projects(html, text)
+                if not items:
+                    box.flag_checked(uid)
+                    continue
+                summary["mails"] += 1
+                results = []
+                for item in items:
+                    summary["projects"] += 1
+                    result = dict(item)
+                    try:
+                        result["canonical"] = resolve(item["url"])
+                        dup = crm.find_opportunity_by_link(result["canonical"])
+                        if dup:
+                            result["status"] = "duplicate"
+                        else:
+                            result.update(send(result))
+                    except Exception as exc:          # noqa: BLE001
+                        result.update(status="failed", error=str(exc))
+                    if result.get("status") not in SUMMARY_KEYS:
+                        result.update(
+                            status="failed",
+                            error=result.get("error")
+                            or f"unbekannter Status: {result.get('status')!r}")
+                    results.append(result)
+                    summary[result["status"]] += 1
+                failed = [r for r in results if r["status"] == "failed"]
+                if failed:
+                    lines = "\n\n".join(
+                        f"- {f.get('title', '?')}\n"
+                        f"  {f.get('canonical', f.get('url', '?'))}\n"
+                        f"  Fehler: {f.get('error', '?')}" for f in failed)
+                    try:
+                        mailer.send_mail(
+                            cfg.imap_user, cfg.imap_password, cfg.alert_to,
+                            f"[Projekt-Match-Fehler] Freelancermap — "
+                            f"{len(failed)} Projekt(e)",
+                            "Folgende Projekte konnten nicht verarbeitet "
+                            "werden:\n\n" + lines + "\n")
+                    except Exception:          # noqa: BLE001
+                        summary["alertFailed"] = summary.get("alertFailed", 0) + 1
+                box.move_to_trash(uid)
+            return summary
+        finally:
+            box.close()
    finally:
        release_lock(lock)
--- a/projekt-matching/projektmatch/mailer.py
+++ b/projekt-matching/projektmatch/mailer.py
@@ -30,6 +30,8 @@ class MailBox:

    def fetch(self, uid):
        typ, data = self.conn.uid("FETCH", uid, "(BODY.PEEK[])")
+        if typ != "OK" or not data or not data[0] or not isinstance(data[0], tuple):
+            raise RuntimeError(f"FETCH {uid} fehlgeschlagen: {typ}")
        return email.message_from_bytes(data[0][1], policy=default_policy)

    def flag_checked(self, uid):
--- a/projekt-matching/tests/test_ingest.py
+++ b/projekt-matching/tests/test_ingest.py
@@ -1,4 +1,6 @@
 import json
+
+import pytest
 from unittest import mock

 from projektmatch import ingest
@@ -118,6 +120,42 @@ def test_alert_send_failure_still_trashes(tmp_path):
    box.move_to_trash.assert_called_once_with("1")


+def test_unreadable_mail_flagged_pipeline_continues(tmp_path):
+    good = mail("Example 2 for Langflow process", PROJECT_HTML)
+    box = mock.Mock()
+    box.unchecked_uids.return_value = ["1", "2"]
+
+    def fetch_side_effect(uid):
+        if uid == "1":
+            raise RuntimeError("FETCH 1 fehlgeschlagen: NO")
+        return good
+    box.fetch.side_effect = fetch_side_effect
+    espo = mock.Mock()
+    espo.find_opportunity_by_link.return_value = None
+    with mock.patch("projektmatch.ingest.mailparse.bodies",
+                    side_effect=lambda m: (m["html"], "")), \
+         mock.patch("projektmatch.ingest.mailer.send_mail",
+                    lambda *a: None):
+        summary = ingest.run_ingest(
+            cfg(tmp_path), mailbox=box, espo=espo,
+            resolve=lambda url: "https://www.freelancermap.de/projekt/eins",
+            dispatch=lambda item: {"status": "created"})
+    assert summary["unreadable"] == 1
+    assert summary["created"] == 1
+    box.flag_checked.assert_called_once_with("1")
+    box.move_to_trash.assert_called_once_with("2")
+    box.close.assert_called_once()
+
+
+def test_close_called_even_if_unchecked_uids_raises(tmp_path):
+    box = mock.Mock()
+    box.unchecked_uids.side_effect = RuntimeError("boom")
+    espo = mock.Mock()
+    with pytest.raises(RuntimeError):
+        ingest.run_ingest(cfg(tmp_path), mailbox=box, espo=espo)
+    box.close.assert_called_once()
+
+
 def test_parse_run_result_finds_status_json():
    inner = json.dumps({"status": "created", "mustMatch": 90})
    payload = {"outputs": [{"outputs": [{"results": {"text": {
--- a/projekt-matching/tests/test_mailer.py
+++ b/projekt-matching/tests/test_mailer.py
@@ -1,4 +1,6 @@
 import email
+
+import pytest
 from unittest import mock

 from projektmatch import mailer
@@ -43,6 +45,22 @@ def test_move_uses_move_then_fallback():
    assert conn.expunge.called


+def test_fetch_raises_on_no_response():
+    conn = fake_conn()
+    conn.uid.return_value = ("NO", [None])
+    box = mailer.MailBox("u", "p", conn=conn)
+    with pytest.raises(RuntimeError):
+        box.fetch("3")
+
+
+def test_fetch_raises_on_malformed_ok_response():
+    conn = fake_conn()
+    conn.uid.return_value = ("OK", [None])
+    box = mailer.MailBox("u", "p", conn=conn)
+    with pytest.raises(RuntimeError):
+        box.fetch("3")
+
+
 def test_send_mail_starttls():
    with mock.patch("projektmatch.mailer.smtplib.SMTP") as smtp:
        server = smtp.return_value.__enter__.return_value