feat(projekt-matching): mail parsing, project splitting, canonical URLs

2026-07-09 10:11:25 +02:00
parent 3bbf6631fd
commit 6c4cd1a4c3
2 changed files with 152 additions and 0 deletions
--- a/projekt-matching/projektmatch/mailparse.py
+++ b/projekt-matching/projektmatch/mailparse.py
@@ -0,0 +1,83 @@
+"""E-mail parsing: own-mail detection, project splitting, canonical URLs."""
+from __future__ import annotations
+
+import re
+from urllib.parse import urlsplit, urlunsplit
+
+import requests
+from bs4 import BeautifulSoup
+
+OWN_PREFIXES = ("[Projekt-Match]", "[Projekt-Match-Fehler]")
+PROJECT_URL_RE = re.compile(
+    r"https?://(?:www\.)?freelancermap\.de/(?:nproj/|projekt/)[^\s\"'<>)\]]+")
+BROWSER_UA = ("Mozilla/5.0 (X11; Linux x86_64; rv:128.0) "
+              "Gecko/20100101 Firefox/128.0")
+
+
+def is_own_mail(subject) -> bool:
+    return bool(subject) and subject.strip().startswith(OWN_PREFIXES)
+
+
+def bodies(msg):
+    """Return (html, text) of the first text/html and text/plain parts."""
+    html = text = ""
+    for part in msg.walk():
+        if part.get_content_maintype() == "multipart":
+            continue
+        try:
+            payload = part.get_content()
+        except Exception:
+            continue
+        ctype = part.get_content_type()
+        if ctype == "text/html" and not html:
+            html = payload
+        elif ctype == "text/plain" and not text:
+            text = payload
+    return html, text
+
+
+def _url_key(url: str) -> str:
+    return urlsplit(url).path
+
+
+def split_projects(html: str, text: str) -> list:
+    """Extract project items {title, url} from a mail body. HTML preferred;
+    per project (URL path) the LONGEST anchor text wins (skips 'Zum Projekt'
+    buttons). Plain-text fallback: URL line + nearest preceding non-empty line."""
+    best = {}          # key -> {"title", "url"}
+    order = []
+    if html:
+        soup = BeautifulSoup(html, "html.parser")
+        for a in soup.find_all("a", href=PROJECT_URL_RE):
+            url = a["href"]
+            key = _url_key(url)
+            title = " ".join(a.get_text(" ", strip=True).split())
+            if key not in best:
+                best[key] = {"title": title, "url": url}
+                order.append(key)
+            elif len(title) > len(best[key]["title"]):
+                best[key]["title"] = title
+    if not best and text:
+        last_line = ""
+        for line in text.splitlines():
+            match = PROJECT_URL_RE.search(line)
+            stripped = line.strip()
+            if match:
+                key = _url_key(match.group(0))
+                if key not in best:
+                    best[key] = {"title": last_line or match.group(0),
+                                 "url": match.group(0)}
+                    order.append(key)
+            elif stripped:
+                last_line = stripped
+    return [best[k] for k in order if best[k]["title"]]
+
+
+def canonical_url(url: str, session=None, timeout=20) -> str:
+    """Follow redirects, strip query string and fragment."""
+    sess = session or requests.Session()
+    resp = sess.get(url, timeout=timeout, allow_redirects=True,
+                    headers={"User-Agent": BROWSER_UA})
+    resp.raise_for_status()
+    parts = urlsplit(resp.url)
+    return urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
--- a/projekt-matching/tests/test_mailparse.py
+++ b/projekt-matching/tests/test_mailparse.py
@@ -0,0 +1,69 @@
+import email
+from email.policy import default as default_policy
+from unittest import mock
+
+from projektmatch import mailparse
+
+HTML = """
+<html><body>
+<h2><a href="https://www.freelancermap.de/nproj/3020338.html?utm_source=x&t=1">
+Python Entwickler für eine KI-Anwendung (m/w/d)</a></h2>
+<p>Beschreibung ...</p>
+<a href="https://www.freelancermap.de/nproj/3020338.html?utm_source=x&t=1">Zum Projekt</a>
+<h2><a href="https://www.freelancermap.de/nproj/3020995.html?agent=2">
+Test Engineer (m/w/d)</a></h2>
+<a href="https://www.freelancermap.de/other/page.html">Impressum</a>
+</body></html>
+"""
+
+TEXT = """Neue Projekte fuer Sie:
+
+Python Entwickler für eine KI-Anwendung (m/w/d)
+https://www.freelancermap.de/nproj/3020338.html?utm_source=x
+
+Test Engineer (m/w/d)
+https://www.freelancermap.de/nproj/3020995.html?agent=2
+"""
+
+
+def test_is_own_mail():
+    assert mailparse.is_own_mail("[Projekt-Match] Freelancermap — 86 % — X")
+    assert mailparse.is_own_mail("[Projekt-Match-Fehler] Freelancermap — 2 Projekt(e)")
+    assert not mailparse.is_own_mail("Example 1 for Langflow process")
+    assert not mailparse.is_own_mail(None)
+
+
+def test_split_projects_html_dedupes_and_prefers_long_title():
+    items = mailparse.split_projects(HTML, "")
+    assert [i["title"] for i in items] == [
+        "Python Entwickler für eine KI-Anwendung (m/w/d)", "Test Engineer (m/w/d)"]
+    assert "3020338" in items[0]["url"]
+
+
+def test_split_projects_text_fallback():
+    items = mailparse.split_projects("", TEXT)
+    assert len(items) == 2
+    assert items[0]["title"] == "Python Entwickler für eine KI-Anwendung (m/w/d)"
+
+
+def test_split_projects_none_for_normal_mail():
+    assert mailparse.split_projects("<p>Hallo</p>", "Hallo") == []
+
+
+def test_bodies_multipart():
+    msg = email.message.EmailMessage(policy=default_policy)
+    msg["Subject"] = "x"
+    msg.set_content("plain body")
+    msg.add_alternative("<p>html body</p>", subtype="html")
+    html, text = mailparse.bodies(msg)
+    assert "html body" in html and "plain body" in text
+
+
+def test_canonical_url_strips_query():
+    session = mock.Mock()
+    session.get.return_value = mock.Mock(
+        url="https://www.freelancermap.de/projekt/python-entwickler?ref=1",
+        raise_for_status=lambda: None)
+    out = mailparse.canonical_url("https://www.freelancermap.de/nproj/1.html?x=1",
+                                  session=session)
+    assert out == "https://www.freelancermap.de/projekt/python-entwickler"