feat(projekt-matching): mail parsing, project splitting, canonical URLs
This commit is contained in:
83
projekt-matching/projektmatch/mailparse.py
Normal file
83
projekt-matching/projektmatch/mailparse.py
Normal file
@@ -0,0 +1,83 @@
|
|||||||
|
"""E-mail parsing: own-mail detection, project splitting, canonical URLs."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
from urllib.parse import urlsplit, urlunsplit
|
||||||
|
|
||||||
|
import requests
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
|
OWN_PREFIXES = ("[Projekt-Match]", "[Projekt-Match-Fehler]")
|
||||||
|
PROJECT_URL_RE = re.compile(
|
||||||
|
r"https?://(?:www\.)?freelancermap\.de/(?:nproj/|projekt/)[^\s\"'<>)\]]+")
|
||||||
|
BROWSER_UA = ("Mozilla/5.0 (X11; Linux x86_64; rv:128.0) "
|
||||||
|
"Gecko/20100101 Firefox/128.0")
|
||||||
|
|
||||||
|
|
||||||
|
def is_own_mail(subject) -> bool:
|
||||||
|
return bool(subject) and subject.strip().startswith(OWN_PREFIXES)
|
||||||
|
|
||||||
|
|
||||||
|
def bodies(msg):
|
||||||
|
"""Return (html, text) of the first text/html and text/plain parts."""
|
||||||
|
html = text = ""
|
||||||
|
for part in msg.walk():
|
||||||
|
if part.get_content_maintype() == "multipart":
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
payload = part.get_content()
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
ctype = part.get_content_type()
|
||||||
|
if ctype == "text/html" and not html:
|
||||||
|
html = payload
|
||||||
|
elif ctype == "text/plain" and not text:
|
||||||
|
text = payload
|
||||||
|
return html, text
|
||||||
|
|
||||||
|
|
||||||
|
def _url_key(url: str) -> str:
|
||||||
|
return urlsplit(url).path
|
||||||
|
|
||||||
|
|
||||||
|
def split_projects(html: str, text: str) -> list:
|
||||||
|
"""Extract project items {title, url} from a mail body. HTML preferred;
|
||||||
|
per project (URL path) the LONGEST anchor text wins (skips 'Zum Projekt'
|
||||||
|
buttons). Plain-text fallback: URL line + nearest preceding non-empty line."""
|
||||||
|
best = {} # key -> {"title", "url"}
|
||||||
|
order = []
|
||||||
|
if html:
|
||||||
|
soup = BeautifulSoup(html, "html.parser")
|
||||||
|
for a in soup.find_all("a", href=PROJECT_URL_RE):
|
||||||
|
url = a["href"]
|
||||||
|
key = _url_key(url)
|
||||||
|
title = " ".join(a.get_text(" ", strip=True).split())
|
||||||
|
if key not in best:
|
||||||
|
best[key] = {"title": title, "url": url}
|
||||||
|
order.append(key)
|
||||||
|
elif len(title) > len(best[key]["title"]):
|
||||||
|
best[key]["title"] = title
|
||||||
|
if not best and text:
|
||||||
|
last_line = ""
|
||||||
|
for line in text.splitlines():
|
||||||
|
match = PROJECT_URL_RE.search(line)
|
||||||
|
stripped = line.strip()
|
||||||
|
if match:
|
||||||
|
key = _url_key(match.group(0))
|
||||||
|
if key not in best:
|
||||||
|
best[key] = {"title": last_line or match.group(0),
|
||||||
|
"url": match.group(0)}
|
||||||
|
order.append(key)
|
||||||
|
elif stripped:
|
||||||
|
last_line = stripped
|
||||||
|
return [best[k] for k in order if best[k]["title"]]
|
||||||
|
|
||||||
|
|
||||||
|
def canonical_url(url: str, session=None, timeout=20) -> str:
|
||||||
|
"""Follow redirects, strip query string and fragment."""
|
||||||
|
sess = session or requests.Session()
|
||||||
|
resp = sess.get(url, timeout=timeout, allow_redirects=True,
|
||||||
|
headers={"User-Agent": BROWSER_UA})
|
||||||
|
resp.raise_for_status()
|
||||||
|
parts = urlsplit(resp.url)
|
||||||
|
return urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
|
||||||
69
projekt-matching/tests/test_mailparse.py
Normal file
69
projekt-matching/tests/test_mailparse.py
Normal file
@@ -0,0 +1,69 @@
|
|||||||
|
import email
|
||||||
|
from email.policy import default as default_policy
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
from projektmatch import mailparse
|
||||||
|
|
||||||
|
HTML = """
|
||||||
|
<html><body>
|
||||||
|
<h2><a href="https://www.freelancermap.de/nproj/3020338.html?utm_source=x&t=1">
|
||||||
|
Python Entwickler für eine KI-Anwendung (m/w/d)</a></h2>
|
||||||
|
<p>Beschreibung ...</p>
|
||||||
|
<a href="https://www.freelancermap.de/nproj/3020338.html?utm_source=x&t=1">Zum Projekt</a>
|
||||||
|
<h2><a href="https://www.freelancermap.de/nproj/3020995.html?agent=2">
|
||||||
|
Test Engineer (m/w/d)</a></h2>
|
||||||
|
<a href="https://www.freelancermap.de/other/page.html">Impressum</a>
|
||||||
|
</body></html>
|
||||||
|
"""
|
||||||
|
|
||||||
|
TEXT = """Neue Projekte fuer Sie:
|
||||||
|
|
||||||
|
Python Entwickler für eine KI-Anwendung (m/w/d)
|
||||||
|
https://www.freelancermap.de/nproj/3020338.html?utm_source=x
|
||||||
|
|
||||||
|
Test Engineer (m/w/d)
|
||||||
|
https://www.freelancermap.de/nproj/3020995.html?agent=2
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def test_is_own_mail():
|
||||||
|
assert mailparse.is_own_mail("[Projekt-Match] Freelancermap — 86 % — X")
|
||||||
|
assert mailparse.is_own_mail("[Projekt-Match-Fehler] Freelancermap — 2 Projekt(e)")
|
||||||
|
assert not mailparse.is_own_mail("Example 1 for Langflow process")
|
||||||
|
assert not mailparse.is_own_mail(None)
|
||||||
|
|
||||||
|
|
||||||
|
def test_split_projects_html_dedupes_and_prefers_long_title():
|
||||||
|
items = mailparse.split_projects(HTML, "")
|
||||||
|
assert [i["title"] for i in items] == [
|
||||||
|
"Python Entwickler für eine KI-Anwendung (m/w/d)", "Test Engineer (m/w/d)"]
|
||||||
|
assert "3020338" in items[0]["url"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_split_projects_text_fallback():
|
||||||
|
items = mailparse.split_projects("", TEXT)
|
||||||
|
assert len(items) == 2
|
||||||
|
assert items[0]["title"] == "Python Entwickler für eine KI-Anwendung (m/w/d)"
|
||||||
|
|
||||||
|
|
||||||
|
def test_split_projects_none_for_normal_mail():
|
||||||
|
assert mailparse.split_projects("<p>Hallo</p>", "Hallo") == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_bodies_multipart():
|
||||||
|
msg = email.message.EmailMessage(policy=default_policy)
|
||||||
|
msg["Subject"] = "x"
|
||||||
|
msg.set_content("plain body")
|
||||||
|
msg.add_alternative("<p>html body</p>", subtype="html")
|
||||||
|
html, text = mailparse.bodies(msg)
|
||||||
|
assert "html body" in html and "plain body" in text
|
||||||
|
|
||||||
|
|
||||||
|
def test_canonical_url_strips_query():
|
||||||
|
session = mock.Mock()
|
||||||
|
session.get.return_value = mock.Mock(
|
||||||
|
url="https://www.freelancermap.de/projekt/python-entwickler?ref=1",
|
||||||
|
raise_for_status=lambda: None)
|
||||||
|
out = mailparse.canonical_url("https://www.freelancermap.de/nproj/1.html?x=1",
|
||||||
|
session=session)
|
||||||
|
assert out == "https://www.freelancermap.de/projekt/python-entwickler"
|
||||||
Reference in New Issue
Block a user