fix: DKB-Empfaenger/Zweck-Split per CSV-Ground-Truth, Vergleichswerkzeug
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -49,6 +49,80 @@ def _is_noise(line: str) -> bool:
|
||||
return bool(RE_NOISE.search(line))
|
||||
|
||||
|
||||
# --- Empfänger/Zweck-Split-Heuristik (D-T5-1/D-T5-2, siehe
|
||||
# .superpowers/sdd/parser-vs-csv-audit.md) --------------------------------
|
||||
# extras[0] enthält "<Empfänger><Leerzeichen><Beginn Verwendungszweck>" ohne
|
||||
# Trenner. Die folgenden Regeln bestimmen, wo der Empfänger endet.
|
||||
_LEGAL_FORM_TOKENS = {
|
||||
"ag", "gmbh", "ev", "eg", "kg", "bv", "sca", "sarl", "se",
|
||||
"ltd", "inc", "ek", "ohg", "gbr", "stiftung", "bank",
|
||||
}
|
||||
_CONNECTOR_TOKENS = {"et", "cie"}
|
||||
_REF_KEYWORDS = {"rechnung", "vertrag", "kundennummer", "kassenzeichen", "rg", "nr"}
|
||||
_REF_KEYWORD_PREFIXES = ("kd.-nr", "kdnr", "beitragsnr")
|
||||
_RE_TOKEN_DIGIT = re.compile(r"\d")
|
||||
_MAX_LEGAL_FORM_SCAN = 7
|
||||
|
||||
|
||||
def _normalize_token(tok: str) -> str:
|
||||
return tok.strip(",;").lower().replace(".", "")
|
||||
|
||||
|
||||
def _is_legal_form(tok: str) -> bool:
|
||||
return _normalize_token(tok) in _LEGAL_FORM_TOKENS
|
||||
|
||||
|
||||
def _is_connector(tok: str) -> bool:
|
||||
return _normalize_token(tok) in _CONNECTOR_TOKENS
|
||||
|
||||
|
||||
def _is_reference_keyword(tok: str) -> bool:
|
||||
norm = _normalize_token(tok)
|
||||
return norm in _REF_KEYWORDS or norm.startswith(_REF_KEYWORD_PREFIXES)
|
||||
|
||||
|
||||
def _split_counterparty(line: str) -> tuple[str, str]:
|
||||
"""Trennt Empfänger von Zweck-Beginn in der ersten Fortsetzungszeile
|
||||
(extras[0]) einer DKB-Buchung. Gibt (counterparty, purpose_prefix)
|
||||
zurück; purpose_prefix ist der abgeschnittene Rest (kann leer sein).
|
||||
|
||||
Reihenfolge (D-T5-1):
|
||||
1. Rechtsform-Grenze (erste ~7 Tokens; "et Cie"-Ketten bis zum letzten
|
||||
Rechtsform-Token in Folge).
|
||||
2. Referenz-Grenze (erstes Token mit Ziffer oder Referenz-Schlüsselwort,
|
||||
frühestens ab Token 1 -> counterparty nie leer).
|
||||
3. Kein Schnitt (reine Wort-Folge, semantisch nicht trennbar).
|
||||
|
||||
D-T5-2: "siehe Anlage"-Zeilen -> counterparty leer, ganze Zeile in den
|
||||
Zweck.
|
||||
"""
|
||||
line = line.strip()
|
||||
if not line:
|
||||
return "", ""
|
||||
if line.lower().startswith("siehe anlage"):
|
||||
return "", line
|
||||
tokens = line.split()
|
||||
n = len(tokens)
|
||||
|
||||
for i in range(min(n, _MAX_LEGAL_FORM_SCAN)):
|
||||
if not _is_legal_form(tokens[i]):
|
||||
continue
|
||||
last = i
|
||||
j = i + 1
|
||||
while j < n and (_is_connector(tokens[j]) or _is_legal_form(tokens[j])):
|
||||
if _is_legal_form(tokens[j]):
|
||||
last = j
|
||||
j += 1
|
||||
cut = last + 1
|
||||
return " ".join(tokens[:cut]), " ".join(tokens[cut:])
|
||||
|
||||
for idx in range(1, n):
|
||||
if _RE_TOKEN_DIGIT.search(tokens[idx]) or _is_reference_keyword(tokens[idx]):
|
||||
return " ".join(tokens[:idx]), " ".join(tokens[idx:])
|
||||
|
||||
return line, ""
|
||||
|
||||
|
||||
def parse(path: Path) -> ParsedStatement:
|
||||
with pdfplumber.open(path) as pdf:
|
||||
text = "\n".join((page.extract_text() or "") for page in pdf.pages)
|
||||
@@ -95,8 +169,16 @@ def parse(path: Path) -> ParsedStatement:
|
||||
|
||||
|
||||
def _finish(p: dict) -> ParsedTransaction:
|
||||
counterparty = p["extra"][0] if p["extra"] else ""
|
||||
purpose = " ".join([p["head"], *p["extra"][1:]]).strip()
|
||||
if p["extra"]:
|
||||
counterparty, remainder = _split_counterparty(p["extra"][0])
|
||||
purpose_parts = [p["head"]]
|
||||
if remainder:
|
||||
purpose_parts.append(remainder)
|
||||
purpose_parts.extend(p["extra"][1:])
|
||||
else:
|
||||
counterparty = ""
|
||||
purpose_parts = [p["head"]]
|
||||
purpose = " ".join(purpose_parts).strip()
|
||||
return ParsedTransaction(booking_date=p["booking"], value_date=p["value"],
|
||||
amount=p["amount"], purpose=purpose,
|
||||
counterparty=counterparty)
|
||||
|
||||
166
finance/scripts/parser_vs_csv.py
Normal file
166
finance/scripts/parser_vs_csv.py
Normal file
@@ -0,0 +1,166 @@
|
||||
"""Vergleichswerkzeug: PDF-Parser-Ergebnis gegen CSV-Ground-Truth (Task 5, Ausbaustufe 3).
|
||||
|
||||
NUR LOKAL VERWENDEN - die Ausgabe enthaelt echte Kontodaten (Betraege,
|
||||
Gegenparteien, Verwendungszwecke) und darf nicht in Commits, Reports oder
|
||||
Tickets uebernommen werden.
|
||||
|
||||
Matcht die aus einem Kontoauszugs-PDF geparsten Transaktionen gegen die
|
||||
Zeilen einer oder mehrerer Kontoumsatz-CSV-Dateien (auf den vom PDF
|
||||
abgedeckten Zeitraum eingeschraenkt) anhand von (booking_date, amount) und
|
||||
vergleicht je Match Gegenpartei (counterparty) und Verwendungszweck
|
||||
(purpose) mittels difflib.SequenceMatcher.
|
||||
|
||||
Aufruf:
|
||||
PYTHONPATH=. .venv/bin/python scripts/parser_vs_csv.py <pdf> <csv> [<csv>...]
|
||||
"""
|
||||
import statistics
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
from difflib import SequenceMatcher
|
||||
from pathlib import Path
|
||||
|
||||
from app.parsers.base import ParsedTransaction
|
||||
from app.parsers.csv_formats import parse_csv
|
||||
from app.parsers.registry import parse_pdf
|
||||
|
||||
TRUNC = 60
|
||||
|
||||
|
||||
@dataclass
|
||||
class CsvRow:
|
||||
booking_date: date
|
||||
amount: Decimal
|
||||
counterparty: str
|
||||
purpose: str
|
||||
|
||||
|
||||
def _norm(s: str) -> str:
|
||||
return " ".join((s or "").casefold().split())
|
||||
|
||||
|
||||
def _sim(a: str, b: str) -> float:
|
||||
return SequenceMatcher(None, _norm(a), _norm(b)).ratio()
|
||||
|
||||
|
||||
def _trunc(s: str) -> str:
|
||||
s = s or ""
|
||||
return s if len(s) <= TRUNC else s[: TRUNC - 1] + "…"
|
||||
|
||||
|
||||
def load_csv_rows(paths: list[Path], period_start: date | None,
|
||||
period_end: date | None) -> list[CsvRow]:
|
||||
rows: list[CsvRow] = []
|
||||
for p in paths:
|
||||
parsed = parse_csv(p)
|
||||
for t in parsed.statement.transactions:
|
||||
if period_start is not None and t.booking_date < period_start:
|
||||
continue
|
||||
if period_end is not None and t.booking_date > period_end:
|
||||
continue
|
||||
rows.append(CsvRow(booking_date=t.booking_date, amount=t.amount,
|
||||
counterparty=t.counterparty, purpose=t.purpose))
|
||||
return rows
|
||||
|
||||
|
||||
def match(pdf_txs: list[ParsedTransaction], csv_rows: list[CsvRow]):
|
||||
"""Greedy 1:1 match auf (booking_date, amount); Reihenfolge = PDF-Reihenfolge.
|
||||
|
||||
Returns (matches, unmatched_pdf, unmatched_csv, ambiguous_keys).
|
||||
matches: list[tuple[ParsedTransaction, CsvRow]]
|
||||
"""
|
||||
buckets: dict[tuple[date, Decimal], list[CsvRow]] = defaultdict(list)
|
||||
for r in csv_rows:
|
||||
buckets[(r.booking_date, r.amount)].append(r)
|
||||
|
||||
ambiguous_keys = {k for k, v in buckets.items() if len(v) > 1}
|
||||
|
||||
matches: list[tuple[ParsedTransaction, CsvRow]] = []
|
||||
unmatched_pdf: list[ParsedTransaction] = []
|
||||
for t in pdf_txs:
|
||||
key = (t.booking_date, t.amount)
|
||||
bucket = buckets.get(key)
|
||||
if bucket:
|
||||
row = bucket.pop(0)
|
||||
matches.append((t, row))
|
||||
else:
|
||||
unmatched_pdf.append(t)
|
||||
|
||||
unmatched_csv = [r for bucket in buckets.values() for r in bucket]
|
||||
return matches, unmatched_pdf, unmatched_csv, ambiguous_keys
|
||||
|
||||
|
||||
def purpose_sim(pdf_purpose: str, csv_counterparty: str, csv_purpose: str) -> float:
|
||||
direct = _sim(pdf_purpose, csv_purpose)
|
||||
combined = _sim(pdf_purpose, f"{csv_counterparty} {csv_purpose}")
|
||||
return max(direct, combined)
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
if len(argv) < 3:
|
||||
print(f"Usage: {argv[0]} <pdf> <csv> [<csv>...]", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
pdf_path = Path(argv[1])
|
||||
csv_paths = [Path(p) for p in argv[2:]]
|
||||
|
||||
stmt = parse_pdf(pdf_path)
|
||||
csv_rows = load_csv_rows(csv_paths, stmt.period_start, stmt.period_end)
|
||||
|
||||
print(f"PDF: {pdf_path.name} bank={stmt.bank} "
|
||||
f"period={stmt.period_start}..{stmt.period_end} n_pdf={len(stmt.transactions)}")
|
||||
print(f"CSV: {', '.join(p.name for p in csv_paths)} "
|
||||
f"n_csv(in Periode)={len(csv_rows)}")
|
||||
print("-" * 100)
|
||||
|
||||
matches, unmatched_pdf, unmatched_csv, ambiguous_keys = match(
|
||||
stmt.transactions, csv_rows)
|
||||
|
||||
cp_sims: list[float] = []
|
||||
p_sims: list[float] = []
|
||||
|
||||
for i, (t, r) in enumerate(matches, start=1):
|
||||
cp_s = _sim(t.counterparty, r.counterparty)
|
||||
p_s = purpose_sim(t.purpose, r.counterparty, r.purpose)
|
||||
cp_sims.append(cp_s)
|
||||
p_sims.append(p_s)
|
||||
print(f"#{i} | {t.booking_date} | {t.amount:>10} "
|
||||
f"| PDF-cp={_trunc(t.counterparty)} | CSV-cp={_trunc(r.counterparty)} "
|
||||
f"| cp-sim={cp_s:.2f} "
|
||||
f"| PDF-p={_trunc(t.purpose)} | CSV-p={_trunc(r.purpose)} "
|
||||
f"| p-sim={p_s:.2f}")
|
||||
|
||||
print("-" * 100)
|
||||
if unmatched_pdf:
|
||||
print(f"Unmatched PDF-Transaktionen ({len(unmatched_pdf)}):")
|
||||
for t in unmatched_pdf:
|
||||
print(f" {t.booking_date} | {t.amount:>10}")
|
||||
if unmatched_csv:
|
||||
print(f"Unmatched CSV-Zeilen ({len(unmatched_csv)}):")
|
||||
for r in unmatched_csv:
|
||||
print(f" {r.booking_date} | {r.amount:>10}")
|
||||
|
||||
print("=" * 100)
|
||||
print("Summary")
|
||||
print(f" n_pdf = {len(stmt.transactions)}")
|
||||
print(f" n_csv(in Periode) = {len(csv_rows)}")
|
||||
print(f" n_matched = {len(matches)}")
|
||||
print(f" ambiguities = {len(ambiguous_keys)} "
|
||||
f"(Buchungs-Keys mit >1 CSV-Zeile gleichen Datums/Betrags)")
|
||||
if cp_sims:
|
||||
print(f" cp-sim mean/median = {statistics.mean(cp_sims):.3f} / "
|
||||
f"{statistics.median(cp_sims):.3f}")
|
||||
print(f" p-sim mean/median = {statistics.mean(p_sims):.3f} / "
|
||||
f"{statistics.median(p_sims):.3f}")
|
||||
low_cp = sum(1 for s in cp_sims if s < 0.5)
|
||||
print(f" cp-sim < 0.5 = {low_cp} von {len(matches)} Matches")
|
||||
else:
|
||||
print(" (keine Matches)")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv))
|
||||
@@ -5,6 +5,7 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.parsers.dkb import _split_counterparty
|
||||
from app.parsers.registry import parse_pdf
|
||||
from app.parsers.validate import balance_difference
|
||||
|
||||
@@ -56,3 +57,64 @@ def test_expected_values(filename, bank):
|
||||
assert str(t.amount) == c["amount"]
|
||||
assert c["counterparty_contains"].lower() in t.counterparty.lower()
|
||||
assert c["purpose_contains"].lower() in t.purpose.lower()
|
||||
|
||||
|
||||
# --- DKB Empfänger/Zweck-Split-Heuristik (D-T5-1/D-T5-2) -------------------
|
||||
# Ausschließlich synthetische Zeilen (fiktive Namen/Nummern), keine echten
|
||||
# Kontodaten.
|
||||
|
||||
def test_split_counterparty_legal_form_single():
|
||||
cp, rest = _split_counterparty(
|
||||
"Musterfirma Beispiel GmbH Rechnung Nr. F000123 vom 01.01.20")
|
||||
assert cp == "Musterfirma Beispiel GmbH"
|
||||
assert rest == "Rechnung Nr. F000123 vom 01.01.20"
|
||||
|
||||
|
||||
def test_split_counterparty_legal_form_et_cie_chain():
|
||||
cp, rest = _split_counterparty(
|
||||
"Fiktiv Global S.a.r.l. et Cie S.C.A Onlinedienst Referenz 999")
|
||||
assert cp == "Fiktiv Global S.a.r.l. et Cie S.C.A"
|
||||
assert rest == "Onlinedienst Referenz 999"
|
||||
|
||||
|
||||
def test_split_counterparty_reference_boundary_digit_token():
|
||||
cp, rest = _split_counterparty(
|
||||
"Beispiel Handelsgesellschaft Auftrag 84210 Sonderzahlung")
|
||||
assert cp == "Beispiel Handelsgesellschaft Auftrag"
|
||||
assert rest == "84210 Sonderzahlung"
|
||||
|
||||
|
||||
def test_split_counterparty_reference_boundary_keyword_token():
|
||||
cp, rest = _split_counterparty(
|
||||
"Servicepartner Nord Vertrag ABC123 Laufzeit")
|
||||
assert cp == "Servicepartner Nord"
|
||||
assert rest == "Vertrag ABC123 Laufzeit"
|
||||
|
||||
|
||||
def test_split_counterparty_no_split_pure_words():
|
||||
cp, rest = _split_counterparty("MUSTER PERSON Nebenkosten")
|
||||
assert cp == "MUSTER PERSON Nebenkosten"
|
||||
assert rest == ""
|
||||
|
||||
|
||||
def test_split_counterparty_siehe_anlage():
|
||||
cp, rest = _split_counterparty("siehe Anlage Nr. 7")
|
||||
assert cp == ""
|
||||
assert rest == "siehe Anlage Nr. 7"
|
||||
|
||||
|
||||
def test_split_counterparty_never_empty_guard():
|
||||
# Erstes Token enthält eine Ziffer, darf aber nie selbst die
|
||||
# Referenz-Grenze bilden (Regel greift frühestens ab Token 1) - da kein
|
||||
# weiteres Token eine Ziffer/Schlüsselwort liefert, bleibt die ganze
|
||||
# Zeile als counterparty erhalten (nie leer).
|
||||
cp, rest = _split_counterparty("42 Testfirma Ausgleich")
|
||||
assert cp == "42 Testfirma Ausgleich"
|
||||
assert rest == ""
|
||||
assert cp != ""
|
||||
|
||||
# Einzelnes Token (auch mit Ziffer): kein Schnitt möglich, bleibt
|
||||
# counterparty (nie leer, nie None).
|
||||
cp2, rest2 = _split_counterparty("12345")
|
||||
assert cp2 == "12345"
|
||||
assert rest2 == ""
|
||||
|
||||
Reference in New Issue
Block a user