feat: CSV-Parser fuer DKB/VR/HVB-Kontoumsatz-Exporte

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-19 21:34:48 +02:00
parent f6a53b7e56
commit e2d713e960
2 changed files with 500 additions and 0 deletions

View File

@@ -0,0 +1,245 @@
"""Parser für Kontoumsatz-CSV-Exporte (DKB, VR, HVB).
Siehe „CSV-Formatreferenz" in docs/superpowers/plans/2026-07-19-ausbaustufe-3.md
für die verbindliche Spalten-/Formatspezifikation je Bank.
"""
import csv
import io
import re
from dataclasses import dataclass
from datetime import date
from decimal import Decimal
from pathlib import Path
from app.parsers.base import (ParsedStatement, ParsedTransaction, ParserError,
parse_german_amount, parse_german_date)
RE_KONTOSTAND = re.compile(r"Kontostand vom (\d{2}\.\d{2}\.\d{4}):")
RE_ZEITRAUM = re.compile(r"(\d{2}\.\d{2}\.\d{4})\s*-\s*(\d{2}\.\d{2}\.\d{4})")
@dataclass
class ParsedCsv:
statement: ParsedStatement
anchor: tuple[date, Decimal] | None
balance_checkable: bool
def _detect_encoding(raw: bytes) -> str:
if raw.startswith(b"\xef\xbb\xbf"):
return "utf-8-sig"
if raw.startswith(b"\xff\xfe"):
return "utf-16-le"
if raw.startswith(b"\xfe\xff"):
return "utf-16-be"
sample = raw[:64]
if b"\x00" in sample:
even_zeros = sum(1 for i in range(0, len(sample), 2) if sample[i] == 0)
odd_zeros = sum(1 for i in range(1, len(sample), 2) if i < len(sample) and sample[i] == 0)
return "utf-16-be" if even_zeros > odd_zeros else "utf-16-le"
return "utf-8"
def _decode(raw: bytes) -> str:
enc = _detect_encoding(raw)
text = raw.decode(enc, errors="ignore")
if text.startswith(""):
text = text[1:]
return text
def _strip_currency(s: str) -> str:
return s.replace("", "").replace("EUR", "").replace("\xa0", " ").strip()
def _parse_amount(s: str) -> Decimal:
return parse_german_amount(_strip_currency(s))
def _parse_two_digit_year_date(s: str) -> date:
m = re.fullmatch(r"(\d{2})\.(\d{2})\.(\d{2})", s.strip())
if not m:
raise ParserError(f"DKB-CSV: Kein Datum im Format TT.MM.JJ: {s!r}")
day, month, yy = int(m.group(1)), int(m.group(2)), int(m.group(3))
year = 2000 + yy if yy <= 69 else 1900 + yy
try:
return date(year, month, day)
except ValueError:
raise ParserError(f"DKB-CSV: Ungueltiges Datum: {s!r}")
def detect_csv_format(head_bytes: bytes) -> str | None:
try:
text = _decode(head_bytes)
except (LookupError, UnicodeError):
return None
lines = text.splitlines()
if not lines:
return None
first = lines[0]
if first.startswith("Kontonummer;") and "Buchungsdatum" in first and "Valuta" in first \
and "Betrag" in first:
return "hvb_csv"
if first.startswith("Bezeichnung Auftragskonto;") and "IBAN Auftragskonto" in first:
return "vr_csv"
head = "\n".join(lines[:6])
if "Kontostand vom" in head and "Buchungsdatum" in head and "Zahlungsempf" in head:
return "dkb_csv"
return None
def parse_csv(path: Path) -> ParsedCsv:
raw = path.read_bytes()
fmt = detect_csv_format(raw)
if fmt is None:
raise ParserError("CSV-Format nicht erkannt")
text = _decode(raw)
if fmt == "dkb_csv":
return _parse_dkb(text)
if fmt == "vr_csv":
return _parse_vr(text)
return _parse_hvb(text)
def _rows(text: str) -> list[list[str]]:
return list(csv.reader(io.StringIO(text), delimiter=";", quotechar='"'))
def _parse_dkb(text: str) -> ParsedCsv:
rows = _rows(text)
if len(rows) < 2:
raise ParserError("DKB-CSV: Datei zu kurz")
iban = rows[0][1].strip() if len(rows[0]) > 1 else None
period_start = period_end = None
kontostand_date: date | None = None
kontostand_amount: Decimal | None = None
header_idx = None
for i, row in enumerate(rows):
if row and row[0].strip() == "Zeitraum:" and len(row) > 1:
m = RE_ZEITRAUM.search(row[1])
if m:
period_start = parse_german_date(m.group(1))
period_end = parse_german_date(m.group(2))
if row and RE_KONTOSTAND.search(row[0] or "") and len(row) > 1:
m = RE_KONTOSTAND.search(row[0])
kontostand_date = parse_german_date(m.group(1))
kontostand_amount = _parse_amount(row[1])
if row and row[0].strip() == "Buchungsdatum":
header_idx = i
break
if header_idx is None:
raise ParserError("DKB-CSV: Spaltenkopf nicht gefunden")
if kontostand_date is None or kontostand_amount is None:
raise ParserError("DKB-CSV: Kontostand-Zeile nicht gefunden")
header = [h.strip() for h in rows[header_idx]]
idx = {name: i for i, name in enumerate(header)}
required = ["Buchungsdatum", "Wertstellung", "Status", "Zahlungspflichtige*r",
"Zahlungsempfänger*in", "Verwendungszweck", "Umsatztyp", "Betrag (€)"]
for col in required:
if col not in idx:
raise ParserError(f"DKB-CSV: Spalte fehlt: {col}")
txs: list[ParsedTransaction] = []
for row in rows[header_idx + 1:]:
if not row or len(row) < len(header):
continue
if row[idx["Status"]].strip() != "Gebucht":
continue
amount = _parse_amount(row[idx["Betrag (€)"]])
counterparty = (row[idx["Zahlungsempfänger*in"]] if amount < 0
else row[idx["Zahlungspflichtige*r"]]).strip()
purpose = f"{row[idx['Umsatztyp']].strip()} {row[idx['Verwendungszweck']].strip()}".strip()
booking = _parse_two_digit_year_date(row[idx["Buchungsdatum"]])
value = _parse_two_digit_year_date(row[idx["Wertstellung"]])
txs.append(ParsedTransaction(booking_date=booking, value_date=value, amount=amount,
purpose=purpose, counterparty=counterparty))
txs.reverse() # file is sorted descending -> chronologically ascending
stmt = ParsedStatement(bank="dkb_csv", iban=iban, period_start=period_start,
period_end=period_end, opening_balance=None,
closing_balance=None, transactions=txs)
return ParsedCsv(statement=stmt, anchor=(kontostand_date, kontostand_amount),
balance_checkable=False)
def _parse_vr(text: str) -> ParsedCsv:
rows = _rows(text)
if not rows:
raise ParserError("VR-CSV: Datei leer")
header = [h.strip() for h in rows[0]]
idx = {name: i for i, name in enumerate(header)}
required = ["IBAN Auftragskonto", "Buchungstag", "Valutadatum", "Name Zahlungsbeteiligter",
"Buchungstext", "Verwendungszweck", "Betrag", "Saldo nach Buchung"]
for col in required:
if col not in idx:
raise ParserError(f"VR-CSV: Spalte fehlt: {col}")
data_rows = [(i, row) for i, row in enumerate(rows[1:], start=2) if row and len(row) >= len(header)]
if not data_rows:
raise ParserError("VR-CSV: keine Buchungszeilen gefunden")
iban = rows[1][idx["IBAN Auftragskonto"]].strip() if len(rows) > 1 else None
# File is sorted descending (newest first) -> reverse for chronological ascending.
asc = list(reversed(data_rows))
txs: list[ParsedTransaction] = []
saldi: list[Decimal] = []
prev_saldo: Decimal | None = None
for line_no, row in asc:
amount = _parse_amount(row[idx["Betrag"]])
saldo = _parse_amount(row[idx["Saldo nach Buchung"]])
if prev_saldo is not None and prev_saldo + amount != saldo:
raise ParserError(f"VR-CSV: Saldo-Kette inkonsistent bei Zeile {line_no}")
prev_saldo = saldo
booking = parse_german_date(row[idx["Buchungstag"]])
value = parse_german_date(row[idx["Valutadatum"]])
purpose = f"{row[idx['Buchungstext']].strip()} {row[idx['Verwendungszweck']].strip()}".strip()
counterparty = row[idx["Name Zahlungsbeteiligter"]].strip()
txs.append(ParsedTransaction(booking_date=booking, value_date=value, amount=amount,
purpose=purpose, counterparty=counterparty))
saldi.append(saldo)
opening_balance = saldi[0] - txs[0].amount
closing_balance = saldi[-1]
anchor = (txs[-1].booking_date, closing_balance)
stmt = ParsedStatement(bank="vr_csv", iban=iban,
period_start=txs[0].booking_date, period_end=txs[-1].booking_date,
opening_balance=opening_balance, closing_balance=closing_balance,
transactions=txs)
return ParsedCsv(statement=stmt, anchor=anchor, balance_checkable=True)
def _parse_hvb(text: str) -> ParsedCsv:
rows = _rows(text)
if not rows:
raise ParserError("HVB-CSV: Datei leer")
header = [h.strip() for h in rows[0]]
idx = {name: i for i, name in enumerate(header)}
required = ["Kontonummer", "Buchungsdatum", "Valuta", "Verwendungszweck", "Betrag"]
for col in required:
if col not in idx:
raise ParserError(f"HVB-CSV: Spalte fehlt: {col}")
data_rows = [row for row in rows[1:] if row and len(row) >= len(header)]
if not data_rows:
raise ParserError("HVB-CSV: keine Buchungszeilen gefunden")
kontonummer = data_rows[0][idx["Kontonummer"]].strip()
txs: list[ParsedTransaction] = []
for row in data_rows: # already chronologically ascending
booking = parse_german_date(row[idx["Buchungsdatum"]])
value = parse_german_date(row[idx["Valuta"]])
amount = _parse_amount(row[idx["Betrag"]])
purpose = row[idx["Verwendungszweck"]].strip()
txs.append(ParsedTransaction(booking_date=booking, value_date=value, amount=amount,
purpose=purpose, counterparty=""))
stmt = ParsedStatement(bank="hvb_csv", iban=kontonummer,
period_start=txs[0].booking_date, period_end=txs[-1].booking_date,
opening_balance=None, closing_balance=None, transactions=txs)
return ParsedCsv(statement=stmt, anchor=None, balance_checkable=False)

View File

@@ -0,0 +1,255 @@
import json
from datetime import date
from decimal import Decimal
from pathlib import Path
import pytest
from app.parsers.base import ParserError
from app.parsers.csv_formats import detect_csv_format, parse_csv
from app.parsers.validate import balance_difference
FIXTURES = Path(__file__).parent / "fixtures"
FIXTURE_CASES = [
("DKB_2025.csv", "dkb_csv"),
("DKB_2026.csv", "dkb_csv"),
("VR_2025.csv", "vr_csv"),
("VR_2026.csv", "vr_csv"),
("HVB_2025.csv", "hvb_csv"),
("HVB_2026.csv", "hvb_csv"),
]
def _read_bytes(filename: str) -> bytes:
return (FIXTURES / filename).read_bytes()
def _gebucht_count(raw: bytes) -> int:
"""Independently count 'Gebucht' rows without keeping/printing any row content."""
# DKB is UTF-8 (with BOM); Status column is ASCII, safe to count on raw text.
text = raw.decode("utf-8-sig")
return text.count(';"Gebucht";')
@pytest.mark.parametrize("filename,fmt", FIXTURE_CASES)
def test_fixture_rowcount_and_ascending(filename, fmt):
path = FIXTURES / filename
if not path.exists():
pytest.skip(f"Fixture {filename} fehlt")
parsed = parse_csv(path)
stmt = parsed.statement
assert stmt.bank == fmt
assert len(stmt.transactions) > 0
dates = [t.booking_date for t in stmt.transactions]
assert dates == sorted(dates), "Transaktionen muessen chronologisch aufsteigend sein"
for t in stmt.transactions:
assert isinstance(t.amount, Decimal)
@pytest.mark.parametrize("filename,fmt", [c for c in FIXTURE_CASES if c[1] == "vr_csv"])
def test_vr_fixture_balance_chain_and_anchor(filename, fmt):
path = FIXTURES / filename
if not path.exists():
pytest.skip(f"Fixture {filename} fehlt")
parsed = parse_csv(path)
assert parsed.balance_checkable is True
assert balance_difference(parsed.statement) == Decimal("0")
assert parsed.anchor is not None
anchor_date, anchor_balance = parsed.anchor
assert isinstance(anchor_date, date)
assert isinstance(anchor_balance, Decimal)
@pytest.mark.parametrize("filename,fmt", [c for c in FIXTURE_CASES if c[1] == "dkb_csv"])
def test_dkb_fixture_anchor_and_gebucht_filter(filename, fmt):
path = FIXTURES / filename
if not path.exists():
pytest.skip(f"Fixture {filename} fehlt")
raw = _read_bytes(filename)
parsed = parse_csv(path)
assert parsed.anchor is not None
assert parsed.balance_checkable is False
assert parsed.statement.opening_balance is None
assert parsed.statement.closing_balance is None
# Filtering: parsed transaction count must equal independently-counted
# "Gebucht" data rows in the raw file (counts only, no content asserted).
assert len(parsed.statement.transactions) == _gebucht_count(raw)
for t in parsed.statement.transactions:
assert t.counterparty != ""
@pytest.mark.parametrize("filename,fmt", [c for c in FIXTURE_CASES if c[1] == "hvb_csv"])
def test_hvb_fixture_no_balance_check(filename, fmt):
path = FIXTURES / filename
if not path.exists():
pytest.skip(f"Fixture {filename} fehlt")
parsed = parse_csv(path)
assert parsed.balance_checkable is False
assert parsed.anchor is None
assert parsed.statement.opening_balance is None
assert parsed.statement.closing_balance is None
for t in parsed.statement.transactions:
assert t.counterparty == ""
# ---------------------------------------------------------------------------
# Synthetic tests: minimal, fictitious CSVs constructed as bytes in-test.
# ---------------------------------------------------------------------------
def _dkb_row(booking, value, status, payer, payee, purpose, umsatztyp, iban, amount):
return (
f'"{booking}";"{value}";"{status}";"{payer}";"{payee}";'
f'"{purpose}";"{umsatztyp}";"{iban}";"{amount}";"";"";""\r\n'
)
def _build_dkb_csv() -> bytes:
header = (
'"Girokonto";"DE00123456780000000042"\r\n'
'"Zeitraum:";"01.01.2025 - 31.01.2025"\r\n'
'"Kontostand vom 31.01.2025:";"1.000,00\xa0"\r\n'
'""\r\n'
'"Buchungsdatum";"Wertstellung";"Status";"Zahlungspflichtige*r";'
'"Zahlungsempfänger*in";"Verwendungszweck";"Umsatztyp";"IBAN";"Betrag (€)";'
'"Gläubiger-ID";"Mandatsreferenz";"Kundenreferenz"\r\n'
)
rows = (
_dkb_row("15.01.25", "15.01.25", "Gebucht", "Max Muster", "Erika Beispiel",
"Testzweck A", "Ausgang", "DE00TESTIBAN1", "-10,00")
+ _dkb_row("10.01.25", "10.01.25", "Vorgemerkt", "Max Muster", "Erika Beispiel",
"Testzweck B (nicht gebucht)", "Ausgang", "DE00TESTIBAN1", "-99,00")
+ _dkb_row("05.01.25", "05.01.25", "Gebucht", "Erika Beispiel", "Max Muster",
"Testzweck C", "Eingang", "DE00TESTIBAN1", "50,00")
)
return (header + rows).encode("utf-8-sig")
def _build_vr_csv(saldi_consistent: bool = True) -> bytes:
header = (
"Bezeichnung Auftragskonto;IBAN Auftragskonto;BIC Auftragskonto;"
"Bankname Auftragskonto;Buchungstag;Valutadatum;Name Zahlungsbeteiligter;"
"IBAN Zahlungsbeteiligter;BIC (SWIFT-Code) Zahlungsbeteiligter;Buchungstext;"
"Verwendungszweck;Betrag;Waehrung;Saldo nach Buchung;Bemerkung;"
"Gekennzeichneter Umsatz;Glaeubiger ID;Mandatsreferenz\r\n"
)
# File order is descending (newest first), matches real format.
saldo_newest = "120,00" if saldi_consistent else "999,00"
rows = (
"Testkonto;DE00TESTVR000000000001;GENODETEST;Test-Bank;15.01.2025;15.01.2025;"
"Beispiel Gegenpartei;DE00GEGEN0001;GENODETEST;Buchung;Testzweck A;20,00;EUR;"
f"{saldo_newest};;;;\r\n"
"Testkonto;DE00TESTVR000000000001;GENODETEST;Test-Bank;05.01.2025;05.01.2025;"
"Beispiel Gegenpartei 2;DE00GEGEN0002;GENODETEST;Buchung;Testzweck B;100,00;EUR;"
"100,00;;;;\r\n"
)
return (header + rows).encode("utf-8-sig")
def _build_hvb_csv() -> bytes:
header = "Kontonummer;Buchungsdatum;Valuta;Verwendungszweck;Betrag;Waehrung\r\n"
rows = (
"123456789;05.01.2025;05.01.2025;Testzweck mit Umlaut ä ö ü ß;-30,00;EUR\r\n"
"123456789;15.01.2025;15.01.2025;Testzweck zwei;40,00;EUR\r\n"
)
return (header + rows).encode("utf-16")
def test_detect_dkb_csv():
assert detect_csv_format(_build_dkb_csv()) == "dkb_csv"
def test_detect_vr_csv():
assert detect_csv_format(_build_vr_csv()) == "vr_csv"
def test_detect_hvb_csv():
assert detect_csv_format(_build_hvb_csv()) == "hvb_csv"
def test_detect_garbage_returns_none():
assert detect_csv_format(b"this,is,not;a\nrecognised\tformat garbage 1234") is None
def test_dkb_synthetic_skips_non_gebucht_rows(tmp_path):
p = tmp_path / "dkb_synth.csv"
p.write_bytes(_build_dkb_csv())
parsed = parse_csv(p)
assert len(parsed.statement.transactions) == 2
for t in parsed.statement.transactions:
assert t.purpose != "" and "nicht gebucht" not in t.purpose
def test_dkb_synthetic_two_digit_year_parsed(tmp_path):
p = tmp_path / "dkb_synth.csv"
p.write_bytes(_build_dkb_csv())
parsed = parse_csv(p)
dates = [t.booking_date for t in parsed.statement.transactions]
assert date(2025, 1, 5) in dates
assert date(2025, 1, 15) in dates
def test_dkb_synthetic_anchor(tmp_path):
p = tmp_path / "dkb_synth.csv"
p.write_bytes(_build_dkb_csv())
parsed = parse_csv(p)
assert parsed.anchor == (date(2025, 1, 31), Decimal("1000.00"))
assert parsed.balance_checkable is False
def test_vr_synthetic_balance_chain_ok(tmp_path):
p = tmp_path / "vr_synth.csv"
p.write_bytes(_build_vr_csv(saldi_consistent=True))
parsed = parse_csv(p)
assert parsed.balance_checkable is True
assert balance_difference(parsed.statement) == Decimal("0")
assert parsed.anchor == (date(2025, 1, 15), Decimal("120.00"))
def test_vr_synthetic_balance_chain_break_raises(tmp_path):
p = tmp_path / "vr_synth_broken.csv"
p.write_bytes(_build_vr_csv(saldi_consistent=False))
with pytest.raises(ParserError):
parse_csv(p)
def test_hvb_synthetic_utf16_decode(tmp_path):
p = tmp_path / "hvb_synth.csv"
p.write_bytes(_build_hvb_csv())
parsed = parse_csv(p)
stmt = parsed.statement
assert len(stmt.transactions) == 2
assert stmt.transactions[0].booking_date == date(2025, 1, 5)
assert stmt.transactions[1].booking_date == date(2025, 1, 15)
assert "ä" in stmt.transactions[0].purpose
assert all(t.counterparty == "" for t in stmt.transactions)
assert parsed.balance_checkable is False
assert parsed.anchor is None
def test_parse_csv_unknown_format_raises(tmp_path):
p = tmp_path / "garbage.csv"
p.write_bytes(b"not,a;known\nformat\n")
with pytest.raises(ParserError):
parse_csv(p)
CSV_FILES = ["DKB_2025", "DKB_2026", "VR_2025", "VR_2026", "HVB_2025", "HVB_2026"]
@pytest.mark.parametrize("name", CSV_FILES)
def test_csv_expected_values(name):
path = FIXTURES / f"{name}.csv"
exp_path = FIXTURES / f"expected_csv_{name}.json"
if not path.exists() or not exp_path.exists():
pytest.skip("Fixture oder expected-Datei fehlt")
result = parse_csv(path)
exp = json.loads(exp_path.read_text())
txs = result.statement.transactions
assert len(txs) == exp["transaction_count"]
for c in exp["spot_checks"]:
t = txs[c["index"]]
assert str(t.booking_date) == c["booking_date"]
assert str(t.amount) == c["amount"]
assert c["counterparty_contains"] in t.counterparty
assert c["purpose_contains"] in t.purpose