187 lines
6.9 KiB
Python
187 lines
6.9 KiB
Python
from datetime import date
|
|
from decimal import Decimal
|
|
from pathlib import Path
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.config import get_settings
|
|
from app.models.tables import Account, Statement, Transaction
|
|
from app.parsers.base import ParsedStatement, ParserError
|
|
from app.parsers.csv_formats import parse_csv
|
|
from app.parsers.registry import parse_pdf
|
|
from app.parsers.validate import balance_difference, dedup_hash
|
|
from app.services.categorize import apply_rules
|
|
|
|
|
|
def _find_or_create_account(session: Session, bank: str, iban: str | None, filename: str) -> Account:
|
|
if not iban:
|
|
iban = f"UNBEKANNT-{filename}"
|
|
if bank == "hvb_csv":
|
|
# HVB-CSV traegt nur die Kontonummer (keine IBAN). Konto-Zuordnung
|
|
# ueber "endet auf Kontonummer" gegen alle bestehenden IBANs -
|
|
# dokumentierte Einschraenkung (siehe Formatreferenz Task 2/3).
|
|
for existing in session.execute(select(Account)).scalars():
|
|
if existing.iban.endswith(iban):
|
|
return existing
|
|
acc = Account(bank=bank, iban=iban, name=iban, type="giro")
|
|
session.add(acc)
|
|
session.flush()
|
|
return acc
|
|
acc = session.execute(select(Account).where(Account.iban == iban)).scalar()
|
|
if acc is not None:
|
|
return acc
|
|
acc = Account(bank=bank, iban=iban, name=iban, type="giro")
|
|
session.add(acc)
|
|
session.flush()
|
|
return acc
|
|
|
|
|
|
def _apply_anchor_autofill(account: Account, anchor: tuple[date, Decimal] | None) -> None:
|
|
"""CSV-Anker-Regel (Ausbaustufe 3 Task 3): der Datei-Kontostand wird nur
|
|
uebernommen, wenn er neuer ist als ein evtl. vorhandener Anker (oder noch
|
|
keiner existiert). Ein neuerer, manuell per PATCH gesetzter Anker bleibt
|
|
unangetastet - so entkommt kein aelterer CSV-Export einem bewusst
|
|
gesetzten aktuellen Stand."""
|
|
if anchor is None:
|
|
return
|
|
anchor_date, anchor_balance = anchor
|
|
if account.anchor_date is None or anchor_date >= account.anchor_date:
|
|
account.anchor_date = anchor_date
|
|
account.anchor_balance = anchor_balance
|
|
|
|
|
|
def _error_statement(session: Session, filename: str, message: str) -> Statement:
|
|
stmt = Statement(filename=filename, bank="unbekannt", account_id=None,
|
|
status="error", error_message=message)
|
|
session.add(stmt)
|
|
session.commit()
|
|
session.refresh(stmt)
|
|
return stmt
|
|
|
|
|
|
def _finish_statement(session: Session, path: Path, parsed: ParsedStatement,
|
|
check_balance: bool,
|
|
anchor: tuple[date, Decimal] | None) -> Statement:
|
|
"""Gemeinsamer Abschluss fuer PDF- und CSV-Importe, sobald ein
|
|
ParsedStatement vorliegt: Konto zuordnen, Statement+Transaktionen als
|
|
Entwurf anlegen, optional den Saldo pruefen, Datei aus dem Posteingang
|
|
verschieben. `check_balance=False` (CSV-Formate ohne Saldodaten, z.B.
|
|
HVB/DKB) laesst den Entwurf OHNE Fehler stehen statt einer
|
|
Saldo-Differenz-Pruefung, die auf None-Feldern crashen wuerde."""
|
|
settings = get_settings()
|
|
filename = path.name
|
|
|
|
account = _find_or_create_account(session, parsed.bank, parsed.iban, filename)
|
|
|
|
stmt = Statement(
|
|
filename=filename,
|
|
bank=parsed.bank,
|
|
account_id=account.id,
|
|
period_start=parsed.period_start,
|
|
period_end=parsed.period_end,
|
|
opening_balance=parsed.opening_balance,
|
|
closing_balance=parsed.closing_balance,
|
|
status="draft",
|
|
)
|
|
session.add(stmt)
|
|
session.flush()
|
|
|
|
if check_balance:
|
|
diff = balance_difference(parsed)
|
|
if diff != 0:
|
|
stmt.status = "error"
|
|
stmt.error_message = f"Saldo-Differenz {diff} EUR"
|
|
session.commit()
|
|
session.refresh(stmt)
|
|
return stmt
|
|
|
|
drafts = []
|
|
for t in parsed.transactions:
|
|
h = dedup_hash(account.id, t.booking_date, t.amount, t.purpose)
|
|
is_duplicate = session.execute(
|
|
select(Transaction).where(
|
|
Transaction.dedup_hash == h,
|
|
Transaction.status == "confirmed",
|
|
)
|
|
).scalar() is not None
|
|
tx = Transaction(
|
|
account_id=account.id,
|
|
statement_id=stmt.id,
|
|
booking_date=t.booking_date,
|
|
value_date=t.value_date,
|
|
amount=t.amount,
|
|
purpose=t.purpose,
|
|
counterparty=t.counterparty,
|
|
status="draft",
|
|
dedup_hash=h,
|
|
is_duplicate=is_duplicate,
|
|
)
|
|
session.add(tx)
|
|
drafts.append(tx)
|
|
|
|
session.flush()
|
|
apply_rules(session, drafts)
|
|
|
|
stmt.status = "draft"
|
|
_apply_anchor_autofill(account, anchor)
|
|
|
|
# Invariant: a committed draft implies the source file left the inbox.
|
|
# Move the file before committing so a failed move can never leave a
|
|
# committed draft with the file still sitting in the inbox.
|
|
uploads_dir = settings.uploads_dir
|
|
uploads_dir.mkdir(parents=True, exist_ok=True)
|
|
try:
|
|
path.replace(uploads_dir / filename)
|
|
except OSError:
|
|
session.rollback()
|
|
raise
|
|
|
|
session.commit()
|
|
session.refresh(stmt)
|
|
|
|
return stmt
|
|
|
|
|
|
def process_pdf(session: Session, path: Path) -> Statement:
|
|
filename = path.name
|
|
try:
|
|
parsed = parse_pdf(path)
|
|
except ParserError as exc:
|
|
return _error_statement(session, filename, str(exc))
|
|
except Exception as exc:
|
|
# Fremdbibliotheken (pdfplumber/pdfminer) werfen bei strukturell
|
|
# kaputten PDFs eigene Exception-Typen statt ParserError (z.B.
|
|
# PdfminerException "No /Root object!"). Ohne diesen Fang wuerde ein
|
|
# kaputtes Upload zu einem unbehandelten 500 fuehren statt zu einem
|
|
# sauberen Fehler-Import wie bei "Bank nicht erkannt". Die Datei
|
|
# bleibt dabei unangetastet im Posteingang.
|
|
return _error_statement(session, filename, f"PDF nicht lesbar: {exc}")
|
|
|
|
return _finish_statement(session, path, parsed, check_balance=True, anchor=None)
|
|
|
|
|
|
def process_csv(session: Session, path: Path) -> Statement:
|
|
filename = path.name
|
|
try:
|
|
parsed_csv = parse_csv(path)
|
|
except ParserError as exc:
|
|
return _error_statement(session, filename, str(exc))
|
|
except Exception as exc:
|
|
# Analog zu process_pdf: eine strukturell kaputte/unlesbare CSV darf
|
|
# nicht als unbehandelter 500 durchschlagen.
|
|
return _error_statement(session, filename, f"CSV nicht lesbar: {exc}")
|
|
|
|
return _finish_statement(session, path, parsed_csv.statement,
|
|
check_balance=parsed_csv.balance_checkable,
|
|
anchor=parsed_csv.anchor)
|
|
|
|
|
|
def process_file(session: Session, path: Path) -> Statement:
|
|
"""Dateiendungs-Weiche fuer die Import-Pipeline (Ausbaustufe 3 Task 3):
|
|
.csv laeuft ueber den CSV-Formatparser, alles andere weiterhin ueber den
|
|
PDF-Parser."""
|
|
if path.suffix.lower() == ".csv":
|
|
return process_csv(session, path)
|
|
return process_pdf(session, path)
|