Coverage for jbank/camt.py : 85%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from datetime import datetime, date
2from decimal import Decimal
3from typing import Tuple, Any, Optional, Dict
5from django.conf import settings
6from django.core.exceptions import ValidationError
7from django.db import transaction
8from django.utils.dateparse import parse_date
9from django.utils.translation import gettext as _
10from jacc.models import Account, EntryType
11from jutil.format import dec2, dec4
12from jutil.parse import parse_datetime
13from jbank.models import (
14 StatementFile,
15 Statement,
16 StatementRecord,
17 DELIVERY_FROM_BANK_SYSTEM,
18 StatementRecordDetail,
19 CurrencyExchange,
20 StatementRecordRemittanceInfo,
21 CurrencyExchangeSource,
22)
23from jbank.parsers import parse_filename_suffix
24from jutil.xml import xml_to_dict
27CAMT053_STATEMENT_SUFFIXES = ("XML", "XT", "CAMT")
29CAMT053_ARRAY_TAGS = ["Bal", "Ntry", "NtryDtls", "TxDtls", "Strd"]
31CAMT053_INT_TAGS = ["NbOfNtries", "NbOfTxs"]
34def camt053_get_iban(data: dict) -> str:
35 return data.get("BkToCstmrStmt", {}).get("Stmt", {}).get("Acct", {}).get("Id", {}).get("IBAN", "")
38def camt053_get_val(data: dict, key: str, default: Any = None, required: bool = True, name: str = "") -> Any:
39 if key not in data:
40 if required:
41 raise ValidationError(_("camt.053 field {} missing").format(name if name else key))
42 return default
43 return data[key]
46def camt053_get_str(data: dict, key: str, default: str = "", required: bool = True, name: str = "") -> str:
47 return str(camt053_get_val(data, key, default, required, name))
50def camt053_get_currency(data: dict, key: str, required: bool = True, name: str = "") -> Tuple[Optional[Decimal], str]:
51 try:
52 v = camt053_get_val(data, key, default=None, required=False, name=name)
53 if v is not None:
54 amount = dec2(v["@"])
55 currency_code = v["@Ccy"]
56 return amount, currency_code
57 except Exception:
58 pass
59 if required:
60 raise ValidationError(_("camt.053 field {} type {} missing or invalid").format(name, "currency"))
61 return None, ""
64def camt053_get_dt(data: Dict[str, Any], key: str, name: str = "") -> datetime:
65 s = camt053_get_val(data, key, None, True, name)
66 val = parse_datetime(s)
67 if val is None:
68 raise ValidationError(
69 _("camt.053 field {} type {} missing or invalid").format(name, "datetime") + ": {}".format(s)
70 )
71 return val
74def camt053_get_int(data: Dict[str, Any], key: str, name: str = "") -> int:
75 s = camt053_get_val(data, key, None, True, name)
76 try:
77 return int(s)
78 except Exception:
79 pass
80 raise ValidationError(_("camt.053 field {} type {} missing or invalid").format(name, "int"))
83def camt053_get_date(
84 data: dict, key: str, default: Optional[date] = None, required: bool = True, name: str = ""
85) -> date:
86 s = camt053_get_val(data, key, default, required, name)
87 try:
88 val = parse_date(s[:10])
89 if val is None:
90 raise ValidationError(_("camt.053 field {} type {} missing or invalid").format(name, "date"))
91 assert isinstance(val, date)
92 return val
93 except Exception:
94 pass
95 raise ValidationError(_("camt.053 field {} type {} missing or invalid").format(name, "date") + ": {}".format(s))
98def camt053_parse_statement_from_file(filename: str) -> dict:
99 if parse_filename_suffix(filename).upper() not in CAMT053_STATEMENT_SUFFIXES:
100 raise ValidationError(
101 _('File {filename} has unrecognized ({suffixes}) suffix for file type "{file_type}"').format(
102 filename=filename, suffixes=", ".join(CAMT053_STATEMENT_SUFFIXES), file_type="camt.053"
103 )
104 )
105 with open(filename, "rb") as fp:
106 data = xml_to_dict(fp.read(), array_tags=CAMT053_ARRAY_TAGS, int_tags=CAMT053_INT_TAGS)
107 return data
110def camt053_get_stmt_bal(d_stmt: dict, bal_type: str) -> Tuple[Decimal, Optional[date]]:
111 for bal in d_stmt.get("Bal", []):
112 if bal.get("Tp", {}).get("CdOrPrtry", {}).get("Cd", "") == bal_type:
113 amt = Decimal(bal.get("Amt", {}).get("@", ""))
114 dt_data = bal.get("Dt", {})
115 dt = None
116 if "Dt" in dt_data:
117 dt = camt053_get_date(dt_data, "Dt", name="Stmt.Bal[{}].Dt.Dt".format(bal_type))
118 return amt, dt
119 raise ValidationError(_("camt.053 field {} type {} missing or invalid").format("Stmt.Bal.Tp.CdOrPrty.Cd", bal_type))
122def camt053_domain_from_record_code(record_domain: str) -> str:
123 if record_domain == "PMNT":
124 return "700"
125 if record_domain == "LDAS":
126 return "761"
127 return ""
130def camt053_get_unified_val(qs, k: str, default: Any) -> Any:
131 v = default
132 for e in qs:
133 v2 = getattr(e, k)
134 if v == default:
135 v = v2
136 elif v and v2 and v2 != v:
137 return default
138 return v
141def camt053_get_unified_str(qs, k: str) -> str:
142 return camt053_get_unified_val(qs, k, "")
145@transaction.atomic # noqa
146def camt053_create_statement(statement_data: dict, name: str, file: StatementFile, **kw) -> Statement: # noqa
147 """
148 Creates camt.053 Statement from statement data parsed by camt053_parse_statement_from_file()
149 :param statement_data: XML data in form of dict
150 :param name: File name of the account statement
151 :param file: Source statement file
152 :return: Statement
153 """
154 account_number = camt053_get_iban(statement_data)
155 if not account_number:
156 raise ValidationError("{name}: ".format(name=name) + _("account.not.found").format(account_number=""))
157 accounts = list(Account.objects.filter(name=account_number))
158 if len(accounts) != 1:
159 raise ValidationError(
160 "{name}: ".format(name=name) + _("account.not.found").format(account_number=account_number)
161 )
162 account = accounts[0]
163 assert isinstance(account, Account)
165 d_stmt = statement_data.get("BkToCstmrStmt", {}).get("Stmt", {})
166 d_acct = d_stmt.get("Acct", {})
167 d_ownr = d_acct.get("Ownr", {})
168 d_ntry = d_stmt.get("Ntry", [])
169 d_frto = d_stmt.get("FrToDt", {})
170 d_txsummary = d_stmt.get("TxsSummry", {})
172 if Statement.objects.filter(name=name, account=account).first():
173 raise ValidationError("Bank account {} statement {} of processed already".format(account_number, name))
174 stm = Statement(name=name, account=account, file=file)
175 stm.account_number = stm.iban = account_number
176 stm.bic = camt053_get_str(d_acct.get("Svcr", {}).get("FinInstnId", {}), "BIC", name="Stmt.Acct.Svcr.FinInstnId.BIC")
177 stm.statement_identifier = camt053_get_str(d_stmt, "Id", name="Stmt.Id")
178 stm.statement_number = camt053_get_str(d_stmt, "LglSeqNb", name="Stmt.LglSeqNb")
179 stm.record_date = camt053_get_dt(d_stmt, "CreDtTm", name="Stmt.CreDtTm")
180 stm.begin_date = camt053_get_dt(d_frto, "FrDtTm", name="Stmt.FrDtTm").date()
181 stm.end_date = camt053_get_dt(d_frto, "ToDtTm", name="Stmt.ToDtTm").date()
182 stm.currency_code = camt053_get_str(d_acct, "Ccy", name="Stmt.Acct.Ccy")
183 if stm.currency_code != account.currency:
184 raise ValidationError(
185 _(
186 "Account currency {account_currency} does not match statement entry currency {statement_currency}".format(
187 statement_currency=stm.currency_code, account_currency=account.currency
188 )
189 )
190 )
191 stm.owner_name = camt053_get_str(d_ownr, "Nm", name="Stm.Acct.Ownr.Nm")
192 stm.begin_balance, stm.begin_balance_date = camt053_get_stmt_bal(d_stmt, "OPBD")
193 if stm.begin_balance_date is None:
194 stm.begin_balance_date = stm.begin_date
195 stm.record_count = camt053_get_int(
196 d_txsummary.get("TtlNtries", {}), "NbOfNtries", name="Stmt.TxsSummry.TtlNtries.NbOfNtries"
197 )
198 stm.bank_specific_info_1 = camt053_get_str(d_stmt, "AddtlStmtInf", required=False)[:1024]
199 for k, v in kw.items():
200 setattr(stm, k, v)
201 stm.full_clean()
202 stm.save()
204 e_deposit = EntryType.objects.filter(code=settings.E_BANK_DEPOSIT).first()
205 if not e_deposit:
206 raise ValidationError(
207 _("entry.type.missing") + " ({}): {}".format("settings.E_BANK_DEPOSIT", settings.E_BANK_DEPOSIT)
208 )
209 assert isinstance(e_deposit, EntryType)
210 e_withdraw = EntryType.objects.filter(code=settings.E_BANK_WITHDRAW).first()
211 if not e_withdraw:
212 raise ValidationError(
213 _("entry.type.missing") + " ({}): {}".format("settings.E_BANK_WITHDRAW", settings.E_BANK_WITHDRAW)
214 )
215 assert isinstance(e_withdraw, EntryType)
216 e_types = {
217 "CRDT": e_deposit,
218 "DBIT": e_withdraw,
219 }
220 record_type_map = {
221 "CRDT": "1",
222 "DBIT": "2",
223 }
225 for ntry in d_ntry:
226 archive_id = ntry.get("AcctSvcrRef", "")
227 amount, cur = camt053_get_currency(ntry, "Amt", name="Stmt.Ntry[{}].Amt".format(archive_id))
228 if cur != account.currency:
229 raise ValidationError(
230 _(
231 "Account currency {account_currency} does not match statement entry currency {statement_currency}".format(
232 statement_currency=cur, account_currency=account.currency
233 )
234 )
235 )
237 cdt_dbt_ind = ntry["CdtDbtInd"]
238 e_type = e_types.get(cdt_dbt_ind, None)
239 if not e_type:
240 raise ValidationError(_("Statement entry type {} not supported").format(cdt_dbt_ind))
242 rec = StatementRecord(statement=stm, account=account, type=e_type)
243 rec.amount = amount
244 rec.archive_identifier = archive_id
245 rec.entry_type = record_type_map[cdt_dbt_ind]
246 rec.record_date = record_date = camt053_get_date(
247 ntry.get("BookgDt", {}), "Dt", name="Stmt.Ntry[{}].BkkgDt.Dt".format(archive_id)
248 )
249 rec.value_date = camt053_get_date(ntry.get("ValDt", {}), "Dt", name="Stmt.Ntry[{}].ValDt.Dt".format(archive_id))
250 rec.delivery_method = DELIVERY_FROM_BANK_SYSTEM
252 d_bktxcd = ntry.get("BkTxCd", {})
253 d_domn = d_bktxcd.get("Domn", {})
254 d_family = d_domn.get("Fmly", {})
255 d_prtry = d_bktxcd.get("Prtry", {})
256 rec.record_domain = record_domain = camt053_get_str(
257 d_domn, "Cd", name="Stmt.Ntry[{}].BkTxCd.Domn.Cd".format(archive_id)
258 )
259 rec.record_code = camt053_domain_from_record_code(record_domain)
260 rec.family_code = camt053_get_str(d_family, "Cd", name="Stmt.Ntry[{}].BkTxCd.Domn.Family.Cd".format(archive_id))
261 rec.sub_family_code = camt053_get_str(
262 d_family, "SubFmlyCd", name="Stmt.Ntry[{}].BkTxCd.Domn.Family.SubFmlyCd".format(archive_id)
263 )
264 rec.record_description = camt053_get_str(d_prtry, "Cd", required=False)
266 rec.full_clean()
267 rec.save()
269 for dtl_batch in ntry.get("NtryDtls", []):
270 batch_identifier = dtl_batch.get("Btch", {}).get("MsgId", "")
271 dtl_ix = 0
272 for dtl in dtl_batch.get("TxDtls", []):
273 d = StatementRecordDetail(record=rec, batch_identifier=batch_identifier)
275 d_amt_dtl = dtl.get("AmtDtls", {})
276 d_txamt = d_amt_dtl.get("TxAmt", {})
277 d_xchg = d_txamt.get("CcyXchg", None)
279 d.amount, d.currency_code = camt053_get_currency(d_txamt, "Amt", required=False)
280 d.instructed_amount, source_currency = camt053_get_currency(
281 d_amt_dtl.get("InstdAmt", {}), "Amt", required=False
282 )
283 if (not d_xchg and source_currency and source_currency != d.currency_code) or (
284 d_xchg and not source_currency
285 ):
286 raise ValidationError(
287 _("Inconsistent Stmt.Ntry[{}].NtryDtls.TxDtls[{}].AmtDtls".format(archive_id, dtl_ix))
288 )
290 if source_currency and source_currency != d.currency_code:
291 source_currency = camt053_get_str(d_xchg, "SrcCcy", default=source_currency, required=False)
292 target_currency = camt053_get_str(d_xchg, "TrgCcy", default=d.currency_code, required=False)
293 unit_currency = camt053_get_str(d_xchg, "UnitCcy", default="", required=False)
294 exchange_rate_str = camt053_get_str(d_xchg, "XchgRate", default="", required=False)
295 exchange_rate = dec4(exchange_rate_str) if exchange_rate_str else None
296 exchange_source = CurrencyExchangeSource.objects.get_or_create(name=account_number)[0]
297 d.exchange = CurrencyExchange.objects.get_or_create(
298 record_date=record_date,
299 source_currency=source_currency,
300 target_currency=target_currency,
301 unit_currency=unit_currency,
302 exchange_rate=exchange_rate,
303 source=exchange_source,
304 )[0]
306 d_refs = dtl.get("Refs", {})
307 d.archive_identifier = d_refs.get("AcctSvcrRef", "")
308 d.end_to_end_identifier = d_refs.get("EndToEndId", "")
310 d_parties = dtl.get("RltdPties", {})
311 d_dbt = d_parties.get("Dbtr", {})
312 d.debtor_name = d_dbt.get("Nm", "")
313 d_udbt = d_parties.get("UltmtDbtr", {})
314 d.ultimate_debtor_name = d_udbt.get("Nm", "")
315 d_cdtr = d_parties.get("Cdtr", {})
316 d.creditor_name = d_cdtr.get("Nm", "")
317 d_cdtr_acct = d_parties.get("CdtrAcct", {})
318 d_cdtr_acct_id = d_cdtr_acct.get("Id", {})
319 d.creditor_account = d_cdtr_acct_id.get("IBAN", "")
320 if d.creditor_account:
321 d.creditor_account_scheme = "IBAN"
322 else:
323 d_cdtr_acct_id_othr = d_cdtr_acct_id.get("Othr") or {}
324 d.creditor_account_scheme = d_cdtr_acct_id_othr.get("SchmeNm", {}).get("Cd", "")
325 d.creditor_account = d_cdtr_acct_id_othr.get("Id") or ""
327 d_rmt = dtl.get("RmtInf", {})
328 d.unstructured_remittance_info = d_rmt.get("Ustrd", "")
330 d_rltd_dts = dtl.get("RltdDts", {})
331 d.paid_date = camt053_get_dt(d_rltd_dts, "AccptncDtTm") if "AccptncDtTm" in d_rltd_dts else None
333 d.full_clean()
334 d.save()
336 st = StatementRecordRemittanceInfo(detail=d)
337 for strd in d_rmt.get("Strd", []):
338 additional_info = strd.get("AddtlRmtInf", "")
339 has_additional_info = bool(additional_info and st.additional_info)
340 amount, currency_code = camt053_get_currency(strd.get("RfrdDocAmt", {}), "RmtdAmt", required=False)
341 has_amount = bool(amount and st.amount)
342 reference = strd.get("CdtrRefInf", {}).get("Ref", "")
343 has_reference = bool(reference and st.reference)
345 # check if new remittance info record is needed
346 if has_additional_info or has_amount or has_reference:
347 st = StatementRecordRemittanceInfo(detail=d)
349 if additional_info:
350 st.additional_info = additional_info
351 if amount:
352 st.amount, st.currency_code = amount, currency_code
353 if reference:
354 st.reference = reference
356 st.full_clean()
357 st.save()
359 dtl_ix += 1
361 # fill record name from details
362 assert rec.type
363 if not rec.name:
364 if rec.type.code == e_withdraw.code:
365 rec.name = camt053_get_unified_str(rec.detail_set.all(), "creditor_name")
366 elif rec.type.code == e_deposit.code:
367 rec.name = camt053_get_unified_str(rec.detail_set.all(), "debtor_name")
368 if not rec.recipient_account_number:
369 rec.recipient_account_number = camt053_get_unified_str(rec.detail_set.all(), "creditor_account")
370 if not rec.remittance_info:
371 rec.remittance_info = camt053_get_unified_str(
372 StatementRecordRemittanceInfo.objects.all().filter(detail__record=rec), "reference"
373 )
374 if not rec.paid_date:
375 paid_date = camt053_get_unified_val(rec.detail_set.all(), "paid_date", default=None)
376 if paid_date:
377 assert isinstance(paid_date, datetime)
378 rec.paid_date = paid_date.date()
380 rec.full_clean()
381 rec.save()
383 return stm