Coverage for jbank/tito.py: 83%
149 statements
« prev ^ index » next coverage.py v7.2.2, created at 2023-03-27 13:36 +0700
« prev ^ index » next coverage.py v7.2.2, created at 2023-03-27 13:36 +0700
1from copy import copy
2from os.path import basename
3from typing import Dict, Any, List
4from django.core.exceptions import ValidationError
5from django.utils.translation import gettext as _
6from pytz import timezone
7from jbank.parsers import parse_filename_suffix, parse_records, convert_date_fields, convert_decimal_fields
9TO_STATEMENT_SUFFIXES = ("TO", "TXT", "TITO")
11TO_FILE_HEADER_TYPES = ("T00",)
13TO_FILE_HEADER_DATES = (
14 "begin_date",
15 "end_date",
16 ("record_date", "record_time"),
17 "begin_balance_date",
18)
20TO_FILE_HEADER_DECIMALS = (
21 ("begin_balance", "begin_balance_sign"),
22 "account_limit",
23)
25TO_FILE_HEADER = (
26 ("statement_type", "X", "P"),
27 ("record_type", "XX", "P"),
28 ("record_length", "9(3)", "P"),
29 ("version", "X(3)", "P"),
30 ("account_number", "X(14)", "P"),
31 ("statement_number", "9(3)", "P"),
32 ("begin_date", "9(6)", "P"),
33 ("end_date", "9(6)", "P"),
34 ("record_date", "9(6)", "P"),
35 ("record_time", "9(4)", "P"),
36 ("customer_identifier", "X(17)", "P"),
37 ("begin_balance_date", "9(6)", "P"),
38 ("begin_balance_sign", "X", "P"),
39 ("begin_balance", "9(18)", "P"),
40 ("record_count", "9(6)", "P"),
41 ("currency_code", "X(3)", "P"),
42 ("account_name", "X(30)", "V"),
43 ("account_limit", "9(18)", "P"),
44 ("owner_name", "X(35)", "P"),
45 ("contact_info_1", "X(40)", "P"),
46 ("contact_info_2", "X(40)", "V"),
47 ("bank_specific_info_1", "X(30)", "V"),
48 ("iban_and_bic", "X(30)", "V"),
49)
51TO_FILE_RECORD_TYPES = ("T10", "T80")
53TO_FILE_RECORD_DATES = (
54 "record_date",
55 "value_date",
56 "paid_date",
57)
59TO_FILE_RECORD_DECIMALS = (("amount", "amount_sign"),)
61TO_FILE_RECORD = (
62 ("statement_type", "X", "P"),
63 ("record_type", "XX", "P"),
64 ("record_length", "9(3)", "P"),
65 ("record_number", "9(6)", "P"),
66 ("archive_identifier", "X(18)", "V"),
67 ("record_date", "9(6)", "P"),
68 ("value_date", "9(6)", "V"),
69 ("paid_date", "9(6)", "V"),
70 ("entry_type", "X", "P"), # 1 = pano, 2 = otto, 3 = panon korjaus, 4 = oton korjaus, 9 = hylätty tapahtuma
71 ("record_code", "X(3)", "P"),
72 ("record_description", "X(35)", "P"),
73 ("amount_sign", "X", "P"),
74 ("amount", "9(18)", "P"),
75 ("receipt_code", "X", "P"),
76 ("delivery_method", "X", "P"),
77 ("name", "X(35)", "V"),
78 ("name_source", "X", "V"),
79 ("recipient_account_number", "X(14)", "V"),
80 ("recipient_account_number_changed", "X", "V"),
81 ("remittance_info", "X(20)", "V"),
82 ("form_number", "X(8)", "V"),
83 ("level_identifier", "X", "P"),
84)
86TO_FILE_RECORD_EXTRA_INFO_TYPES = ("T11", "T81")
88TO_FILE_RECORD_EXTRA_INFO_HEADER = (
89 ("statement_type", "X", "P"),
90 ("record_type", "XX", "P"),
91 ("record_length", "9(3)", "P"),
92 ("extra_info_type", "9(2)", "P"),
93)
95TO_FILE_RECORD_EXTRA_INFO_COUNTS = (("entry_count", "9(8)", "P"),)
97TO_FILE_RECORD_EXTRA_INFO_INVOICE = (
98 ("customer_number", "X(10)", "P"),
99 ("pad01", "X", "P"),
100 ("invoice_number", "X(15)", "P"),
101 ("pad02", "X", "P"),
102 ("invoice_date", "X(6)", "P"),
103)
105TO_FILE_RECORD_EXTRA_INFO_CARD = (
106 ("card_number", "X(19)", "P"),
107 ("pad01", "X", "P"),
108 ("merchant_reference", "X(14)", "P"),
109)
111TO_FILE_RECORD_EXTRA_INFO_CORRECTION = (("original_archive_identifier", "X(18)", "P"),)
113TO_FILE_RECORD_EXTRA_INFO_CURRENCY_DECIMALS = (("amount", "amount_sign"),)
115TO_FILE_RECORD_EXTRA_INFO_CURRENCY = (
116 ("amount_sign", "X", "P"),
117 ("amount", "9(18)", "P"),
118 ("pad01", "X", "P"),
119 ("currency_code", "X(3)", "P"),
120 ("pad02", "X", "P"),
121 ("exchange_rate", "9(11)", "P"),
122 ("rate_reference", "X(6)", "V"),
123)
125TO_FILE_RECORD_EXTRA_INFO_REASON = (
126 ("reason_code", "9(3)", "P"),
127 ("pad01", "X", "P"),
128 ("reason_description", "X(31)", "P"),
129)
131TO_FILE_RECORD_EXTRA_INFO_NAME_DETAIL = (("name_detail", "X(35)", "P"),)
133TO_FILE_RECORD_EXTRA_INFO_SEPA = (
134 ("reference", "X(35)", "V"),
135 ("iban_account_number", "X(35)", "V"),
136 ("bic_code", "X(35)", "V"),
137 ("recipient_name_detail", "X(70)", "V"),
138 ("payer_name_detail", "X(70)", "V"),
139 ("identifier", "X(35)", "V"),
140 ("archive_identifier", "X(35)", "V"),
141)
143TO_BALANCE_RECORD_DATES = ("record_date",)
145TO_BALANCE_RECORD_DECIMALS = (
146 ("end_balance", "end_balance_sign"),
147 ("available_balance", "available_balance_sign"),
148)
150TO_BALANCE_RECORD = (
151 ("statement_type", "X", "P"),
152 ("record_type", "XX", "P"),
153 ("record_length", "9(3)", "P"),
154 ("record_date", "9(6)", "P"),
155 ("end_balance_sign", "X", "P"),
156 ("end_balance", "9(18)", "P"),
157 ("available_balance_sign", "X", "P"),
158 ("available_balance", "9(18)", "P"),
159)
161TO_CUMULATIVE_RECORD_DATES = ("period_date",)
163TO_CUMULATIVE_RECORD_DECIMALS = (
164 ("deposits_amount", "deposits_sign"),
165 ("withdrawals_amount", "withdrawals_sign"),
166)
168TO_CUMULATIVE_RECORD = (
169 ("statement_type", "X", "P"),
170 ("record_type", "XX", "P"),
171 ("record_length", "9(3)", "P"),
172 ("period_identifier", "X", "P"), # 1=day, 2=term, 3=month, 4=year
173 ("period_date", "9(6)", "P"),
174 ("deposits_count", "9(8)", "P"),
175 ("deposits_sign", "X", "P"),
176 ("deposits_amount", "9(18)", "P"),
177 ("withdrawals_count", "9(8)", "P"),
178 ("withdrawals_sign", "X", "P"),
179 ("withdrawals_amount", "9(18)", "P"),
180)
182TO_SPECIAL_RECORD = (
183 ("statement_type", "X", "P"),
184 ("record_type", "XX", "P"),
185 ("record_length", "9(3)", "P"),
186 ("bank_group_identifier", "X(3)", "P"),
187)
190def parse_tiliote_statements_from_file(filename: str) -> list:
191 if parse_filename_suffix(filename).upper() not in TO_STATEMENT_SUFFIXES:
192 raise ValidationError(
193 _('File {filename} has unrecognized ({suffixes}) suffix for file type "{file_type}"').format(
194 filename=filename, suffixes=", ".join(TO_STATEMENT_SUFFIXES), file_type="tiliote"
195 )
196 )
197 with open(filename, "rt", encoding="ISO-8859-1") as fp:
198 return parse_tiliote_statements(fp.read(), filename=basename(filename)) # type: ignore
201def parse_tiliote_statements(content: str, filename: str) -> List[dict]: # pylint: disable=too-many-locals
202 lines = content.split("\n")
203 nlines = len(lines)
205 line_number = 1
206 tz = timezone("Europe/Helsinki")
208 header = None
209 records = []
210 balance = None
211 cumulative = None
212 cumulative_adjustment = None
213 special_records = []
214 statements = []
216 while line_number <= nlines:
217 line = lines[line_number - 1]
218 if line.strip() == "":
219 line_number += 1
220 continue
221 # print('parsing line {}: {}'.format(line_number, line))
222 record_type = line[:3]
224 if record_type in TO_FILE_HEADER_TYPES:
225 if header:
226 statements.append(combine_statement(header, records, balance, cumulative, cumulative_adjustment, special_records))
227 header, records, balance, cumulative, cumulative_adjustment, special_records = (
228 None,
229 [],
230 None,
231 None,
232 None,
233 [],
234 )
236 header = parse_records(lines[line_number - 1], TO_FILE_HEADER, line_number=line_number)
237 convert_date_fields(header, TO_FILE_HEADER_DATES, tz)
238 convert_decimal_fields(header, TO_FILE_HEADER_DECIMALS)
239 iban_and_bic = str(header.get("iban_and_bic", "")).split(" ")
240 if len(iban_and_bic) == 2:
241 header["iban"], header["bic"] = iban_and_bic
242 line_number += 1
244 elif record_type in TO_FILE_RECORD_TYPES:
245 record = parse_records(line, TO_FILE_RECORD, line_number=line_number)
246 convert_date_fields(record, TO_FILE_RECORD_DATES, tz)
247 convert_decimal_fields(record, TO_FILE_RECORD_DECIMALS)
249 line_number += 1
250 # check for record extra info
251 if line_number <= nlines:
252 line = lines[line_number - 1]
253 while line[:3] in TO_FILE_RECORD_EXTRA_INFO_TYPES:
254 parse_record_extra_info(record, line, line_number=line_number)
255 line_number += 1
256 line = lines[line_number - 1]
258 records.append(record)
259 elif record_type == "T40":
260 balance = parse_records(line, TO_BALANCE_RECORD, line_number=line_number)
261 convert_date_fields(balance, TO_BALANCE_RECORD_DATES, tz)
262 convert_decimal_fields(balance, TO_BALANCE_RECORD_DECIMALS)
263 line_number += 1
264 elif record_type == "T50":
265 cumulative = parse_records(line, TO_CUMULATIVE_RECORD, line_number=line_number)
266 convert_date_fields(cumulative, TO_CUMULATIVE_RECORD_DATES, tz)
267 convert_decimal_fields(cumulative, TO_CUMULATIVE_RECORD_DECIMALS)
268 line_number += 1
269 elif record_type == "T51":
270 cumulative_adjustment = parse_records(line, TO_CUMULATIVE_RECORD, line_number=line_number)
271 convert_date_fields(cumulative_adjustment, TO_CUMULATIVE_RECORD_DATES, tz)
272 convert_decimal_fields(cumulative_adjustment, TO_CUMULATIVE_RECORD_DECIMALS)
273 line_number += 1
274 elif record_type in ("T60", "T70"):
275 special_records.append(parse_records(line, TO_SPECIAL_RECORD, line_number=line_number, check_record_length=False))
276 line_number += 1
277 else:
278 raise ValidationError(_("Unknown record type on {}({}): {}").format(filename, line_number, record_type))
280 statements.append(combine_statement(header, records, balance, cumulative, cumulative_adjustment, special_records))
281 return statements
284def combine_statement(header, records, balance, cumulative, cumulative_adjustment, special_records) -> Dict[str, Any]: # pylint: disable=too-many-arguments
285 data = {
286 "header": header,
287 "records": records,
288 }
289 if balance is not None:
290 data["balance"] = balance
291 if cumulative is not None:
292 data["cumulative"] = cumulative
293 if cumulative_adjustment is not None:
294 data["cumulative_adjustment"] = cumulative_adjustment
295 if special_records:
296 data["special_records"] = special_records
297 return data
300def parse_record_extra_info(record: Dict[str, Any], line: str, line_number: int):
301 if line[:3] not in TO_FILE_RECORD_EXTRA_INFO_TYPES:
302 raise ValidationError("SVM record extra info validation error on line {}".format(line_number))
304 header = parse_records(line, TO_FILE_RECORD_EXTRA_INFO_HEADER, line_number, check_record_length=False)
305 extra_info_type = header["extra_info_type"]
306 # print(line)
307 extra_data = copy(header["extra_data"])
308 assert isinstance(extra_data, str)
309 if extra_info_type == "00":
310 record["messages"] = parse_record_messages(extra_data)
311 elif extra_info_type == "01":
312 record["counts"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_COUNTS, line_number, record_length=8)
313 elif extra_info_type == "02":
314 record["invoice"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_INVOICE, line_number, record_length=33)
315 elif extra_info_type == "03":
316 record["card"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_CARD, line_number, record_length=34)
317 elif extra_info_type == "04":
318 record["correction"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_CORRECTION, line_number, record_length=18)
319 elif extra_info_type == "05":
320 record["currency"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_CURRENCY, line_number, record_length=41)
321 convert_decimal_fields(record["currency"], TO_FILE_RECORD_EXTRA_INFO_CURRENCY_DECIMALS)
322 elif extra_info_type == "06":
323 record["client_messages"] = parse_record_messages(extra_data)
324 elif extra_info_type == "07":
325 record["bank_messages"] = parse_record_messages(extra_data)
326 elif extra_info_type == "08":
327 record["reason"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_REASON, line_number, record_length=35)
328 elif extra_info_type == "09":
329 record["name_detail"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_NAME_DETAIL, line_number, record_length=35)
330 elif extra_info_type == "11":
331 record["sepa"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_SEPA, line_number, record_length=323)
332 else:
333 raise ValidationError(_('Line {line}: Invalid record extra info type "{extra_info_type}"').format(line=line_number, extra_info_type=extra_info_type))
336def parse_record_messages(extra_data: str) -> List[str]:
337 msg = []
338 while extra_data:
339 msg.append(extra_data[:35])
340 extra_data = extra_data[35:]
341 return msg