Coverage for jbank/tito.py: 83%

149 statements  

« prev     ^ index     » next       coverage.py v7.2.2, created at 2023-03-27 13:36 +0700

1from copy import copy 

2from os.path import basename 

3from typing import Dict, Any, List 

4from django.core.exceptions import ValidationError 

5from django.utils.translation import gettext as _ 

6from pytz import timezone 

7from jbank.parsers import parse_filename_suffix, parse_records, convert_date_fields, convert_decimal_fields 

8 

9TO_STATEMENT_SUFFIXES = ("TO", "TXT", "TITO") 

10 

11TO_FILE_HEADER_TYPES = ("T00",) 

12 

13TO_FILE_HEADER_DATES = ( 

14 "begin_date", 

15 "end_date", 

16 ("record_date", "record_time"), 

17 "begin_balance_date", 

18) 

19 

20TO_FILE_HEADER_DECIMALS = ( 

21 ("begin_balance", "begin_balance_sign"), 

22 "account_limit", 

23) 

24 

25TO_FILE_HEADER = ( 

26 ("statement_type", "X", "P"), 

27 ("record_type", "XX", "P"), 

28 ("record_length", "9(3)", "P"), 

29 ("version", "X(3)", "P"), 

30 ("account_number", "X(14)", "P"), 

31 ("statement_number", "9(3)", "P"), 

32 ("begin_date", "9(6)", "P"), 

33 ("end_date", "9(6)", "P"), 

34 ("record_date", "9(6)", "P"), 

35 ("record_time", "9(4)", "P"), 

36 ("customer_identifier", "X(17)", "P"), 

37 ("begin_balance_date", "9(6)", "P"), 

38 ("begin_balance_sign", "X", "P"), 

39 ("begin_balance", "9(18)", "P"), 

40 ("record_count", "9(6)", "P"), 

41 ("currency_code", "X(3)", "P"), 

42 ("account_name", "X(30)", "V"), 

43 ("account_limit", "9(18)", "P"), 

44 ("owner_name", "X(35)", "P"), 

45 ("contact_info_1", "X(40)", "P"), 

46 ("contact_info_2", "X(40)", "V"), 

47 ("bank_specific_info_1", "X(30)", "V"), 

48 ("iban_and_bic", "X(30)", "V"), 

49) 

50 

51TO_FILE_RECORD_TYPES = ("T10", "T80") 

52 

53TO_FILE_RECORD_DATES = ( 

54 "record_date", 

55 "value_date", 

56 "paid_date", 

57) 

58 

59TO_FILE_RECORD_DECIMALS = (("amount", "amount_sign"),) 

60 

61TO_FILE_RECORD = ( 

62 ("statement_type", "X", "P"), 

63 ("record_type", "XX", "P"), 

64 ("record_length", "9(3)", "P"), 

65 ("record_number", "9(6)", "P"), 

66 ("archive_identifier", "X(18)", "V"), 

67 ("record_date", "9(6)", "P"), 

68 ("value_date", "9(6)", "V"), 

69 ("paid_date", "9(6)", "V"), 

70 ("entry_type", "X", "P"), # 1 = pano, 2 = otto, 3 = panon korjaus, 4 = oton korjaus, 9 = hylätty tapahtuma 

71 ("record_code", "X(3)", "P"), 

72 ("record_description", "X(35)", "P"), 

73 ("amount_sign", "X", "P"), 

74 ("amount", "9(18)", "P"), 

75 ("receipt_code", "X", "P"), 

76 ("delivery_method", "X", "P"), 

77 ("name", "X(35)", "V"), 

78 ("name_source", "X", "V"), 

79 ("recipient_account_number", "X(14)", "V"), 

80 ("recipient_account_number_changed", "X", "V"), 

81 ("remittance_info", "X(20)", "V"), 

82 ("form_number", "X(8)", "V"), 

83 ("level_identifier", "X", "P"), 

84) 

85 

86TO_FILE_RECORD_EXTRA_INFO_TYPES = ("T11", "T81") 

87 

88TO_FILE_RECORD_EXTRA_INFO_HEADER = ( 

89 ("statement_type", "X", "P"), 

90 ("record_type", "XX", "P"), 

91 ("record_length", "9(3)", "P"), 

92 ("extra_info_type", "9(2)", "P"), 

93) 

94 

95TO_FILE_RECORD_EXTRA_INFO_COUNTS = (("entry_count", "9(8)", "P"),) 

96 

97TO_FILE_RECORD_EXTRA_INFO_INVOICE = ( 

98 ("customer_number", "X(10)", "P"), 

99 ("pad01", "X", "P"), 

100 ("invoice_number", "X(15)", "P"), 

101 ("pad02", "X", "P"), 

102 ("invoice_date", "X(6)", "P"), 

103) 

104 

105TO_FILE_RECORD_EXTRA_INFO_CARD = ( 

106 ("card_number", "X(19)", "P"), 

107 ("pad01", "X", "P"), 

108 ("merchant_reference", "X(14)", "P"), 

109) 

110 

111TO_FILE_RECORD_EXTRA_INFO_CORRECTION = (("original_archive_identifier", "X(18)", "P"),) 

112 

113TO_FILE_RECORD_EXTRA_INFO_CURRENCY_DECIMALS = (("amount", "amount_sign"),) 

114 

115TO_FILE_RECORD_EXTRA_INFO_CURRENCY = ( 

116 ("amount_sign", "X", "P"), 

117 ("amount", "9(18)", "P"), 

118 ("pad01", "X", "P"), 

119 ("currency_code", "X(3)", "P"), 

120 ("pad02", "X", "P"), 

121 ("exchange_rate", "9(11)", "P"), 

122 ("rate_reference", "X(6)", "V"), 

123) 

124 

125TO_FILE_RECORD_EXTRA_INFO_REASON = ( 

126 ("reason_code", "9(3)", "P"), 

127 ("pad01", "X", "P"), 

128 ("reason_description", "X(31)", "P"), 

129) 

130 

131TO_FILE_RECORD_EXTRA_INFO_NAME_DETAIL = (("name_detail", "X(35)", "P"),) 

132 

133TO_FILE_RECORD_EXTRA_INFO_SEPA = ( 

134 ("reference", "X(35)", "V"), 

135 ("iban_account_number", "X(35)", "V"), 

136 ("bic_code", "X(35)", "V"), 

137 ("recipient_name_detail", "X(70)", "V"), 

138 ("payer_name_detail", "X(70)", "V"), 

139 ("identifier", "X(35)", "V"), 

140 ("archive_identifier", "X(35)", "V"), 

141) 

142 

143TO_BALANCE_RECORD_DATES = ("record_date",) 

144 

145TO_BALANCE_RECORD_DECIMALS = ( 

146 ("end_balance", "end_balance_sign"), 

147 ("available_balance", "available_balance_sign"), 

148) 

149 

150TO_BALANCE_RECORD = ( 

151 ("statement_type", "X", "P"), 

152 ("record_type", "XX", "P"), 

153 ("record_length", "9(3)", "P"), 

154 ("record_date", "9(6)", "P"), 

155 ("end_balance_sign", "X", "P"), 

156 ("end_balance", "9(18)", "P"), 

157 ("available_balance_sign", "X", "P"), 

158 ("available_balance", "9(18)", "P"), 

159) 

160 

161TO_CUMULATIVE_RECORD_DATES = ("period_date",) 

162 

163TO_CUMULATIVE_RECORD_DECIMALS = ( 

164 ("deposits_amount", "deposits_sign"), 

165 ("withdrawals_amount", "withdrawals_sign"), 

166) 

167 

168TO_CUMULATIVE_RECORD = ( 

169 ("statement_type", "X", "P"), 

170 ("record_type", "XX", "P"), 

171 ("record_length", "9(3)", "P"), 

172 ("period_identifier", "X", "P"), # 1=day, 2=term, 3=month, 4=year 

173 ("period_date", "9(6)", "P"), 

174 ("deposits_count", "9(8)", "P"), 

175 ("deposits_sign", "X", "P"), 

176 ("deposits_amount", "9(18)", "P"), 

177 ("withdrawals_count", "9(8)", "P"), 

178 ("withdrawals_sign", "X", "P"), 

179 ("withdrawals_amount", "9(18)", "P"), 

180) 

181 

182TO_SPECIAL_RECORD = ( 

183 ("statement_type", "X", "P"), 

184 ("record_type", "XX", "P"), 

185 ("record_length", "9(3)", "P"), 

186 ("bank_group_identifier", "X(3)", "P"), 

187) 

188 

189 

190def parse_tiliote_statements_from_file(filename: str) -> list: 

191 if parse_filename_suffix(filename).upper() not in TO_STATEMENT_SUFFIXES: 

192 raise ValidationError( 

193 _('File {filename} has unrecognized ({suffixes}) suffix for file type "{file_type}"').format( 

194 filename=filename, suffixes=", ".join(TO_STATEMENT_SUFFIXES), file_type="tiliote" 

195 ) 

196 ) 

197 with open(filename, "rt", encoding="ISO-8859-1") as fp: 

198 return parse_tiliote_statements(fp.read(), filename=basename(filename)) # type: ignore 

199 

200 

201def parse_tiliote_statements(content: str, filename: str) -> List[dict]: # pylint: disable=too-many-locals 

202 lines = content.split("\n") 

203 nlines = len(lines) 

204 

205 line_number = 1 

206 tz = timezone("Europe/Helsinki") 

207 

208 header = None 

209 records = [] 

210 balance = None 

211 cumulative = None 

212 cumulative_adjustment = None 

213 special_records = [] 

214 statements = [] 

215 

216 while line_number <= nlines: 

217 line = lines[line_number - 1] 

218 if line.strip() == "": 

219 line_number += 1 

220 continue 

221 # print('parsing line {}: {}'.format(line_number, line)) 

222 record_type = line[:3] 

223 

224 if record_type in TO_FILE_HEADER_TYPES: 

225 if header: 

226 statements.append(combine_statement(header, records, balance, cumulative, cumulative_adjustment, special_records)) 

227 header, records, balance, cumulative, cumulative_adjustment, special_records = ( 

228 None, 

229 [], 

230 None, 

231 None, 

232 None, 

233 [], 

234 ) 

235 

236 header = parse_records(lines[line_number - 1], TO_FILE_HEADER, line_number=line_number) 

237 convert_date_fields(header, TO_FILE_HEADER_DATES, tz) 

238 convert_decimal_fields(header, TO_FILE_HEADER_DECIMALS) 

239 iban_and_bic = str(header.get("iban_and_bic", "")).split(" ") 

240 if len(iban_and_bic) == 2: 

241 header["iban"], header["bic"] = iban_and_bic 

242 line_number += 1 

243 

244 elif record_type in TO_FILE_RECORD_TYPES: 

245 record = parse_records(line, TO_FILE_RECORD, line_number=line_number) 

246 convert_date_fields(record, TO_FILE_RECORD_DATES, tz) 

247 convert_decimal_fields(record, TO_FILE_RECORD_DECIMALS) 

248 

249 line_number += 1 

250 # check for record extra info 

251 if line_number <= nlines: 

252 line = lines[line_number - 1] 

253 while line[:3] in TO_FILE_RECORD_EXTRA_INFO_TYPES: 

254 parse_record_extra_info(record, line, line_number=line_number) 

255 line_number += 1 

256 line = lines[line_number - 1] 

257 

258 records.append(record) 

259 elif record_type == "T40": 

260 balance = parse_records(line, TO_BALANCE_RECORD, line_number=line_number) 

261 convert_date_fields(balance, TO_BALANCE_RECORD_DATES, tz) 

262 convert_decimal_fields(balance, TO_BALANCE_RECORD_DECIMALS) 

263 line_number += 1 

264 elif record_type == "T50": 

265 cumulative = parse_records(line, TO_CUMULATIVE_RECORD, line_number=line_number) 

266 convert_date_fields(cumulative, TO_CUMULATIVE_RECORD_DATES, tz) 

267 convert_decimal_fields(cumulative, TO_CUMULATIVE_RECORD_DECIMALS) 

268 line_number += 1 

269 elif record_type == "T51": 

270 cumulative_adjustment = parse_records(line, TO_CUMULATIVE_RECORD, line_number=line_number) 

271 convert_date_fields(cumulative_adjustment, TO_CUMULATIVE_RECORD_DATES, tz) 

272 convert_decimal_fields(cumulative_adjustment, TO_CUMULATIVE_RECORD_DECIMALS) 

273 line_number += 1 

274 elif record_type in ("T60", "T70"): 

275 special_records.append(parse_records(line, TO_SPECIAL_RECORD, line_number=line_number, check_record_length=False)) 

276 line_number += 1 

277 else: 

278 raise ValidationError(_("Unknown record type on {}({}): {}").format(filename, line_number, record_type)) 

279 

280 statements.append(combine_statement(header, records, balance, cumulative, cumulative_adjustment, special_records)) 

281 return statements 

282 

283 

284def combine_statement(header, records, balance, cumulative, cumulative_adjustment, special_records) -> Dict[str, Any]: # pylint: disable=too-many-arguments 

285 data = { 

286 "header": header, 

287 "records": records, 

288 } 

289 if balance is not None: 

290 data["balance"] = balance 

291 if cumulative is not None: 

292 data["cumulative"] = cumulative 

293 if cumulative_adjustment is not None: 

294 data["cumulative_adjustment"] = cumulative_adjustment 

295 if special_records: 

296 data["special_records"] = special_records 

297 return data 

298 

299 

300def parse_record_extra_info(record: Dict[str, Any], line: str, line_number: int): 

301 if line[:3] not in TO_FILE_RECORD_EXTRA_INFO_TYPES: 

302 raise ValidationError("SVM record extra info validation error on line {}".format(line_number)) 

303 

304 header = parse_records(line, TO_FILE_RECORD_EXTRA_INFO_HEADER, line_number, check_record_length=False) 

305 extra_info_type = header["extra_info_type"] 

306 # print(line) 

307 extra_data = copy(header["extra_data"]) 

308 assert isinstance(extra_data, str) 

309 if extra_info_type == "00": 

310 record["messages"] = parse_record_messages(extra_data) 

311 elif extra_info_type == "01": 

312 record["counts"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_COUNTS, line_number, record_length=8) 

313 elif extra_info_type == "02": 

314 record["invoice"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_INVOICE, line_number, record_length=33) 

315 elif extra_info_type == "03": 

316 record["card"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_CARD, line_number, record_length=34) 

317 elif extra_info_type == "04": 

318 record["correction"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_CORRECTION, line_number, record_length=18) 

319 elif extra_info_type == "05": 

320 record["currency"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_CURRENCY, line_number, record_length=41) 

321 convert_decimal_fields(record["currency"], TO_FILE_RECORD_EXTRA_INFO_CURRENCY_DECIMALS) 

322 elif extra_info_type == "06": 

323 record["client_messages"] = parse_record_messages(extra_data) 

324 elif extra_info_type == "07": 

325 record["bank_messages"] = parse_record_messages(extra_data) 

326 elif extra_info_type == "08": 

327 record["reason"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_REASON, line_number, record_length=35) 

328 elif extra_info_type == "09": 

329 record["name_detail"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_NAME_DETAIL, line_number, record_length=35) 

330 elif extra_info_type == "11": 

331 record["sepa"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_SEPA, line_number, record_length=323) 

332 else: 

333 raise ValidationError(_('Line {line}: Invalid record extra info type "{extra_info_type}"').format(line=line_number, extra_info_type=extra_info_type)) 

334 

335 

336def parse_record_messages(extra_data: str) -> List[str]: 

337 msg = [] 

338 while extra_data: 

339 msg.append(extra_data[:35]) 

340 extra_data = extra_data[35:] 

341 return msg