Coverage for testrail_api_reporter/engines/results_reporter.py: 12%

302 statements  

« prev     ^ index     » next       coverage.py v7.6.1, created at 2024-08-29 15:21 +0200

1# -*- coding: utf-8 -*- 

2""" Module for reporting results to TestRails from xml report results, obtained by pytest """ 

3 

4import datetime 

5from os.path import exists 

6 

7from requests.exceptions import ReadTimeout 

8from testrail_api import TestRailAPI, StatusCodeError 

9from xmltodict import parse 

10 

11from ..utils.logger_config import setup_logger, DEFAULT_LOGGING_LEVEL 

12from ..utils.reporter_utils import format_error, init_get_cases_process 

13 

14 

15class TestRailResultsReporter: 

16 """Reporter to TestRails from xml report results, obtained by pytest""" 

17 

18 def __init__( 

19 self, 

20 url: str, 

21 email: str, 

22 password: str, 

23 project_id: int, 

24 xml_report="junit-report.xml", 

25 suite_id=None, 

26 logger=None, 

27 log_level=DEFAULT_LOGGING_LEVEL, 

28 ): 

29 """ 

30 Default init 

31 

32 :param url: url of TestRail, string, required 

33 :param email: email of TestRail user with proper access rights, string, required 

34 :param password: password of TestRail user with proper access rights, string, required 

35 :param project_id: project id, integer, required 

36 :param xml_report: filename (maybe with path) of xml test report 

37 :param suite_id: suite id, integer, optional, if no suite-management is activated 

38 :param logger: logger object, optional 

39 :param log_level: logging level, optional, by default is 'logging.DEBUG' 

40 """ 

41 if not logger: 

42 self.___logger = setup_logger( 

43 name="TestRailResultsReporter", log_file="TestRailResultsReporter.log", level=log_level 

44 ) 

45 else: 

46 self.___logger = logger 

47 self.___logger.debug("Initializing TestRail Results Reporter") 

48 if url is None or email is None or password is None: 

49 raise ValueError("No TestRails credentials are provided!") 

50 self.__api = TestRailAPI(url, email, password) 

51 self.__xml_report = xml_report if self.__check_report_exists(xml_report=xml_report) else None 

52 self.__project_id = project_id if self.__check_project(project_id=project_id) else None 

53 self.__suite_id = suite_id if self.__check_suite(suite_id=suite_id) else None 

54 self.__at_section = self.__ensure_automation_section() if self.__project_id else None 

55 self.__check_section(section_id=self.__at_section) 

56 self.__timestamp = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S") 

57 

58 def __xml_to_dict(self, filename="junit-report.xml"): 

59 """ 

60 Converts xml file to python dict 

61 :param filename: filename, string, maybe with path 

62 :return: dict with a list of cases 

63 """ 

64 if not self.__check_report_exists(xml_report=self.__xml_report): 

65 return None 

66 with open(filename, "r", encoding="utf-8") as file: 

67 xml = file.read() 

68 

69 list_of_cases = [] 

70 

71 parsed_xml = parse(xml) 

72 self.__timestamp = parsed_xml["testsuites"]["testsuite"]["@timestamp"].split(".")[0] 

73 

74 cases = parsed_xml["testsuites"]["testsuite"]["testcase"] 

75 cases = cases if isinstance(cases, list) else [cases] 

76 for item in cases: 

77 status = 1 

78 if "failure" in item.keys(): 

79 status = 5 

80 if "skipped" in item.keys(): 

81 if item["skipped"]["@type"] == "pytest.xfail": 

82 status = 5 

83 else: 

84 status = 7 

85 list_of_cases.append( 

86 { 

87 "automation_id": f'{item["@classname"]}.{item["@name"]}', 

88 "time": item["@time"], 

89 "status": status, 

90 "comment": ( 

91 f'{item["failure"]["@message"]} : ' f'{item["failure"]["#text"]}' 

92 if "failure" in item.keys() 

93 else "" 

94 ), 

95 } 

96 ) 

97 self.___logger.debug("Found test run at %s, found %s test results", self.__timestamp, len(list_of_cases)) 

98 return list_of_cases 

99 

100 @staticmethod 

101 def __search_for_item(searched_value, list_to_seek, field): 

102 """ 

103 Item seeker by value within a list of dicts 

104 

105 :param searched_value: value what we're looking for 

106 :param list_to_seek: a list where we perform the search 

107 :param field: field of a list dict 

108 :return: element 

109 """ 

110 for item in list_to_seek: 

111 if item[field] == searched_value: 

112 return item 

113 

114 return [element for element in list_to_seek if element[field] == searched_value] 

115 

116 def __ensure_automation_section(self, title="pytest"): 

117 """ 

118 Service function, checks that special (default) placeholder for automation non-classified tests exists 

119 

120 :param title: title for default folder, string 

121 :return: id of a section 

122 """ 

123 first_run = True 

124 item_id = None 

125 criteria = None 

126 response = None 

127 while criteria is not None or first_run: 

128 if first_run: 

129 try: 

130 response = self.__api.sections.get_sections(project_id=self.__project_id, suite_id=self.__suite_id) 

131 except Exception as error: 

132 self.___logger.error( 

133 "Get sections failed. Please validate your settings!\nError%s", format_error(error) 

134 ) 

135 self.__self_check() 

136 return None 

137 first_run = False 

138 elif response["_links"]["next"] is not None: # pylint: disable=E1136 

139 offset = int(response["_links"]["next"].split("&offset=")[1].split("&")[0]) # pylint: disable=E1136 

140 response = self.__api.sections.get_sections( 

141 project_id=self.__project_id, suite_id=self.__suite_id, offset=offset 

142 ) 

143 sections = response["sections"] 

144 for item in sections: 

145 if item["name"] == title: 

146 item_id = item["id"] 

147 criteria = response["_links"]["next"] 

148 if not item_id: 

149 try: 

150 item_id = self.__api.sections.add_section( 

151 project_id=self.__project_id, suite_id=self.__suite_id, name=title 

152 )["id"] 

153 except Exception as error: 

154 self.___logger.error("Can't add section. Something nasty happened.\nError%s", format_error(error)) 

155 self.__self_check() 

156 return None 

157 self.___logger.debug("No default automation folder is found, created new one with name '%s'", title) 

158 return item_id 

159 

160 def __enrich_with_tc_num(self, xml_dict_list, tc_dict_list): 

161 """ 

162 Add a test case id to case result 

163 

164 :param xml_dict_list: list of dict, with test cases, obtained from xml report 

165 :param tc_dict_list: list of dict, with test cases, obtained from TestRails 

166 :return: enriched list of dict with test cases 

167 """ 

168 enriched_list = [] 

169 missed_tests_counter = 0 

170 for item in xml_dict_list: 

171 cases = self.__search_for_item( 

172 searched_value=item["automation_id"], list_to_seek=tc_dict_list, field="custom_automation_id" 

173 ) 

174 if not cases: 

175 try: 

176 cases = [ 

177 self.__api.cases.add_case( 

178 section_id=self.__at_section, 

179 title=item["automation_id"], 

180 custom_automation_id=item["automation_id"], 

181 ) 

182 ] 

183 except Exception as error: 

184 self.___logger.error( 

185 "Add case failed. Please validate your settings!\nError: %s", format_error(error) 

186 ) 

187 self.__self_check() 

188 return None 

189 missed_tests_counter = missed_tests_counter + 1 

190 cases = cases if isinstance(cases, list) else [cases] 

191 for case in cases: 

192 comment = item["message"] if "failure" in item.keys() else "" 

193 elapsed = item["time"].split(".")[0] 

194 elapsed = 1 if elapsed == 0 else elapsed 

195 enriched_list.append( 

196 { 

197 "case_id": case["id"], # type: ignore 

198 "status_id": item["status"], 

199 "comment": comment, 

200 "elapsed": elapsed, 

201 "attachments": [], 

202 } 

203 ) 

204 if missed_tests_counter: 

205 self.___logger.debug("Missed %s test cases, they was automatically created", missed_tests_counter) 

206 self.___logger.debug("Found %s test cases in TestRails", len(enriched_list)) 

207 return enriched_list 

208 

209 def ___handle_read_timeout(self, retry, retries, error): 

210 if retry < retries: 

211 retry += 1 

212 self.___logger.debug("Timeout error, retrying %s/%s...", retry, retries) 

213 return retry, True 

214 raise ValueError(f"Get cases failed. Please validate your settings!nError{format_error(error)}") from error 

215 

216 # pylint: disable=R0912 

217 def __get_all_auto_cases(self, retries=3): 

218 """ 

219 Collects all test cases from TestRails with non-empty automation_id 

220 

221 :param retries: number of retries, integer 

222 :return: list of dict with cases 

223 """ 

224 cases_list, first_run, criteria, response, retry = init_get_cases_process() 

225 while criteria is not None or first_run: 

226 if first_run: 

227 try: 

228 response = self.__api.cases.get_cases(project_id=self.__project_id, suite_id=self.__suite_id) 

229 except (ReadTimeout, StatusCodeError) as error: 

230 if ( 

231 isinstance(error, StatusCodeError) 

232 and error.status_code == 504 # type: ignore # pylint: disable=no-member 

233 or isinstance(error, ReadTimeout) 

234 ): 

235 retry, should_continue = self.___handle_read_timeout(retry, retries, error) 

236 if should_continue: 

237 continue 

238 except Exception as error: 

239 self.___logger.error( 

240 "Get cases failed. Please validate your settings!\nError%s", format_error(error) 

241 ) 

242 self.__self_check() 

243 return None 

244 first_run = False 

245 retry = 0 

246 elif response["_links"]["next"] is not None: # pylint: disable=E1136 

247 offset = int(response["_links"]["next"].split("&offset=")[1].split("&")[0]) # pylint: disable=E1136 

248 try: 

249 response = self.__api.cases.get_cases( 

250 project_id=self.__project_id, suite_id=self.__suite_id, offset=offset 

251 ) 

252 except (ReadTimeout, StatusCodeError) as error: 

253 if ( 

254 isinstance(error, StatusCodeError) 

255 and error.status_code == 504 # type: ignore # pylint: disable=no-member 

256 or isinstance(error, ReadTimeout) 

257 ): 

258 retry, should_continue = self.___handle_read_timeout(retry, retries, error) 

259 if should_continue: 

260 continue 

261 retry = 0 

262 cases = response["cases"] 

263 for item in cases: 

264 if item["custom_automation_id"] is not None: 

265 cases_list.append({"id": item["id"], "custom_automation_id": item["custom_automation_id"]}) 

266 criteria = response["_links"]["next"] 

267 self.___logger.debug("Found %s test cases in TestRails with automation_id", len(cases_list)) 

268 return cases_list 

269 

270 def __prepare_payload(self): 

271 """ 

272 Prepares payload from xml report for sending to TestRails 

273 

274 :return: payload in proper format (list of dicts) 

275 """ 

276 parsed_xml = self.__xml_to_dict(filename=self.__xml_report) 

277 parsed_cases = self.__get_all_auto_cases() 

278 if not parsed_xml: 

279 self.___logger.error("Preparation of payload failed, aborted") 

280 return None 

281 payload = self.__enrich_with_tc_num(xml_dict_list=parsed_xml, tc_dict_list=parsed_cases) 

282 return payload 

283 

284 def __prepare_title(self, environment=None, timestamp=None): 

285 """ 

286 Format test run name based on input string (most probably environment) and timestamp 

287 

288 :param environment: some string run identifier 

289 :param timestamp: custom timestamp 

290 :return: string of prepared string for AT run name 

291 """ 

292 if timestamp is None: 

293 timestamp = self.__timestamp 

294 title = f"AT run {timestamp}" 

295 if environment: 

296 title = f"{title} on {environment}" 

297 return title 

298 

299 def send_results( 

300 self, 

301 run_id=None, 

302 environment=None, 

303 title=None, 

304 timestamp=None, 

305 close_run=True, 

306 run_name=None, 

307 delete_old_run=False, 

308 ): 

309 """ 

310 Send results to TestRail 

311 

312 :param run_id: specific run id, if any, optional 

313 :param environment: custom name pattern for run name, optional 

314 :param title: custom title, if provided, will be used as priority, optional 

315 :param timestamp: custom timestamp, optional 

316 :param close_run: close or not run, True or False 

317 :param run_name: name of test run, will be used if provided at top priority 

318 :param delete_old_run: delete or not previous run if old one exists with the same name 

319 :return: run id where results were submitted 

320 """ 

321 if ( 

322 not self.__project_id 

323 or not self.__at_section 

324 or not self.__check_report_exists(xml_report=self.__xml_report) 

325 ): 

326 self.___logger.error("Error! Please specify all required params!") 

327 self.__self_check() 

328 return True 

329 title = self.__prepare_title(environment, timestamp) if not title else title 

330 title = run_name if run_name else title 

331 payload = self.__prepare_payload() 

332 run_id = self.__prepare_runs( 

333 cases=payload, title=title, run_id=run_id, run_name=run_name, delete_run=delete_old_run 

334 ) 

335 retval = self.__add_results(run_id=run_id, results=payload) 

336 if close_run: 

337 self.__close_run(run_id=run_id, title=title) 

338 self.___logger.debug("%s results were added to test run '%s', cases updated. Done", len(payload), title) 

339 return retval 

340 

341 def set_project_id(self, project_id): 

342 """ 

343 Set project id 

344 

345 :param project_id: project id, integer 

346 """ 

347 self.__project_id = project_id if self.__check_project(project_id=project_id) else None 

348 

349 def set_suite_id(self, suite_id): 

350 """ 

351 Set suite id 

352 

353 :param suite_id: suite id, integer 

354 """ 

355 if self.__check_project(): 

356 self.__suite_id = suite_id if self.__check_suite(suite_id=suite_id) else None 

357 

358 def set_xml_filename(self, xml_filename): 

359 """ 

360 Set xml filename 

361 

362 :param xml_filename: filename of xml report, string 

363 """ 

364 self.__xml_report = xml_filename if self.__check_report_exists(xml_report=xml_filename) else None 

365 

366 def set_at_report_section(self, section_name): 

367 """ 

368 Set section name for AT a report 

369 

370 :param section_name: name of a section, string 

371 """ 

372 if self.__check_project() and self.__check_suite(): 

373 self.__at_section = self.__ensure_automation_section(title=section_name) 

374 

375 def set_timestamp(self, new_timestamp): 

376 """ 

377 Set timestamp 

378 

379 :param new_timestamp: timestamp, string 

380 """ 

381 self.__timestamp = new_timestamp 

382 

383 def __check_project(self, project_id=None): 

384 """ 

385 Check that the project exists 

386 

387 :param project_id: project id, integer 

388 :return: True or False 

389 """ 

390 retval = True 

391 try: 

392 self.__api.projects.get_project(project_id=project_id) 

393 except Exception as error: 

394 self.___logger.error("No such project is found, please set valid project ID.\nError%s", format_error(error)) 

395 retval = False 

396 return retval 

397 

398 def __check_suite(self, suite_id=None): 

399 """ 

400 Check that the suite exists 

401 

402 :param suite_id: id of suite, integer 

403 :return: True or False 

404 """ 

405 retval = True 

406 try: 

407 self.__api.suites.get_suite(suite_id=suite_id) 

408 except Exception as error: 

409 self.___logger.error("No such suite is found, please set valid suite ID.\nError%s", format_error(error)) 

410 retval = False 

411 return retval 

412 

413 def __check_section(self, section_id=None): 

414 """ 

415 Check that the section exists 

416 

417 :param section_id: id of suite, integer 

418 :return: True or False 

419 """ 

420 retval = True 

421 try: 

422 self.__api.sections.get_section(section_id=section_id) 

423 except Exception as error: 

424 self.___logger.error("No such section is found, please set valid section ID.\nError%s", format_error(error)) 

425 retval = False 

426 return retval 

427 

428 def __check_report_exists(self, xml_report=None): 

429 """ 

430 Check that the xml report exists 

431 

432 :param xml_report: filename of the xml report, maybe with path 

433 :return: True or False 

434 """ 

435 retval = False 

436 if not xml_report: 

437 xml_report = self.__xml_report 

438 if xml_report: 

439 if exists(xml_report): 

440 retval = True 

441 if not retval: 

442 self.___logger.error("Please specify correct path.\nError 404: No XML file found") 

443 return retval 

444 

445 def __check_run_exists(self, run_id=None): 

446 """ 

447 Check that the run exists 

448 

449 :param run_id: id of suite, integer 

450 :return: True or False 

451 """ 

452 retval = True 

453 try: 

454 self.__api.runs.get_run(run_id=run_id) 

455 except Exception as error: 

456 self.___logger.error("No such run is found, please set valid run ID.\nError%s", format_error(error)) 

457 retval = False 

458 return retval 

459 

460 def __self_check(self): 

461 """ 

462 Health checker, calls checks 

463 """ 

464 self.__check_project(project_id=self.__project_id) 

465 self.__check_suite(suite_id=self.__suite_id) 

466 self.__check_section(section_id=self.__at_section) 

467 self.__check_report_exists(xml_report=self.__xml_report) 

468 

469 def __search_for_run_by_name(self, title=None): 

470 """ 

471 Search run by name 

472 

473 :param title: name of the run 

474 :return: id, integer 

475 """ 

476 retval = None 

477 first_run = True 

478 criteria = None 

479 while criteria is not None or first_run: 

480 response = None 

481 if first_run: 

482 try: 

483 response = self.__api.runs.get_runs(project_id=self.__project_id, suite_id=self.__suite_id) 

484 except Exception as error: 

485 self.___logger.error("Can't get run list. Something nasty happened.\nError%s", format_error(error)) 

486 break 

487 first_run = False 

488 elif response["_links"]["next"] is not None: # pylint: disable=E1136 

489 offset = int(response["_links"]["next"].split("&offset=")[1].split("&")[0]) # pylint: disable=E1136 

490 response = self.__api.runs.get_runs( 

491 project_id=self.__project_id, suite_id=self.__suite_id, offset=offset 

492 ) 

493 if response["runs"]["name"] == title: 

494 retval = response["runs"]["id"] 

495 break 

496 criteria = response["_links"]["next"] 

497 return retval 

498 

499 def __delete_run(self, run_id=None): 

500 """ 

501 Delete run 

502 

503 :param run_id: run id, integer 

504 :return: True if deleted, False in case of error 

505 """ 

506 retval = True 

507 try: 

508 self.__api.runs.delete_run(run_id=run_id) 

509 except Exception as error: 

510 self.___logger.error("Can't delete run. Something nasty happened.\nError%s", format_error(error)) 

511 retval = False 

512 return retval 

513 

514 def __add_run(self, title, cases_list=None, include_all=False): 

515 """ 

516 Add a run 

517 

518 :param title: title of run 

519 :param cases_list: test cases, which will be added to run 

520 :param include_all: every existing testcases may be included, by default - False 

521 :return: id of run, integer 

522 """ 

523 retval = None 

524 self.___logger.debug("Creating new test run '%s'", title) 

525 try: 

526 retval = self.__api.runs.add_run( 

527 project_id=self.__project_id, 

528 suite_id=self.__suite_id, 

529 name=title, 

530 include_all=include_all, 

531 case_ids=cases_list, 

532 )["id"] 

533 except Exception as error: 

534 self.___logger.error("Can't add run. Something nasty happened.\nError%s", format_error(error)) 

535 self.__self_check() 

536 return retval 

537 

538 def __add_results(self, run_id=None, results=None): 

539 """ 

540 Add results for test cases to TestRail 

541 

542 :param run_id: run id 

543 :param results: payload (list of dicts) 

544 :return: run id or False in case of error 

545 """ 

546 retval = False 

547 try: 

548 self.__api.results.add_results_for_cases(run_id=run_id, results=results) 

549 return run_id 

550 except Exception as error: 

551 self.___logger.error("Add results failed. Please validate your settings!\nError%s", format_error(error)) 

552 self.__self_check() 

553 self.__check_run_exists(run_id=run_id) 

554 return retval 

555 

556 def __prepare_runs(self, cases=None, title=None, run_id=None, run_name=None, delete_run=False): 

557 """ 

558 Prepare run for submitting 

559 

560 :param cases: list of cases (list of dicts) 

561 :param title: title of test run (which will be submitted) 

562 :param run_id: run id 

563 :param run_name: 

564 :param delete_run: delete existing run or not (True or False), will be checked via run_name 

565 :return: run id 

566 """ 

567 cases_list = [] 

568 for item in cases: 

569 cases_list.append(item["case_id"]) 

570 if run_name: 

571 run_id = self.__search_for_run_by_name(title=run_name) 

572 if not run_id: 

573 self.___logger.debug("No run has been found by given name") 

574 if delete_run and run_id: 

575 self.__delete_run(run_id=run_id) 

576 run_id = None 

577 if not run_id: 

578 run_id = self.__add_run(title=title, cases_list=cases_list, include_all=False) 

579 return run_id 

580 

581 def __close_run(self, title=None, run_id=None): 

582 """ 

583 Closes run 

584 

585 :param title: title of test run 

586 :param run_id: run id, integer 

587 :return: True or False 

588 """ 

589 retval = True 

590 try: 

591 self.__api.runs.close_run(run_id=run_id) 

592 self.___logger.debug("Test run '%s' is closed", title) 

593 except Exception as error: 

594 self.___logger.error("Can't close run. Something nasty happened.\nError%s", format_error(error)) 

595 retval = False 

596 return retval