Coverage for testrail_api_reporter/engines/results_reporter.py: 12%
302 statements
« prev ^ index » next coverage.py v7.6.1, created at 2024-08-29 15:21 +0200
« prev ^ index » next coverage.py v7.6.1, created at 2024-08-29 15:21 +0200
1# -*- coding: utf-8 -*-
2""" Module for reporting results to TestRails from xml report results, obtained by pytest """
4import datetime
5from os.path import exists
7from requests.exceptions import ReadTimeout
8from testrail_api import TestRailAPI, StatusCodeError
9from xmltodict import parse
11from ..utils.logger_config import setup_logger, DEFAULT_LOGGING_LEVEL
12from ..utils.reporter_utils import format_error, init_get_cases_process
15class TestRailResultsReporter:
16 """Reporter to TestRails from xml report results, obtained by pytest"""
18 def __init__(
19 self,
20 url: str,
21 email: str,
22 password: str,
23 project_id: int,
24 xml_report="junit-report.xml",
25 suite_id=None,
26 logger=None,
27 log_level=DEFAULT_LOGGING_LEVEL,
28 ):
29 """
30 Default init
32 :param url: url of TestRail, string, required
33 :param email: email of TestRail user with proper access rights, string, required
34 :param password: password of TestRail user with proper access rights, string, required
35 :param project_id: project id, integer, required
36 :param xml_report: filename (maybe with path) of xml test report
37 :param suite_id: suite id, integer, optional, if no suite-management is activated
38 :param logger: logger object, optional
39 :param log_level: logging level, optional, by default is 'logging.DEBUG'
40 """
41 if not logger:
42 self.___logger = setup_logger(
43 name="TestRailResultsReporter", log_file="TestRailResultsReporter.log", level=log_level
44 )
45 else:
46 self.___logger = logger
47 self.___logger.debug("Initializing TestRail Results Reporter")
48 if url is None or email is None or password is None:
49 raise ValueError("No TestRails credentials are provided!")
50 self.__api = TestRailAPI(url, email, password)
51 self.__xml_report = xml_report if self.__check_report_exists(xml_report=xml_report) else None
52 self.__project_id = project_id if self.__check_project(project_id=project_id) else None
53 self.__suite_id = suite_id if self.__check_suite(suite_id=suite_id) else None
54 self.__at_section = self.__ensure_automation_section() if self.__project_id else None
55 self.__check_section(section_id=self.__at_section)
56 self.__timestamp = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
58 def __xml_to_dict(self, filename="junit-report.xml"):
59 """
60 Converts xml file to python dict
61 :param filename: filename, string, maybe with path
62 :return: dict with a list of cases
63 """
64 if not self.__check_report_exists(xml_report=self.__xml_report):
65 return None
66 with open(filename, "r", encoding="utf-8") as file:
67 xml = file.read()
69 list_of_cases = []
71 parsed_xml = parse(xml)
72 self.__timestamp = parsed_xml["testsuites"]["testsuite"]["@timestamp"].split(".")[0]
74 cases = parsed_xml["testsuites"]["testsuite"]["testcase"]
75 cases = cases if isinstance(cases, list) else [cases]
76 for item in cases:
77 status = 1
78 if "failure" in item.keys():
79 status = 5
80 if "skipped" in item.keys():
81 if item["skipped"]["@type"] == "pytest.xfail":
82 status = 5
83 else:
84 status = 7
85 list_of_cases.append(
86 {
87 "automation_id": f'{item["@classname"]}.{item["@name"]}',
88 "time": item["@time"],
89 "status": status,
90 "comment": (
91 f'{item["failure"]["@message"]} : ' f'{item["failure"]["#text"]}'
92 if "failure" in item.keys()
93 else ""
94 ),
95 }
96 )
97 self.___logger.debug("Found test run at %s, found %s test results", self.__timestamp, len(list_of_cases))
98 return list_of_cases
100 @staticmethod
101 def __search_for_item(searched_value, list_to_seek, field):
102 """
103 Item seeker by value within a list of dicts
105 :param searched_value: value what we're looking for
106 :param list_to_seek: a list where we perform the search
107 :param field: field of a list dict
108 :return: element
109 """
110 for item in list_to_seek:
111 if item[field] == searched_value:
112 return item
114 return [element for element in list_to_seek if element[field] == searched_value]
116 def __ensure_automation_section(self, title="pytest"):
117 """
118 Service function, checks that special (default) placeholder for automation non-classified tests exists
120 :param title: title for default folder, string
121 :return: id of a section
122 """
123 first_run = True
124 item_id = None
125 criteria = None
126 response = None
127 while criteria is not None or first_run:
128 if first_run:
129 try:
130 response = self.__api.sections.get_sections(project_id=self.__project_id, suite_id=self.__suite_id)
131 except Exception as error:
132 self.___logger.error(
133 "Get sections failed. Please validate your settings!\nError%s", format_error(error)
134 )
135 self.__self_check()
136 return None
137 first_run = False
138 elif response["_links"]["next"] is not None: # pylint: disable=E1136
139 offset = int(response["_links"]["next"].split("&offset=")[1].split("&")[0]) # pylint: disable=E1136
140 response = self.__api.sections.get_sections(
141 project_id=self.__project_id, suite_id=self.__suite_id, offset=offset
142 )
143 sections = response["sections"]
144 for item in sections:
145 if item["name"] == title:
146 item_id = item["id"]
147 criteria = response["_links"]["next"]
148 if not item_id:
149 try:
150 item_id = self.__api.sections.add_section(
151 project_id=self.__project_id, suite_id=self.__suite_id, name=title
152 )["id"]
153 except Exception as error:
154 self.___logger.error("Can't add section. Something nasty happened.\nError%s", format_error(error))
155 self.__self_check()
156 return None
157 self.___logger.debug("No default automation folder is found, created new one with name '%s'", title)
158 return item_id
160 def __enrich_with_tc_num(self, xml_dict_list, tc_dict_list):
161 """
162 Add a test case id to case result
164 :param xml_dict_list: list of dict, with test cases, obtained from xml report
165 :param tc_dict_list: list of dict, with test cases, obtained from TestRails
166 :return: enriched list of dict with test cases
167 """
168 enriched_list = []
169 missed_tests_counter = 0
170 for item in xml_dict_list:
171 cases = self.__search_for_item(
172 searched_value=item["automation_id"], list_to_seek=tc_dict_list, field="custom_automation_id"
173 )
174 if not cases:
175 try:
176 cases = [
177 self.__api.cases.add_case(
178 section_id=self.__at_section,
179 title=item["automation_id"],
180 custom_automation_id=item["automation_id"],
181 )
182 ]
183 except Exception as error:
184 self.___logger.error(
185 "Add case failed. Please validate your settings!\nError: %s", format_error(error)
186 )
187 self.__self_check()
188 return None
189 missed_tests_counter = missed_tests_counter + 1
190 cases = cases if isinstance(cases, list) else [cases]
191 for case in cases:
192 comment = item["message"] if "failure" in item.keys() else ""
193 elapsed = item["time"].split(".")[0]
194 elapsed = 1 if elapsed == 0 else elapsed
195 enriched_list.append(
196 {
197 "case_id": case["id"], # type: ignore
198 "status_id": item["status"],
199 "comment": comment,
200 "elapsed": elapsed,
201 "attachments": [],
202 }
203 )
204 if missed_tests_counter:
205 self.___logger.debug("Missed %s test cases, they was automatically created", missed_tests_counter)
206 self.___logger.debug("Found %s test cases in TestRails", len(enriched_list))
207 return enriched_list
209 def ___handle_read_timeout(self, retry, retries, error):
210 if retry < retries:
211 retry += 1
212 self.___logger.debug("Timeout error, retrying %s/%s...", retry, retries)
213 return retry, True
214 raise ValueError(f"Get cases failed. Please validate your settings!nError{format_error(error)}") from error
216 # pylint: disable=R0912
217 def __get_all_auto_cases(self, retries=3):
218 """
219 Collects all test cases from TestRails with non-empty automation_id
221 :param retries: number of retries, integer
222 :return: list of dict with cases
223 """
224 cases_list, first_run, criteria, response, retry = init_get_cases_process()
225 while criteria is not None or first_run:
226 if first_run:
227 try:
228 response = self.__api.cases.get_cases(project_id=self.__project_id, suite_id=self.__suite_id)
229 except (ReadTimeout, StatusCodeError) as error:
230 if (
231 isinstance(error, StatusCodeError)
232 and error.status_code == 504 # type: ignore # pylint: disable=no-member
233 or isinstance(error, ReadTimeout)
234 ):
235 retry, should_continue = self.___handle_read_timeout(retry, retries, error)
236 if should_continue:
237 continue
238 except Exception as error:
239 self.___logger.error(
240 "Get cases failed. Please validate your settings!\nError%s", format_error(error)
241 )
242 self.__self_check()
243 return None
244 first_run = False
245 retry = 0
246 elif response["_links"]["next"] is not None: # pylint: disable=E1136
247 offset = int(response["_links"]["next"].split("&offset=")[1].split("&")[0]) # pylint: disable=E1136
248 try:
249 response = self.__api.cases.get_cases(
250 project_id=self.__project_id, suite_id=self.__suite_id, offset=offset
251 )
252 except (ReadTimeout, StatusCodeError) as error:
253 if (
254 isinstance(error, StatusCodeError)
255 and error.status_code == 504 # type: ignore # pylint: disable=no-member
256 or isinstance(error, ReadTimeout)
257 ):
258 retry, should_continue = self.___handle_read_timeout(retry, retries, error)
259 if should_continue:
260 continue
261 retry = 0
262 cases = response["cases"]
263 for item in cases:
264 if item["custom_automation_id"] is not None:
265 cases_list.append({"id": item["id"], "custom_automation_id": item["custom_automation_id"]})
266 criteria = response["_links"]["next"]
267 self.___logger.debug("Found %s test cases in TestRails with automation_id", len(cases_list))
268 return cases_list
270 def __prepare_payload(self):
271 """
272 Prepares payload from xml report for sending to TestRails
274 :return: payload in proper format (list of dicts)
275 """
276 parsed_xml = self.__xml_to_dict(filename=self.__xml_report)
277 parsed_cases = self.__get_all_auto_cases()
278 if not parsed_xml:
279 self.___logger.error("Preparation of payload failed, aborted")
280 return None
281 payload = self.__enrich_with_tc_num(xml_dict_list=parsed_xml, tc_dict_list=parsed_cases)
282 return payload
284 def __prepare_title(self, environment=None, timestamp=None):
285 """
286 Format test run name based on input string (most probably environment) and timestamp
288 :param environment: some string run identifier
289 :param timestamp: custom timestamp
290 :return: string of prepared string for AT run name
291 """
292 if timestamp is None:
293 timestamp = self.__timestamp
294 title = f"AT run {timestamp}"
295 if environment:
296 title = f"{title} on {environment}"
297 return title
299 def send_results(
300 self,
301 run_id=None,
302 environment=None,
303 title=None,
304 timestamp=None,
305 close_run=True,
306 run_name=None,
307 delete_old_run=False,
308 ):
309 """
310 Send results to TestRail
312 :param run_id: specific run id, if any, optional
313 :param environment: custom name pattern for run name, optional
314 :param title: custom title, if provided, will be used as priority, optional
315 :param timestamp: custom timestamp, optional
316 :param close_run: close or not run, True or False
317 :param run_name: name of test run, will be used if provided at top priority
318 :param delete_old_run: delete or not previous run if old one exists with the same name
319 :return: run id where results were submitted
320 """
321 if (
322 not self.__project_id
323 or not self.__at_section
324 or not self.__check_report_exists(xml_report=self.__xml_report)
325 ):
326 self.___logger.error("Error! Please specify all required params!")
327 self.__self_check()
328 return True
329 title = self.__prepare_title(environment, timestamp) if not title else title
330 title = run_name if run_name else title
331 payload = self.__prepare_payload()
332 run_id = self.__prepare_runs(
333 cases=payload, title=title, run_id=run_id, run_name=run_name, delete_run=delete_old_run
334 )
335 retval = self.__add_results(run_id=run_id, results=payload)
336 if close_run:
337 self.__close_run(run_id=run_id, title=title)
338 self.___logger.debug("%s results were added to test run '%s', cases updated. Done", len(payload), title)
339 return retval
341 def set_project_id(self, project_id):
342 """
343 Set project id
345 :param project_id: project id, integer
346 """
347 self.__project_id = project_id if self.__check_project(project_id=project_id) else None
349 def set_suite_id(self, suite_id):
350 """
351 Set suite id
353 :param suite_id: suite id, integer
354 """
355 if self.__check_project():
356 self.__suite_id = suite_id if self.__check_suite(suite_id=suite_id) else None
358 def set_xml_filename(self, xml_filename):
359 """
360 Set xml filename
362 :param xml_filename: filename of xml report, string
363 """
364 self.__xml_report = xml_filename if self.__check_report_exists(xml_report=xml_filename) else None
366 def set_at_report_section(self, section_name):
367 """
368 Set section name for AT a report
370 :param section_name: name of a section, string
371 """
372 if self.__check_project() and self.__check_suite():
373 self.__at_section = self.__ensure_automation_section(title=section_name)
375 def set_timestamp(self, new_timestamp):
376 """
377 Set timestamp
379 :param new_timestamp: timestamp, string
380 """
381 self.__timestamp = new_timestamp
383 def __check_project(self, project_id=None):
384 """
385 Check that the project exists
387 :param project_id: project id, integer
388 :return: True or False
389 """
390 retval = True
391 try:
392 self.__api.projects.get_project(project_id=project_id)
393 except Exception as error:
394 self.___logger.error("No such project is found, please set valid project ID.\nError%s", format_error(error))
395 retval = False
396 return retval
398 def __check_suite(self, suite_id=None):
399 """
400 Check that the suite exists
402 :param suite_id: id of suite, integer
403 :return: True or False
404 """
405 retval = True
406 try:
407 self.__api.suites.get_suite(suite_id=suite_id)
408 except Exception as error:
409 self.___logger.error("No such suite is found, please set valid suite ID.\nError%s", format_error(error))
410 retval = False
411 return retval
413 def __check_section(self, section_id=None):
414 """
415 Check that the section exists
417 :param section_id: id of suite, integer
418 :return: True or False
419 """
420 retval = True
421 try:
422 self.__api.sections.get_section(section_id=section_id)
423 except Exception as error:
424 self.___logger.error("No such section is found, please set valid section ID.\nError%s", format_error(error))
425 retval = False
426 return retval
428 def __check_report_exists(self, xml_report=None):
429 """
430 Check that the xml report exists
432 :param xml_report: filename of the xml report, maybe with path
433 :return: True or False
434 """
435 retval = False
436 if not xml_report:
437 xml_report = self.__xml_report
438 if xml_report:
439 if exists(xml_report):
440 retval = True
441 if not retval:
442 self.___logger.error("Please specify correct path.\nError 404: No XML file found")
443 return retval
445 def __check_run_exists(self, run_id=None):
446 """
447 Check that the run exists
449 :param run_id: id of suite, integer
450 :return: True or False
451 """
452 retval = True
453 try:
454 self.__api.runs.get_run(run_id=run_id)
455 except Exception as error:
456 self.___logger.error("No such run is found, please set valid run ID.\nError%s", format_error(error))
457 retval = False
458 return retval
460 def __self_check(self):
461 """
462 Health checker, calls checks
463 """
464 self.__check_project(project_id=self.__project_id)
465 self.__check_suite(suite_id=self.__suite_id)
466 self.__check_section(section_id=self.__at_section)
467 self.__check_report_exists(xml_report=self.__xml_report)
469 def __search_for_run_by_name(self, title=None):
470 """
471 Search run by name
473 :param title: name of the run
474 :return: id, integer
475 """
476 retval = None
477 first_run = True
478 criteria = None
479 while criteria is not None or first_run:
480 response = None
481 if first_run:
482 try:
483 response = self.__api.runs.get_runs(project_id=self.__project_id, suite_id=self.__suite_id)
484 except Exception as error:
485 self.___logger.error("Can't get run list. Something nasty happened.\nError%s", format_error(error))
486 break
487 first_run = False
488 elif response["_links"]["next"] is not None: # pylint: disable=E1136
489 offset = int(response["_links"]["next"].split("&offset=")[1].split("&")[0]) # pylint: disable=E1136
490 response = self.__api.runs.get_runs(
491 project_id=self.__project_id, suite_id=self.__suite_id, offset=offset
492 )
493 if response["runs"]["name"] == title:
494 retval = response["runs"]["id"]
495 break
496 criteria = response["_links"]["next"]
497 return retval
499 def __delete_run(self, run_id=None):
500 """
501 Delete run
503 :param run_id: run id, integer
504 :return: True if deleted, False in case of error
505 """
506 retval = True
507 try:
508 self.__api.runs.delete_run(run_id=run_id)
509 except Exception as error:
510 self.___logger.error("Can't delete run. Something nasty happened.\nError%s", format_error(error))
511 retval = False
512 return retval
514 def __add_run(self, title, cases_list=None, include_all=False):
515 """
516 Add a run
518 :param title: title of run
519 :param cases_list: test cases, which will be added to run
520 :param include_all: every existing testcases may be included, by default - False
521 :return: id of run, integer
522 """
523 retval = None
524 self.___logger.debug("Creating new test run '%s'", title)
525 try:
526 retval = self.__api.runs.add_run(
527 project_id=self.__project_id,
528 suite_id=self.__suite_id,
529 name=title,
530 include_all=include_all,
531 case_ids=cases_list,
532 )["id"]
533 except Exception as error:
534 self.___logger.error("Can't add run. Something nasty happened.\nError%s", format_error(error))
535 self.__self_check()
536 return retval
538 def __add_results(self, run_id=None, results=None):
539 """
540 Add results for test cases to TestRail
542 :param run_id: run id
543 :param results: payload (list of dicts)
544 :return: run id or False in case of error
545 """
546 retval = False
547 try:
548 self.__api.results.add_results_for_cases(run_id=run_id, results=results)
549 return run_id
550 except Exception as error:
551 self.___logger.error("Add results failed. Please validate your settings!\nError%s", format_error(error))
552 self.__self_check()
553 self.__check_run_exists(run_id=run_id)
554 return retval
556 def __prepare_runs(self, cases=None, title=None, run_id=None, run_name=None, delete_run=False):
557 """
558 Prepare run for submitting
560 :param cases: list of cases (list of dicts)
561 :param title: title of test run (which will be submitted)
562 :param run_id: run id
563 :param run_name:
564 :param delete_run: delete existing run or not (True or False), will be checked via run_name
565 :return: run id
566 """
567 cases_list = []
568 for item in cases:
569 cases_list.append(item["case_id"])
570 if run_name:
571 run_id = self.__search_for_run_by_name(title=run_name)
572 if not run_id:
573 self.___logger.debug("No run has been found by given name")
574 if delete_run and run_id:
575 self.__delete_run(run_id=run_id)
576 run_id = None
577 if not run_id:
578 run_id = self.__add_run(title=title, cases_list=cases_list, include_all=False)
579 return run_id
581 def __close_run(self, title=None, run_id=None):
582 """
583 Closes run
585 :param title: title of test run
586 :param run_id: run id, integer
587 :return: True or False
588 """
589 retval = True
590 try:
591 self.__api.runs.close_run(run_id=run_id)
592 self.___logger.debug("Test run '%s' is closed", title)
593 except Exception as error:
594 self.___logger.error("Can't close run. Something nasty happened.\nError%s", format_error(error))
595 retval = False
596 return retval