1"""Report test results in JUnit-XML format, for use with Jenkins and build

2 integration servers.

3

4 Based on initial code from Ross Lawley.

5

6 Output conforms to

7 https://github.com/jenkinsci/xunit-plugin/blob/master/src/main/resources/org/jenkinsci/plugins/xunit/types/model/xsd/junit-10.xsd

8"""

9 import functools

10 import os

11 import platform

12 import re

13 import xml.etree.ElementTree as ET

14 from datetime import datetime

15 from typing import Callable

16 from typing import Dict

17 from typing import List

18 from typing import Match

19 from typing import Optional

20 from typing import Tuple

21 from typing import Union

22

23 import pytest

24 from _pytest import nodes

25 from _pytest import timing

26 from _pytest._code.code import ExceptionRepr

27 from _pytest._code.code import ReprFileLocation

28 from _pytest.config import Config

29 from _pytest.config import filename_arg

30 from _pytest.config.argparsing import Parser

31 from _pytest.fixtures import FixtureRequest

32 from _pytest.reports import TestReport

33 from _pytest.stash import StashKey

34 from _pytest.terminal import TerminalReporter

35

36

37 xml_key = StashKey["LogXML"]()

38

39

40 def bin_xml_escape(arg: object) -> str:

41 r"""Visually escape invalid XML characters.

42

43 For example, transforms

44 'hello\aworld\b'

45 into

46 'hello#x07world#x08'

47 Note that the #xABs are *not* XML escapes - missing the ampersand &#xAB.

48 The idea is to escape visually for the user rather than for XML itself.

49 """

50

51 def repl(matchobj: Match[str]) -> str:

52 i = ord(matchobj.group())

53 if i <= 0xFF:

54 return "#x%02X" % i

55 else:

56 return "#x%04X" % i

57

58 # The spec range of valid chars is:

59 # Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]

60 # For an unknown(?) reason, we disallow #x7F (DEL) as well.

61 illegal_xml_re = (

62 "[^\u0009\u000A\u000D\u0020-\u007E\u0080-\uD7FF\uE000-\uFFFD\u10000-\u10FFFF]"

63 )

64 return re.sub(illegal_xml_re, repl, str(arg))

65

66

67 def merge_family(left, right) -> None:

68 result = {}

69 for kl, vl in left.items():

70 for kr, vr in right.items():

71 if not isinstance(vl, list):

72 raise TypeError(type(vl))

73 result[kl] = vl + vr

74 left.update(result)

75

76

77 families = {}

78 families["_base"] = {"testcase": ["classname", "name"]}

79 families["_base_legacy"] = {"testcase": ["file", "line", "url"]}

80

81# xUnit 1.x inherits legacy attributes.

82 families["xunit1"] = families["_base"].copy()

83 merge_family(families["xunit1"], families["_base_legacy"])

84

85# xUnit 2.x uses strict base attributes.

86 families["xunit2"] = families["_base"]

87

88

89 class _NodeReporter:

90 def __init__(self, nodeid: Union[str, TestReport], xml: "LogXML") -> None:

91 self.id = nodeid

92 self.xml = xml

93 self.add_stats = self.xml.add_stats

94 self.family = self.xml.family

95 self.duration = 0.0

96 self.properties: List[Tuple[str, str]] = []

97 self.nodes: List[ET.Element] = []

98 self.attrs: Dict[str, str] = {}

99

100 def append(self, node: ET.Element) -> None:

101 self.xml.add_stats(node.tag)

102 self.nodes.append(node)

103

104 def add_property(self, name: str, value: object) -> None:

105 self.properties.append((str(name), bin_xml_escape(value)))

106

107 def add_attribute(self, name: str, value: object) -> None:

108 self.attrs[str(name)] = bin_xml_escape(value)

109

110 def make_properties_node(self) -> Optional[ET.Element]:

111 """Return a Junit node containing custom properties, if any."""

112 if self.properties:

113 properties = ET.Element("properties")

114 for name, value in self.properties:

115 properties.append(ET.Element("property", name=name, value=value))

116 return properties

117 return None

118

119 def record_testreport(self, testreport: TestReport) -> None:

120 names = mangle_test_address(testreport.nodeid)

121 existing_attrs = self.attrs

122 classnames = names[:-1]

123 if self.xml.prefix:

124 classnames.insert(0, self.xml.prefix)

125 attrs: Dict[str, str] = {

126 "classname": ".".join(classnames),

127 "name": bin_xml_escape(names[-1]),

128 "file": testreport.location[0],

129 }

130 if testreport.location[1] is not None:

131 attrs["line"] = str(testreport.location[1])

132 if hasattr(testreport, "url"):

133 attrs["url"] = testreport.url

134 self.attrs = attrs

135 self.attrs.update(existing_attrs) # Restore any user-defined attributes.

136

137 # Preserve legacy testcase behavior.

138 if self.family == "xunit1":

139 return

140

141 # Filter out attributes not permitted by this test family.

142 # Including custom attributes because they are not valid here.

143 temp_attrs = {}

144 for key in self.attrs.keys():

145 if key in families[self.family]["testcase"]:

146 temp_attrs[key] = self.attrs[key]

147 self.attrs = temp_attrs

148

149 def to_xml(self) -> ET.Element:

150 testcase = ET.Element("testcase", self.attrs, time="%.3f" % self.duration)

151 properties = self.make_properties_node()

152 if properties is not None:

153 testcase.append(properties)

154 testcase.extend(self.nodes)

155 return testcase

156

157 def _add_simple(self, tag: str, message: str, data: Optional[str] = None) -> None:

158 node = ET.Element(tag, message=message)

159 node.text = bin_xml_escape(data)

160 self.append(node)

161

162 def write_captured_output(self, report: TestReport) -> None:

163 if not self.xml.log_passing_tests and report.passed:

164 return

165

166 content_out = report.capstdout

167 content_log = report.caplog

168 content_err = report.capstderr

169 if self.xml.logging == "no":

170 return

171 content_all = ""

172 if self.xml.logging in ["log", "all"]:

173 content_all = self._prepare_content(content_log, " Captured Log ")

174 if self.xml.logging in ["system-out", "out-err", "all"]:

175 content_all += self._prepare_content(content_out, " Captured Out ")

176 self._write_content(report, content_all, "system-out")

177 content_all = ""

178 if self.xml.logging in ["system-err", "out-err", "all"]:

179 content_all += self._prepare_content(content_err, " Captured Err ")

180 self._write_content(report, content_all, "system-err")

181 content_all = ""

182 if content_all:

183 self._write_content(report, content_all, "system-out")

184

185 def _prepare_content(self, content: str, header: str) -> str:

186 return "\n".join([header.center(80, "-"), content, ""])

187

188 def _write_content(self, report: TestReport, content: str, jheader: str) -> None:

189 tag = ET.Element(jheader)

190 tag.text = bin_xml_escape(content)

191 self.append(tag)

192

193 def append_pass(self, report: TestReport) -> None:

194 self.add_stats("passed")

195

196 def append_failure(self, report: TestReport) -> None:

197 # msg = str(report.longrepr.reprtraceback.extraline)

198 if hasattr(report, "wasxfail"):

199 self._add_simple("skipped", "xfail-marked test passes unexpectedly")

200 else:

201 assert report.longrepr is not None

202 reprcrash: Optional[ReprFileLocation] = getattr(

203 report.longrepr, "reprcrash", None

204 )

205 if reprcrash is not None:

206 message = reprcrash.message

207 else:

208 message = str(report.longrepr)

209 message = bin_xml_escape(message)

210 self._add_simple("failure", message, str(report.longrepr))

211

212 def append_collect_error(self, report: TestReport) -> None:

213 # msg = str(report.longrepr.reprtraceback.extraline)

214 assert report.longrepr is not None

215 self._add_simple("error", "collection failure", str(report.longrepr))

216

217 def append_collect_skipped(self, report: TestReport) -> None:

218 self._add_simple("skipped", "collection skipped", str(report.longrepr))

219

220 def append_error(self, report: TestReport) -> None:

221 assert report.longrepr is not None

222 reprcrash: Optional[ReprFileLocation] = getattr(

223 report.longrepr, "reprcrash", None

224 )

225 if reprcrash is not None:

226 reason = reprcrash.message

227 else:

228 reason = str(report.longrepr)

229

230 if report.when == "teardown":

231 msg = f'failed on teardown with "{reason}"'

232 else:

233 msg = f'failed on setup with "{reason}"'

234 self._add_simple("error", bin_xml_escape(msg), str(report.longrepr))

235

236 def append_skipped(self, report: TestReport) -> None:

237 if hasattr(report, "wasxfail"):

238 xfailreason = report.wasxfail

239 if xfailreason.startswith("reason: "):

240 xfailreason = xfailreason[8:]

241 xfailreason = bin_xml_escape(xfailreason)

242 skipped = ET.Element("skipped", type="pytest.xfail", message=xfailreason)

243 self.append(skipped)

244 else:

245 assert isinstance(report.longrepr, tuple)

246 filename, lineno, skipreason = report.longrepr

247 if skipreason.startswith("Skipped: "):

248 skipreason = skipreason[9:]

249 details = f"{filename}:{lineno}: {skipreason}"

250

251 skipped = ET.Element("skipped", type="pytest.skip", message=skipreason)

252 skipped.text = bin_xml_escape(details)

253 self.append(skipped)

254 self.write_captured_output(report)

255

256 def finalize(self) -> None:

257 data = self.to_xml()

258 self.__dict__.clear()

259 # Type ignored because mypy doesn't like overriding a method.

260 # Also the return value doesn't match...

261 self.to_xml = lambda: data # type: ignore[assignment]

262

263

264 def _warn_incompatibility_with_xunit2(

265 request: FixtureRequest, fixture_name: str

266) -> None:

267 """Emit a PytestWarning about the given fixture being incompatible with newer xunit revisions."""

268 from _pytest.warning_types import PytestWarning

269

270 xml = request.config.stash.get(xml_key, None)

271 if xml is not None and xml.family not in ("xunit1", "legacy"):

272 request.node.warn(

273 PytestWarning(

274 "{fixture_name} is incompatible with junit_family '{family}' (use 'legacy' or 'xunit1')".format(

275 fixture_name=fixture_name, family=xml.family

276 )

277 )

278 )

279

280

281@pytest.fixture

282 def record_property(request: FixtureRequest) -> Callable[[str, object], None]:

283 """Add extra properties to the calling test.

284

285 User properties become part of the test report and are available to the

286 configured reporters, like JUnit XML.

287

288 The fixture is callable with ``name, value``. The value is automatically

289 XML-encoded.

290

291 Example::

292

293 def test_function(record_property):

294 record_property("example_key", 1)

295 """

296 _warn_incompatibility_with_xunit2(request, "record_property")

297

298 def append_property(name: str, value: object) -> None:

299 request.node.user_properties.append((name, value))

300

301 return append_property

302

303

304@pytest.fixture

305 def record_xml_attribute(request: FixtureRequest) -> Callable[[str, object], None]:

306 """Add extra xml attributes to the tag for the calling test.

307

308 The fixture is callable with ``name, value``. The value is

309 automatically XML-encoded.

310 """

311 from _pytest.warning_types import PytestExperimentalApiWarning

312

313 request.node.warn(

314 PytestExperimentalApiWarning("record_xml_attribute is an experimental feature")

315 )

316

317 _warn_incompatibility_with_xunit2(request, "record_xml_attribute")

318

319 # Declare noop

320 def add_attr_noop(name: str, value: object) -> None:

321 pass

322

323 attr_func = add_attr_noop

324

325 xml = request.config.stash.get(xml_key, None)

326 if xml is not None:

327 node_reporter = xml.node_reporter(request.node.nodeid)

328 attr_func = node_reporter.add_attribute

329

330 return attr_func

331

332

333 def _check_record_param_type(param: str, v: str) -> None:

334 """Used by record_testsuite_property to check that the given parameter name is of the proper

335 type."""

336 __tracebackhide__ = True

337 if not isinstance(v, str):

338 msg = "{param} parameter needs to be a string, but {g} given" # type: ignore[unreachable]

339 raise TypeError(msg.format(param=param, g=type(v).__name__))

340

341

342@pytest.fixture(scope="session")

343 def record_testsuite_property(request: FixtureRequest) -> Callable[[str, object], None]:

344 """Record a new ``<property>`` tag as child of the root ``<testsuite>``.

345

346 This is suitable to writing global information regarding the entire test

347 suite, and is compatible with ``xunit2`` JUnit family.

348

349 This is a ``session``-scoped fixture which is called with ``(name, value)``. Example:

350

351 .. code-block:: python

352

353 def test_foo(record_testsuite_property):

354 record_testsuite_property("ARCH", "PPC")

355 record_testsuite_property("STORAGE_TYPE", "CEPH")

356

357 :param name:

358 The property name.

359 :param value:

360 The property value. Will be converted to a string.

361

362 .. warning::

363

364 Currently this fixture **does not work** with the

365 `pytest-xdist <https://github.com/pytest-dev/pytest-xdist>`__ plugin. See

366 :issue:`7767` for details.

367 """

368

369 __tracebackhide__ = True

370

371 def record_func(name: str, value: object) -> None:

372 """No-op function in case --junitxml was not passed in the command-line."""

373 __tracebackhide__ = True

374 _check_record_param_type("name", name)

375

376 xml = request.config.stash.get(xml_key, None)

377 if xml is not None:

378 record_func = xml.add_global_property # noqa

379 return record_func

380

381

382 def pytest_addoption(parser: Parser) -> None:

383 group = parser.getgroup("terminal reporting")

384 group.addoption(

385 "--junitxml",

386 "--junit-xml",

387 action="store",

388 dest="xmlpath",

389 metavar="path",

390 type=functools.partial(filename_arg, optname="--junitxml"),

391 default=None,

392 help="Create junit-xml style report file at given path",

393 )

394 group.addoption(

395 "--junitprefix",

396 "--junit-prefix",

397 action="store",

398 metavar="str",

399 default=None,

400 help="Prepend prefix to classnames in junit-xml output",

401 )

402 parser.addini(

403 "junit_suite_name", "Test suite name for JUnit report", default="pytest"

404 )

405 parser.addini(

406 "junit_logging",

407 "Write captured log messages to JUnit report: "

408 "one of no|log|system-out|system-err|out-err|all",

409 default="no",

410 )

411 parser.addini(

412 "junit_log_passing_tests",

413 "Capture log information for passing tests to JUnit report: ",

414 type="bool",

415 default=True,

416 )

417 parser.addini(

418 "junit_duration_report",

419 "Duration time to report: one of total|call",

420 default="total",

421 ) # choices=['total', 'call'])

422 parser.addini(

423 "junit_family",

424 "Emit XML for schema: one of legacy|xunit1|xunit2",

425 default="xunit2",

426 )

427

428

429 def pytest_configure(config: Config) -> None:

430 xmlpath = config.option.xmlpath

431 # Prevent opening xmllog on worker nodes (xdist).

432 if xmlpath and not hasattr(config, "workerinput"):

433 junit_family = config.getini("junit_family")

434 config.stash[xml_key] = LogXML(

435 xmlpath,

436 config.option.junitprefix,

437 config.getini("junit_suite_name"),

438 config.getini("junit_logging"),

439 config.getini("junit_duration_report"),

440 junit_family,

441 config.getini("junit_log_passing_tests"),

442 )

443 config.pluginmanager.register(config.stash[xml_key])

444

445

446 def pytest_unconfigure(config: Config) -> None:

447 xml = config.stash.get(xml_key, None)

448 if xml:

449 del config.stash[xml_key]

450 config.pluginmanager.unregister(xml)

451

452

453 def mangle_test_address(address: str) -> List[str]:

454 path, possible_open_bracket, params = address.partition("[")

455 names = path.split("::")

456 # Convert file path to dotted path.

457 names[0] = names[0].replace(nodes.SEP, ".")

458 names[0] = re.sub(r"\.py$", "", names[0])

459 # Put any params back.

460 names[-1] += possible_open_bracket + params

461 return names

462

463

464 class LogXML:

465 def __init__(

466 self,

467 logfile,

468 prefix: Optional[str],

469 suite_name: str = "pytest",

470 logging: str = "no",

471 report_duration: str = "total",

472 family="xunit1",

473 log_passing_tests: bool = True,

474 ) -> None:

475 logfile = os.path.expanduser(os.path.expandvars(logfile))

476 self.logfile = os.path.normpath(os.path.abspath(logfile))

477 self.prefix = prefix

478 self.suite_name = suite_name

479 self.logging = logging

480 self.log_passing_tests = log_passing_tests

481 self.report_duration = report_duration

482 self.family = family

483 self.stats: Dict[str, int] = dict.fromkeys(

484 ["error", "passed", "failure", "skipped"], 0

485 )

486 self.node_reporters: Dict[

487 Tuple[Union[str, TestReport], object], _NodeReporter

488 ] = {}

489 self.node_reporters_ordered: List[_NodeReporter] = []

490 self.global_properties: List[Tuple[str, str]] = []

491

492 # List of reports that failed on call but teardown is pending.

493 self.open_reports: List[TestReport] = []

494 self.cnt_double_fail_tests = 0

495

496 # Replaces convenience family with real family.

497 if self.family == "legacy":

498 self.family = "xunit1"

499

500 def finalize(self, report: TestReport) -> None:

501 nodeid = getattr(report, "nodeid", report)

502 # Local hack to handle xdist report order.

503 workernode = getattr(report, "node", None)

504 reporter = self.node_reporters.pop((nodeid, workernode))

505 if reporter is not None:

506 reporter.finalize()

507

508 def node_reporter(self, report: Union[TestReport, str]) -> _NodeReporter:

509 nodeid: Union[str, TestReport] = getattr(report, "nodeid", report)

510 # Local hack to handle xdist report order.

511 workernode = getattr(report, "node", None)

512

513 key = nodeid, workernode

514

515 if key in self.node_reporters:

516 # TODO: breaks for --dist=each

517 return self.node_reporters[key]

518

519 reporter = _NodeReporter(nodeid, self)

520

521 self.node_reporters[key] = reporter

522 self.node_reporters_ordered.append(reporter)

523

524 return reporter

525

526 def add_stats(self, key: str) -> None:

527 if key in self.stats:

528 self.stats[key] += 1

529

530 def _opentestcase(self, report: TestReport) -> _NodeReporter:

531 reporter = self.node_reporter(report)

532 reporter.record_testreport(report)

533 return reporter

534

535 def pytest_runtest_logreport(self, report: TestReport) -> None:

536 """Handle a setup/call/teardown report, generating the appropriate

537 XML tags as necessary.

538

539 Note: due to plugins like xdist, this hook may be called in interlaced

540 order with reports from other nodes. For example:

541

542 Usual call order:

543 -> setup node1

544 -> call node1

545 -> teardown node1

546 -> setup node2

547 -> call node2

548 -> teardown node2

549

550 Possible call order in xdist:

551 -> setup node1

552 -> call node1

553 -> setup node2

554 -> call node2

555 -> teardown node2

556 -> teardown node1

557 """

558 close_report = None

559 if report.passed:

560 if report.when == "call": # ignore setup/teardown

561 reporter = self._opentestcase(report)

562 reporter.append_pass(report)

563 elif report.failed:

564 if report.when == "teardown":

565 # The following vars are needed when xdist plugin is used.

566 report_wid = getattr(report, "worker_id", None)

567 report_ii = getattr(report, "item_index", None)

568 close_report = next(

569 (

570 rep

571 for rep in self.open_reports

572 if (

573 rep.nodeid == report.nodeid

574 and getattr(rep, "item_index", None) == report_ii

575 and getattr(rep, "worker_id", None) == report_wid

576 )

577 ),

578 None,

579 )

580 if close_report:

581 # We need to open new testcase in case we have failure in

582 # call and error in teardown in order to follow junit

583 # schema.

584 self.finalize(close_report)

585 self.cnt_double_fail_tests += 1

586 reporter = self._opentestcase(report)

587 if report.when == "call":

588 reporter.append_failure(report)

589 self.open_reports.append(report)

590 if not self.log_passing_tests:

591 reporter.write_captured_output(report)

592 else:

593 reporter.append_error(report)

594 elif report.skipped:

595 reporter = self._opentestcase(report)

596 reporter.append_skipped(report)

597 self.update_testcase_duration(report)

598 if report.when == "teardown":

599 reporter = self._opentestcase(report)

600 reporter.write_captured_output(report)

601

602 for propname, propvalue in report.user_properties:

603 reporter.add_property(propname, str(propvalue))

604

605 self.finalize(report)

606 report_wid = getattr(report, "worker_id", None)

607 report_ii = getattr(report, "item_index", None)

608 close_report = next(

609 (

610 rep

611 for rep in self.open_reports

612 if (

613 rep.nodeid == report.nodeid

614 and getattr(rep, "item_index", None) == report_ii

615 and getattr(rep, "worker_id", None) == report_wid

616 )

617 ),

618 None,

619 )

620 if close_report:

621 self.open_reports.remove(close_report)

622

623 def update_testcase_duration(self, report: TestReport) -> None:

624 """Accumulate total duration for nodeid from given report and update

625 the Junit.testcase with the new total if already created."""

626 if self.report_duration == "total" or report.when == self.report_duration:

627 reporter = self.node_reporter(report)

628 reporter.duration += getattr(report, "duration", 0.0)

629

630 def pytest_collectreport(self, report: TestReport) -> None:

631 if not report.passed:

632 reporter = self._opentestcase(report)

633 if report.failed:

634 reporter.append_collect_error(report)

635 else:

636 reporter.append_collect_skipped(report)

637

638 def pytest_internalerror(self, excrepr: ExceptionRepr) -> None:

639 reporter = self.node_reporter("internal")

640 reporter.attrs.update(classname="pytest", name="internal")

641 reporter._add_simple("error", "internal error", str(excrepr))

642

643 def pytest_sessionstart(self) -> None:

644 self.suite_start_time = timing.time()

645

646 def pytest_sessionfinish(self) -> None:

647 dirname = os.path.dirname(os.path.abspath(self.logfile))

648 # exist_ok avoids filesystem race conditions between checking path existence and requesting creation

649 os.makedirs(dirname, exist_ok=True)

650

651 with open(self.logfile, "w", encoding="utf-8") as logfile:

652 suite_stop_time = timing.time()

653 suite_time_delta = suite_stop_time - self.suite_start_time

654

655 numtests = (

656 self.stats["passed"]

657 + self.stats["failure"]

658 + self.stats["skipped"]

659 + self.stats["error"]

660 - self.cnt_double_fail_tests

661 )

662 logfile.write('<?xml version="1.0" encoding="utf-8"?>')

663

664 suite_node = ET.Element(

665 "testsuite",

666 name=self.suite_name,

667 errors=str(self.stats["error"]),

668 failures=str(self.stats["failure"]),

669 skipped=str(self.stats["skipped"]),

670 tests=str(numtests),

671 time="%.3f" % suite_time_delta,

672 timestamp=datetime.fromtimestamp(self.suite_start_time).isoformat(),

673 hostname=platform.node(),

674 )

675 global_properties = self._get_global_properties_node()

676 if global_properties is not None:

677 suite_node.append(global_properties)

678 for node_reporter in self.node_reporters_ordered:

679 suite_node.append(node_reporter.to_xml())

680 testsuites = ET.Element("testsuites")

681 testsuites.append(suite_node)

682 logfile.write(ET.tostring(testsuites, encoding="unicode"))

683

684 def pytest_terminal_summary(self, terminalreporter: TerminalReporter) -> None:

685 terminalreporter.write_sep("-", f"generated xml file: {self.logfile}")

686

687 def add_global_property(self, name: str, value: object) -> None:

688 __tracebackhide__ = True

689 _check_record_param_type("name", name)

690 self.global_properties.append((name, bin_xml_escape(value)))

691

692 def _get_global_properties_node(self) -> Optional[ET.Element]:

693 """Return a Junit node containing custom properties, if any."""

694 if self.global_properties:

695 properties = ET.Element("properties")

696 for name, value in self.global_properties:

697 properties.append(ET.Element("property", name=name, value=value))

698 return properties

699 return None