Coverage for Applications/PyCharm.app/Contents/plugins/python/helpers/pycharm/teamcity/pytest_plugin.py: 43%
294 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-12 16:26 -0700
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-12 16:26 -0700
1# coding=utf-8
2"""
3Aaron Buchanan
4Nov. 2012
6Plug-in for py.test for reporting to TeamCity server
7Report results to TeamCity during test execution for immediate reporting
8 when using TeamCity.
10This should be installed as a py.test plugin and will be automatically enabled by running
11tests under TeamCity build.
12"""
14import os
15import re
16import sys
17import traceback
18from datetime import timedelta
20from teamcity import diff_tools
21from teamcity import is_running_under_teamcity
22from teamcity.common import convert_error_to_string, dump_test_stderr, dump_test_stdout
23from teamcity.messages import TeamcityServiceMessages
25diff_tools.patch_unittest_diff()
26_ASSERTION_FAILURE_KEY = '_teamcity_assertion_failure'
29def pytest_addoption(parser):
30 group = parser.getgroup("terminal reporting", "reporting", after="general")
32 group._addoption('--teamcity', action="count",
33 dest="teamcity", default=0, help="force output of JetBrains TeamCity service messages")
34 group._addoption('--no-teamcity', action="count",
35 dest="no_teamcity", default=0, help="disable output of JetBrains TeamCity service messages")
36 parser.addoption('--jb-swapdiff', action="store_true", dest="swapdiff", default=False, help="Swap actual/expected in diff")
38 kwargs = {"help": "skip output of passed tests for JetBrains TeamCity service messages"}
39 kwargs.update({"type": "bool"})
41 parser.addini("skippassedoutput", **kwargs)
42 parser.addini("swapdiff", **kwargs)
45def pytest_configure(config):
46 if config.option.no_teamcity >= 1:
47 enabled = False
48 elif config.option.teamcity >= 1:
49 enabled = True
50 else:
51 enabled = is_running_under_teamcity()
53 if enabled:
54 output_capture_enabled = getattr(config.option, 'capture', 'fd') != 'no'
55 coverage_controller = _get_coverage_controller(config)
56 skip_passed_output = bool(config.getini('skippassedoutput'))
58 config.option.verbose = 2 # don't truncate assert explanations
59 config._teamcityReporting = EchoTeamCityMessages(
60 output_capture_enabled,
61 coverage_controller,
62 skip_passed_output,
63 bool(config.getini('swapdiff') or config.option.swapdiff)
64 )
65 config.pluginmanager.register(config._teamcityReporting)
68def pytest_unconfigure(config):
69 teamcity_reporting = getattr(config, '_teamcityReporting', None)
70 if teamcity_reporting:
71 del config._teamcityReporting
72 config.pluginmanager.unregister(teamcity_reporting)
75def _get_coverage_controller(config):
76 cov_plugin = config.pluginmanager.getplugin('_cov')
77 if not cov_plugin:
78 return None
80 return cov_plugin.cov_controller
83class EchoTeamCityMessages(object):
84 def __init__(self, output_capture_enabled, coverage_controller, skip_passed_output, swap_diff):
85 self.coverage_controller = coverage_controller
86 self.output_capture_enabled = output_capture_enabled
87 self.skip_passed_output = skip_passed_output
89 self.teamcity = TeamcityServiceMessages()
90 self.test_start_reported_mark = set()
91 self.current_test_item = None
93 self.max_reported_output_size = 1 * 1024 * 1024
94 self.reported_output_chunk_size = 50000
95 self.swap_diff = swap_diff
97 def get_id_from_location(self, location):
98 if type(location) is not tuple or len(location) != 3 or not hasattr(location[2], "startswith"):
99 return None
101 def convert_file_to_id(filename):
102 filename = re.sub(r"\.pyc?$", "", filename)
103 return filename.replace(os.sep, ".").replace("/", ".")
105 def add_prefix_to_filename_id(filename_id, prefix):
106 dot_location = filename_id.rfind('.')
107 if dot_location <= 0 or dot_location >= len(filename_id) - 1:
108 return None
110 return filename_id[:dot_location + 1] + prefix + filename_id[dot_location + 1:]
112 pylint_prefix = '[pylint] '
113 if location[2].startswith(pylint_prefix):
114 id_from_file = convert_file_to_id(location[2][len(pylint_prefix):])
115 return id_from_file + ".Pylint"
117 if location[2] == "PEP8-check":
118 id_from_file = convert_file_to_id(location[0])
119 return id_from_file + ".PEP8"
121 return None
123 def format_test_id(self, nodeid, location):
124 id_from_location = self.get_id_from_location(location)
126 if id_from_location is not None:
127 return id_from_location
129 test_id = nodeid
131 if test_id:
132 if test_id.find("::") < 0:
133 test_id += "::top_level"
134 else:
135 test_id = "top_level"
137 first_bracket = test_id.find("[")
138 if first_bracket > 0:
139 # [] -> (), make it look like nose parameterized tests
140 params = "(" + test_id[first_bracket + 1:]
141 if params.endswith("]"):
142 params = params[:-1] + ")"
143 test_id = test_id[:first_bracket]
144 if test_id.endswith("::"):
145 test_id = test_id[:-2]
146 else:
147 params = ""
149 test_id = test_id.replace("::()::", "::")
150 test_id = re.sub(r"\.pyc?::", r"::", test_id)
151 test_id = test_id.replace(".", "_").replace(os.sep, ".").replace("/", ".").replace('::', '.')
153 if params:
154 params = params.replace(".", "_")
155 test_id += params
157 return test_id
159 def format_location(self, location):
160 if type(location) is tuple and len(location) == 3:
161 return "%s:%s (%s)" % (str(location[0]), str(location[1]), str(location[2]))
162 return str(location)
164 def pytest_collection_finish(self, session):
165 self.teamcity.testCount(len(session.items))
167 def pytest_runtest_logstart(self, nodeid, location):
168 # test name fetched from location passed as metainfo to PyCharm
169 # it will be used to run specific test
170 # See IDEA-176950, PY-31836
171 test_name = location[2]
172 if test_name:
173 test_name = str(test_name).split(".")[-1]
174 self.ensure_test_start_reported(self.format_test_id(nodeid, location), test_name)
176 def pytest_runtest_protocol(self, item):
177 self.current_test_item = item
178 return None # continue to next hook
180 def ensure_test_start_reported(self, test_id, metainfo=None):
181 if test_id not in self.test_start_reported_mark:
182 if self.output_capture_enabled:
183 capture_standard_output = "false"
184 else:
185 capture_standard_output = "true"
186 self.teamcity.testStarted(test_id, flowId=test_id, captureStandardOutput=capture_standard_output, metainfo=metainfo)
187 self.test_start_reported_mark.add(test_id)
189 def report_has_output(self, report):
190 for (secname, data) in report.sections:
191 if report.when in secname and ('stdout' in secname or 'stderr' in secname):
192 return True
193 return False
195 def report_test_output(self, report, test_id):
196 for (secname, data) in report.sections:
197 # https://github.com/JetBrains/teamcity-messages/issues/112
198 # CollectReport didn't have 'when' property, but now it has.
199 # But we still need output on 'collect' state
200 if hasattr(report, "when") and report.when not in secname and report.when != 'collect':
201 continue
202 if not data:
203 continue
205 if 'stdout' in secname:
206 dump_test_stdout(self.teamcity, test_id, test_id, data)
207 elif 'stderr' in secname:
208 dump_test_stderr(self.teamcity, test_id, test_id, data)
210 def report_test_finished(self, test_id, duration=None):
211 self.teamcity.testFinished(test_id, testDuration=duration, flowId=test_id)
212 self.test_start_reported_mark.remove(test_id)
214 def report_test_failure(self, test_id, report, message=None, report_output=True):
215 if hasattr(report, 'duration'):
216 duration = timedelta(seconds=report.duration)
217 else:
218 duration = None
220 if message is None:
221 message = self.format_location(report.location)
223 self.ensure_test_start_reported(test_id)
224 if report_output:
225 self.report_test_output(report, test_id)
227 diff_error = None
228 try:
229 err_message = str(report.longrepr.reprcrash.message)
230 diff_name = diff_tools.EqualsAssertionError.__name__
231 # There is a string like "foo.bar.DiffError: [serialized_data]"
232 if diff_name in err_message:
233 serialized_data = err_message[err_message.index(diff_name) + len(diff_name) + 1:]
234 diff_error = diff_tools.deserialize_error(serialized_data)
236 assertion_tuple = getattr(self.current_test_item, _ASSERTION_FAILURE_KEY, None)
237 if assertion_tuple:
238 op, left, right = assertion_tuple
239 if self.swap_diff:
240 left, right = right, left
241 diff_error = diff_tools.EqualsAssertionError(expected=right, actual=left)
242 except Exception:
243 pass
245 if not diff_error:
246 from .jb_local_exc_store import get_exception
247 diff_error = get_exception()
249 if diff_error:
250 # Cut everything after postfix: it is internal view of DiffError
251 strace = str(report.longrepr)
252 data_postfix = "_ _ _ _ _"
253 # Error message in pytest must be in "file.py:22 AssertionError" format
254 # This message goes to strace
255 # With custom error we must add real exception class explicitly
256 if data_postfix in strace:
257 strace = strace[0:strace.index(data_postfix)].strip()
258 if strace.endswith(":") and diff_error.real_exception:
259 strace += " " + type(diff_error.real_exception).__name__
260 self.teamcity.testFailed(test_id, diff_error.msg or message, strace,
261 flowId=test_id,
262 comparison_failure=diff_error
263 )
264 else:
265 self.teamcity.testFailed(test_id, message, str(report.longrepr), flowId=test_id)
266 self.report_test_finished(test_id, duration)
268 def report_test_skip(self, test_id, report):
269 if type(report.longrepr) is tuple and len(report.longrepr) == 3:
270 reason = report.longrepr[2]
271 else:
272 reason = str(report.longrepr)
274 if hasattr(report, 'duration'):
275 duration = timedelta(seconds=report.duration)
276 else:
277 duration = None
279 self.ensure_test_start_reported(test_id)
280 self.report_test_output(report, test_id)
281 self.teamcity.testIgnored(test_id, reason, flowId=test_id)
282 self.report_test_finished(test_id, duration)
284 def pytest_assertrepr_compare(self, config, op, left, right):
285 setattr(self.current_test_item, _ASSERTION_FAILURE_KEY, (op, left, right))
287 def pytest_runtest_logreport(self, report):
288 """
289 :type report: _pytest.runner.TestReport
290 """
291 test_id = self.format_test_id(report.nodeid, report.location)
293 duration = timedelta(seconds=report.duration)
295 if report.passed:
296 # Do not report passed setup/teardown if no output
297 if report.when == 'call':
298 self.ensure_test_start_reported(test_id)
299 if not self.skip_passed_output:
300 self.report_test_output(report, test_id)
301 self.report_test_finished(test_id, duration)
302 else:
303 if self.report_has_output(report) and not self.skip_passed_output:
304 block_name = "test " + report.when
305 self.teamcity.blockOpened(block_name, flowId=test_id)
306 self.report_test_output(report, test_id)
307 self.teamcity.blockClosed(block_name, flowId=test_id)
308 elif report.failed:
309 if report.when == 'call':
310 self.report_test_failure(test_id, report)
311 elif report.when == 'setup':
312 if self.report_has_output(report):
313 self.teamcity.blockOpened("test setup", flowId=test_id)
314 self.report_test_output(report, test_id)
315 self.teamcity.blockClosed("test setup", flowId=test_id)
317 self.report_test_failure(test_id, report, message="test setup failed", report_output=False)
318 elif report.when == 'teardown':
319 # Report failed teardown as a separate test as original test is already finished
320 self.report_test_failure(test_id + "_teardown", report)
321 elif report.skipped:
322 self.report_test_skip(test_id, report)
324 def pytest_collectreport(self, report):
325 test_id = self.format_test_id(report.nodeid, report.location) + "_collect"
327 if report.failed:
328 self.report_test_failure(test_id, report)
329 elif report.skipped:
330 self.report_test_skip(test_id, report)
332 def pytest_terminal_summary(self):
333 if self.coverage_controller is not None:
334 try:
335 self._report_coverage()
336 except Exception:
337 tb = traceback.format_exc()
338 self.teamcity.customMessage("Coverage statistics reporting failed", "ERROR", errorDetails=tb)
340 def _report_coverage(self):
341 from coverage.misc import NotPython
342 from coverage.results import Numbers
344 class _Reporter(object):
345 def __init__(self, coverage, config):
346 try:
347 from coverage.report import Reporter
348 except ImportError:
349 # Support for coverage >= 5.0.1.
350 from coverage.report import get_analysis_to_report
352 class Reporter(object):
354 def __init__(self, coverage, config):
355 self.coverage = coverage
356 self.config = config
357 self._file_reporters = []
359 def find_file_reporters(self, morfs):
360 return [fr for fr, _ in get_analysis_to_report(self.coverage, morfs)]
362 self._reporter = Reporter(coverage, config)
364 def find_file_reporters(self, morfs):
365 self.file_reporters = self._reporter.find_file_reporters(morfs)
367 def __getattr__(self, name):
368 return getattr(self._reporter, name)
370 class _CoverageReporter(_Reporter):
371 def __init__(self, coverage, config, messages):
372 super(_CoverageReporter, self).__init__(coverage, config)
374 if hasattr(coverage, 'data'):
375 self.branches = coverage.data.has_arcs()
376 else:
377 self.branches = coverage.get_data().has_arcs()
378 self.messages = messages
380 def report(self, morfs, outfile=None):
381 if hasattr(self, 'find_code_units'):
382 self.find_code_units(morfs)
383 else:
384 self.find_file_reporters(morfs)
386 total = Numbers()
388 if hasattr(self, 'code_units'):
389 units = self.code_units
390 else:
391 units = self.file_reporters
393 for cu in units:
394 try:
395 analysis = self.coverage._analyze(cu)
396 nums = analysis.numbers
397 total += nums
398 except KeyboardInterrupt:
399 raise
400 except Exception:
401 if self.config.ignore_errors:
402 continue
404 err = sys.exc_info()
405 typ, msg = err[:2]
406 if typ is NotPython and not cu.should_be_python():
407 continue
409 test_id = cu.name
410 details = convert_error_to_string(err)
412 self.messages.testStarted(test_id, flowId=test_id)
413 self.messages.testFailed(test_id, message="Coverage analysis failed", details=details, flowId=test_id)
414 self.messages.testFinished(test_id, flowId=test_id)
416 if total.n_files > 0:
417 covered = total.n_executed
418 total_statements = total.n_statements
420 if self.branches:
421 covered += total.n_executed_branches
422 total_statements += total.n_branches
424 self.messages.buildStatisticLinesCovered(covered)
425 self.messages.buildStatisticTotalLines(total_statements)
426 self.messages.buildStatisticLinesUncovered(total_statements - covered)
427 reporter = _CoverageReporter(
428 self.coverage_controller.cov,
429 self.coverage_controller.cov.config,
430 self.teamcity,
431 )
432 reporter.report(None)