Coverage for Applications/PyCharm.app/Contents/plugins/python/helpers/pycharm/teamcity/pytest_plugin.py: 16%

294 statements  

« prev     ^ index     » next       coverage.py v7.3.1, created at 2023-09-19 10:04 +0530

1# coding=utf-8 

2""" 

3Aaron Buchanan 

4Nov. 2012 

5 

6Plug-in for py.test for reporting to TeamCity server 

7Report results to TeamCity during test execution for immediate reporting 

8 when using TeamCity. 

9 

10This should be installed as a py.test plugin and will be automatically enabled by running 

11tests under TeamCity build. 

12""" 

13 

14import os 

15import re 

16import sys 

17import traceback 

18from datetime import timedelta 

19 

20from teamcity import diff_tools 

21from teamcity import is_running_under_teamcity 

22from teamcity.common import convert_error_to_string, dump_test_stderr, dump_test_stdout 

23from teamcity.messages import TeamcityServiceMessages 

24 

25diff_tools.patch_unittest_diff() 

26_ASSERTION_FAILURE_KEY = '_teamcity_assertion_failure' 

27 

28 

29def pytest_addoption(parser): 

30 group = parser.getgroup("terminal reporting", "reporting", after="general") 

31 

32 group._addoption('--teamcity', action="count", 

33 dest="teamcity", default=0, help="force output of JetBrains TeamCity service messages") 

34 group._addoption('--no-teamcity', action="count", 

35 dest="no_teamcity", default=0, help="disable output of JetBrains TeamCity service messages") 

36 parser.addoption('--jb-swapdiff', action="store_true", dest="swapdiff", default=False, help="Swap actual/expected in diff") 

37 

38 kwargs = {"help": "skip output of passed tests for JetBrains TeamCity service messages"} 

39 kwargs.update({"type": "bool"}) 

40 

41 parser.addini("skippassedoutput", **kwargs) 

42 parser.addini("swapdiff", **kwargs) 

43 

44 

45def pytest_configure(config): 

46 if config.option.no_teamcity >= 1: 

47 enabled = False 

48 elif config.option.teamcity >= 1: 

49 enabled = True 

50 else: 

51 enabled = is_running_under_teamcity() 

52 

53 if enabled: 

54 output_capture_enabled = getattr(config.option, 'capture', 'fd') != 'no' 

55 coverage_controller = _get_coverage_controller(config) 

56 skip_passed_output = bool(config.getini('skippassedoutput')) 

57 

58 config.option.verbose = 2 # don't truncate assert explanations 

59 config._teamcityReporting = EchoTeamCityMessages( 

60 output_capture_enabled, 

61 coverage_controller, 

62 skip_passed_output, 

63 bool(config.getini('swapdiff') or config.option.swapdiff) 

64 ) 

65 config.pluginmanager.register(config._teamcityReporting) 

66 

67 

68def pytest_unconfigure(config): 

69 teamcity_reporting = getattr(config, '_teamcityReporting', None) 

70 if teamcity_reporting: 

71 del config._teamcityReporting 

72 config.pluginmanager.unregister(teamcity_reporting) 

73 

74 

75def _get_coverage_controller(config): 

76 cov_plugin = config.pluginmanager.getplugin('_cov') 

77 if not cov_plugin: 

78 return None 

79 

80 return cov_plugin.cov_controller 

81 

82 

83class EchoTeamCityMessages(object): 

84 def __init__(self, output_capture_enabled, coverage_controller, skip_passed_output, swap_diff): 

85 self.coverage_controller = coverage_controller 

86 self.output_capture_enabled = output_capture_enabled 

87 self.skip_passed_output = skip_passed_output 

88 

89 self.teamcity = TeamcityServiceMessages() 

90 self.test_start_reported_mark = set() 

91 self.current_test_item = None 

92 

93 self.max_reported_output_size = 1 * 1024 * 1024 

94 self.reported_output_chunk_size = 50000 

95 self.swap_diff = swap_diff 

96 

97 def get_id_from_location(self, location): 

98 if type(location) is not tuple or len(location) != 3 or not hasattr(location[2], "startswith"): 

99 return None 

100 

101 def convert_file_to_id(filename): 

102 filename = re.sub(r"\.pyc?$", "", filename) 

103 return filename.replace(os.sep, ".").replace("/", ".") 

104 

105 def add_prefix_to_filename_id(filename_id, prefix): 

106 dot_location = filename_id.rfind('.') 

107 if dot_location <= 0 or dot_location >= len(filename_id) - 1: 

108 return None 

109 

110 return filename_id[:dot_location + 1] + prefix + filename_id[dot_location + 1:] 

111 

112 pylint_prefix = '[pylint] ' 

113 if location[2].startswith(pylint_prefix): 

114 id_from_file = convert_file_to_id(location[2][len(pylint_prefix):]) 

115 return id_from_file + ".Pylint" 

116 

117 if location[2] == "PEP8-check": 

118 id_from_file = convert_file_to_id(location[0]) 

119 return id_from_file + ".PEP8" 

120 

121 return None 

122 

123 def format_test_id(self, nodeid, location): 

124 id_from_location = self.get_id_from_location(location) 

125 

126 if id_from_location is not None: 

127 return id_from_location 

128 

129 test_id = nodeid 

130 

131 if test_id: 

132 if test_id.find("::") < 0: 

133 test_id += "::top_level" 

134 else: 

135 test_id = "top_level" 

136 

137 first_bracket = test_id.find("[") 

138 if first_bracket > 0: 

139 # [] -> (), make it look like nose parameterized tests 

140 params = "(" + test_id[first_bracket + 1:] 

141 if params.endswith("]"): 

142 params = params[:-1] + ")" 

143 test_id = test_id[:first_bracket] 

144 if test_id.endswith("::"): 

145 test_id = test_id[:-2] 

146 else: 

147 params = "" 

148 

149 test_id = test_id.replace("::()::", "::") 

150 test_id = re.sub(r"\.pyc?::", r"::", test_id) 

151 test_id = test_id.replace(".", "_").replace(os.sep, ".").replace("/", ".").replace('::', '.') 

152 

153 if params: 

154 params = params.replace(".", "_") 

155 test_id += params 

156 

157 return test_id 

158 

159 def format_location(self, location): 

160 if type(location) is tuple and len(location) == 3: 

161 return "%s:%s (%s)" % (str(location[0]), str(location[1]), str(location[2])) 

162 return str(location) 

163 

164 def pytest_collection_finish(self, session): 

165 self.teamcity.testCount(len(session.items)) 

166 

167 def pytest_runtest_logstart(self, nodeid, location): 

168 # test name fetched from location passed as metainfo to PyCharm 

169 # it will be used to run specific test 

170 # See IDEA-176950, PY-31836 

171 test_name = location[2] 

172 if test_name: 

173 test_name = str(test_name).split(".")[-1] 

174 self.ensure_test_start_reported(self.format_test_id(nodeid, location), test_name) 

175 

176 def pytest_runtest_protocol(self, item): 

177 self.current_test_item = item 

178 return None # continue to next hook 

179 

180 def ensure_test_start_reported(self, test_id, metainfo=None): 

181 if test_id not in self.test_start_reported_mark: 

182 if self.output_capture_enabled: 

183 capture_standard_output = "false" 

184 else: 

185 capture_standard_output = "true" 

186 self.teamcity.testStarted(test_id, flowId=test_id, captureStandardOutput=capture_standard_output, metainfo=metainfo) 

187 self.test_start_reported_mark.add(test_id) 

188 

189 def report_has_output(self, report): 

190 for (secname, data) in report.sections: 

191 if report.when in secname and ('stdout' in secname or 'stderr' in secname): 

192 return True 

193 return False 

194 

195 def report_test_output(self, report, test_id): 

196 for (secname, data) in report.sections: 

197 # https://github.com/JetBrains/teamcity-messages/issues/112 

198 # CollectReport didn't have 'when' property, but now it has. 

199 # But we still need output on 'collect' state 

200 if hasattr(report, "when") and report.when not in secname and report.when != 'collect': 

201 continue 

202 if not data: 

203 continue 

204 

205 if 'stdout' in secname: 

206 dump_test_stdout(self.teamcity, test_id, test_id, data) 

207 elif 'stderr' in secname: 

208 dump_test_stderr(self.teamcity, test_id, test_id, data) 

209 

210 def report_test_finished(self, test_id, duration=None): 

211 self.teamcity.testFinished(test_id, testDuration=duration, flowId=test_id) 

212 self.test_start_reported_mark.remove(test_id) 

213 

214 def report_test_failure(self, test_id, report, message=None, report_output=True): 

215 if hasattr(report, 'duration'): 

216 duration = timedelta(seconds=report.duration) 

217 else: 

218 duration = None 

219 

220 if message is None: 

221 message = self.format_location(report.location) 

222 

223 self.ensure_test_start_reported(test_id) 

224 if report_output: 

225 self.report_test_output(report, test_id) 

226 

227 diff_error = None 

228 try: 

229 err_message = str(report.longrepr.reprcrash.message) 

230 diff_name = diff_tools.EqualsAssertionError.__name__ 

231 # There is a string like "foo.bar.DiffError: [serialized_data]" 

232 if diff_name in err_message: 

233 serialized_data = err_message[err_message.index(diff_name) + len(diff_name) + 1:] 

234 diff_error = diff_tools.deserialize_error(serialized_data) 

235 

236 assertion_tuple = getattr(self.current_test_item, _ASSERTION_FAILURE_KEY, None) 

237 if assertion_tuple: 

238 op, left, right = assertion_tuple 

239 if self.swap_diff: 

240 left, right = right, left 

241 diff_error = diff_tools.EqualsAssertionError(expected=right, actual=left) 

242 except Exception: 

243 pass 

244 

245 if not diff_error: 

246 from .jb_local_exc_store import get_exception 

247 diff_error = get_exception() 

248 

249 if diff_error: 

250 # Cut everything after postfix: it is internal view of DiffError 

251 strace = str(report.longrepr) 

252 data_postfix = "_ _ _ _ _" 

253 # Error message in pytest must be in "file.py:22 AssertionError" format 

254 # This message goes to strace 

255 # With custom error we must add real exception class explicitly 

256 if data_postfix in strace: 

257 strace = strace[0:strace.index(data_postfix)].strip() 

258 if strace.endswith(":") and diff_error.real_exception: 

259 strace += " " + type(diff_error.real_exception).__name__ 

260 self.teamcity.testFailed(test_id, diff_error.msg or message, strace, 

261 flowId=test_id, 

262 comparison_failure=diff_error 

263 ) 

264 else: 

265 self.teamcity.testFailed(test_id, message, str(report.longrepr), flowId=test_id) 

266 self.report_test_finished(test_id, duration) 

267 

268 def report_test_skip(self, test_id, report): 

269 if type(report.longrepr) is tuple and len(report.longrepr) == 3: 

270 reason = report.longrepr[2] 

271 else: 

272 reason = str(report.longrepr) 

273 

274 if hasattr(report, 'duration'): 

275 duration = timedelta(seconds=report.duration) 

276 else: 

277 duration = None 

278 

279 self.ensure_test_start_reported(test_id) 

280 self.report_test_output(report, test_id) 

281 self.teamcity.testIgnored(test_id, reason, flowId=test_id) 

282 self.report_test_finished(test_id, duration) 

283 

284 def pytest_assertrepr_compare(self, config, op, left, right): 

285 setattr(self.current_test_item, _ASSERTION_FAILURE_KEY, (op, left, right)) 

286 

287 def pytest_runtest_logreport(self, report): 

288 """ 

289 :type report: _pytest.runner.TestReport 

290 """ 

291 test_id = self.format_test_id(report.nodeid, report.location) 

292 

293 duration = timedelta(seconds=report.duration) 

294 

295 if report.passed: 

296 # Do not report passed setup/teardown if no output 

297 if report.when == 'call': 

298 self.ensure_test_start_reported(test_id) 

299 if not self.skip_passed_output: 

300 self.report_test_output(report, test_id) 

301 self.report_test_finished(test_id, duration) 

302 else: 

303 if self.report_has_output(report) and not self.skip_passed_output: 

304 block_name = "test " + report.when 

305 self.teamcity.blockOpened(block_name, flowId=test_id) 

306 self.report_test_output(report, test_id) 

307 self.teamcity.blockClosed(block_name, flowId=test_id) 

308 elif report.failed: 

309 if report.when == 'call': 

310 self.report_test_failure(test_id, report) 

311 elif report.when == 'setup': 

312 if self.report_has_output(report): 

313 self.teamcity.blockOpened("test setup", flowId=test_id) 

314 self.report_test_output(report, test_id) 

315 self.teamcity.blockClosed("test setup", flowId=test_id) 

316 

317 self.report_test_failure(test_id, report, message="test setup failed", report_output=False) 

318 elif report.when == 'teardown': 

319 # Report failed teardown as a separate test as original test is already finished 

320 self.report_test_failure(test_id + "_teardown", report) 

321 elif report.skipped: 

322 self.report_test_skip(test_id, report) 

323 

324 def pytest_collectreport(self, report): 

325 test_id = self.format_test_id(report.nodeid, report.location) + "_collect" 

326 

327 if report.failed: 

328 self.report_test_failure(test_id, report) 

329 elif report.skipped: 

330 self.report_test_skip(test_id, report) 

331 

332 def pytest_terminal_summary(self): 

333 if self.coverage_controller is not None: 

334 try: 

335 self._report_coverage() 

336 except Exception: 

337 tb = traceback.format_exc() 

338 self.teamcity.customMessage("Coverage statistics reporting failed", "ERROR", errorDetails=tb) 

339 

340 def _report_coverage(self): 

341 from coverage.misc import NotPython 

342 from coverage.results import Numbers 

343 

344 class _Reporter(object): 

345 def __init__(self, coverage, config): 

346 try: 

347 from coverage.report import Reporter 

348 except ImportError: 

349 # Support for coverage >= 5.0.1. 

350 from coverage.report import get_analysis_to_report 

351 

352 class Reporter(object): 

353 

354 def __init__(self, coverage, config): 

355 self.coverage = coverage 

356 self.config = config 

357 self._file_reporters = [] 

358 

359 def find_file_reporters(self, morfs): 

360 return [fr for fr, _ in get_analysis_to_report(self.coverage, morfs)] 

361 

362 self._reporter = Reporter(coverage, config) 

363 

364 def find_file_reporters(self, morfs): 

365 self.file_reporters = self._reporter.find_file_reporters(morfs) 

366 

367 def __getattr__(self, name): 

368 return getattr(self._reporter, name) 

369 

370 class _CoverageReporter(_Reporter): 

371 def __init__(self, coverage, config, messages): 

372 super(_CoverageReporter, self).__init__(coverage, config) 

373 

374 if hasattr(coverage, 'data'): 

375 self.branches = coverage.data.has_arcs() 

376 else: 

377 self.branches = coverage.get_data().has_arcs() 

378 self.messages = messages 

379 

380 def report(self, morfs, outfile=None): 

381 if hasattr(self, 'find_code_units'): 

382 self.find_code_units(morfs) 

383 else: 

384 self.find_file_reporters(morfs) 

385 

386 total = Numbers() 

387 

388 if hasattr(self, 'code_units'): 

389 units = self.code_units 

390 else: 

391 units = self.file_reporters 

392 

393 for cu in units: 

394 try: 

395 analysis = self.coverage._analyze(cu) 

396 nums = analysis.numbers 

397 total += nums 

398 except KeyboardInterrupt: 

399 raise 

400 except Exception: 

401 if self.config.ignore_errors: 

402 continue 

403 

404 err = sys.exc_info() 

405 typ, msg = err[:2] 

406 if typ is NotPython and not cu.should_be_python(): 

407 continue 

408 

409 test_id = cu.name 

410 details = convert_error_to_string(err) 

411 

412 self.messages.testStarted(test_id, flowId=test_id) 

413 self.messages.testFailed(test_id, message="Coverage analysis failed", details=details, flowId=test_id) 

414 self.messages.testFinished(test_id, flowId=test_id) 

415 

416 if total.n_files > 0: 

417 covered = total.n_executed 

418 total_statements = total.n_statements 

419 

420 if self.branches: 

421 covered += total.n_executed_branches 

422 total_statements += total.n_branches 

423 

424 self.messages.buildStatisticLinesCovered(covered) 

425 self.messages.buildStatisticTotalLines(total_statements) 

426 self.messages.buildStatisticLinesUncovered(total_statements - covered) 

427 reporter = _CoverageReporter( 

428 self.coverage_controller.cov, 

429 self.coverage_controller.cov.config, 

430 self.teamcity, 

431 ) 

432 reporter.report(None)