From 6e680b8c18fc447f0ba64291ef5eb482bbb6b4f0 Mon Sep 17 00:00:00 2001 From: "Ilya.Kazakevich" Date: Wed, 9 Aug 2017 23:24:42 +0300 Subject: [PATCH] PY-22550: support diff_error TC messages --- python/helpers/pycharm/_jb_runner_tools.py | 6 +- python/helpers/pycharm/teamcity/common.py | 7 +- python/helpers/pycharm/teamcity/diff_tools.py | 82 +++++++++++++++++++ .../pycharm/teamcity/flake8_v2_plugin.py | 3 +- .../pycharm/teamcity/flake8_v3_plugin.py | 3 +- python/helpers/pycharm/teamcity/messages.py | 18 +++- .../helpers/pycharm/teamcity/nose_report.py | 10 +++ .../helpers/pycharm/teamcity/pytest_plugin.py | 57 ++++++++++++- python/helpers/pycharm/teamcity/unittestpy.py | 23 +++++- 9 files changed, 195 insertions(+), 14 deletions(-) create mode 100644 python/helpers/pycharm/teamcity/diff_tools.py diff --git a/python/helpers/pycharm/_jb_runner_tools.py b/python/helpers/pycharm/_jb_runner_tools.py index 9588590009f8..d4653f20adf6 100644 --- a/python/helpers/pycharm/_jb_runner_tools.py +++ b/python/helpers/pycharm/_jb_runner_tools.py @@ -265,11 +265,9 @@ class NewTeamcityServiceMessages(_old_service_messages): self.do_command(commands[0], commands[1]) self.testStarted(testName, captureStandardOutput) - def testFailed(self, testName, message='', details='', flowId=None): + def testFailed(self, testName, message='', details='', flowId=None, comparison_failure=None): testName = ".".join(self._test_to_list(testName)) - args = {"name": testName, "message": str(message), - "details": details} - self.message("testFailed", **args) + _old_service_messages.testFailed(self, testName, message, details, comparison_failure=comparison_failure) def testFinished(self, testName, testDuration=None, flowId=None, is_suite=False): testName = ".".join(self._test_to_list(testName)) diff --git a/python/helpers/pycharm/teamcity/common.py b/python/helpers/pycharm/teamcity/common.py index 6319c2008029..0d52b8e9019d 100644 --- a/python/helpers/pycharm/teamcity/common.py +++ b/python/helpers/pycharm/teamcity/common.py @@ -115,10 +115,13 @@ def get_class_fullname(something): return module + '.' + cls.__name__ -def convert_error_to_string(err): +def convert_error_to_string(err, frames_to_skip_from_tail=0): try: exctype, value, tb = err - return ''.join(traceback.format_exception(exctype, value, tb)) + trace = traceback.format_exception(exctype, value, tb) + if frames_to_skip_from_tail: + trace = trace[:-frames_to_skip_from_tail] + return ''.join(trace) except: tb = traceback.format_exc() return "*FAILED TO GET TRACEBACK*: " + tb diff --git a/python/helpers/pycharm/teamcity/diff_tools.py b/python/helpers/pycharm/teamcity/diff_tools.py new file mode 100644 index 000000000000..0f5a88eddb9d --- /dev/null +++ b/python/helpers/pycharm/teamcity/diff_tools.py @@ -0,0 +1,82 @@ +import base64 +import pprint +import sys +import unittest + +_PY2K = sys.version_info < (3,) +_PRIMITIVES = [int, str, bool] +if _PY2K: + # Not available in py3 + # noinspection PyUnresolvedReferences + _PRIMITIVES.append(unicode) # noqa + # Not available in py3 + # noinspection PyUnresolvedReferences + _STR_F = unicode # noqa +else: + _STR_F = str + + +def patch_unittest_diff(): + """ + Patches "assertEquals" to throw DiffError + """ + if sys.version_info < (2, 7): + return + old = unittest.TestCase.assertEqual + + def _patched_equals(self, first, second, msg=None): + if first != second: + error = EqualsAssertionError(first, second, msg) + if error.is_too_big(): + old(self, first, second, msg) + else: + raise error + + unittest.TestCase.assertEqual = _patched_equals + + +def _format_and_convert(val): + # No need to pretty-print primitives + return val if any(x for x in _PRIMITIVES if isinstance(val, x)) else pprint.pformat(val) + + +class EqualsAssertionError(AssertionError): + + def __init__(self, expected, actual, msg=None, preformated=False): + super(AssertionError, self).__init__() + self.expected = expected + self.actual = actual + self.msg = msg + + if not preformated: + self.expected = _format_and_convert(self.expected) + self.actual = _format_and_convert(self.actual) + self.msg = msg if msg else "" + + self.expected = _STR_F(self.expected) + self.actual = _STR_F(self.actual) + + def is_too_big(self): + return len(self.actual) + len(self.expected) > 10000 + + def __str__(self): + return self._serialize() + + def __unicode__(self): + return self._serialize() + + def _serialize(self): + def fix_type(msg): + return msg if _PY2K else bytes(str(msg), "utf-8") + + encoded_fields = [base64.b64encode(fix_type(x)) for x in [self.expected, self.actual, self.msg]] + if not _PY2K: + encoded_fields = [bytes.decode(x) for x in encoded_fields] + return "|".join(encoded_fields) + + +def deserialize_error(serialized_message): + parts = [base64.b64decode(x) for x in str(serialized_message).split("|")] + if not _PY2K: + parts = [bytes.decode(x) for x in parts] + return EqualsAssertionError(parts[0], parts[1], parts[2], preformated=True) diff --git a/python/helpers/pycharm/teamcity/flake8_v2_plugin.py b/python/helpers/pycharm/teamcity/flake8_v2_plugin.py index e60937f4c798..89d831fb2589 100644 --- a/python/helpers/pycharm/teamcity/flake8_v2_plugin.py +++ b/python/helpers/pycharm/teamcity/flake8_v2_plugin.py @@ -54,7 +54,7 @@ class TeamcityReport(pep8.StandardReport): } error_message = '%s %s' % (code, text) - test_name = 'pep8: %s: %s' % (position, error_message) + test_name = 'pep8: %s: %s' % (normalized_filename, error_message) messages.testStarted(test_name) @@ -64,6 +64,7 @@ class TeamcityReport(pep8.StandardReport): line = self.lines[line_number - 1] details = [ + position, line.rstrip(), re.sub(r'\S', ' ', line[:offset]) + '^', ] diff --git a/python/helpers/pycharm/teamcity/flake8_v3_plugin.py b/python/helpers/pycharm/teamcity/flake8_v3_plugin.py index a15b0da132a7..a95052838d9c 100644 --- a/python/helpers/pycharm/teamcity/flake8_v3_plugin.py +++ b/python/helpers/pycharm/teamcity/flake8_v3_plugin.py @@ -35,11 +35,12 @@ class TeamcityReport(base.BaseFormatter): position = '%s:%d:%d' % ( normalized_filename, error.line_number, error.column_number) error_message = '%s %s' % (error.code, error.text) - test_name = 'pep8: %s: %s' % (position, error_message) + test_name = 'pep8: %s: %s' % (normalized_filename, error_message) line = error.physical_line offset = error.column_number details = [ + position, line.rstrip(), re.sub(r'\S', ' ', line[:offset]) + '^', ] diff --git a/python/helpers/pycharm/teamcity/messages.py b/python/helpers/pycharm/teamcity/messages.py index 3e528407cadb..1f9f9f5afbec 100644 --- a/python/helpers/pycharm/teamcity/messages.py +++ b/python/helpers/pycharm/teamcity/messages.py @@ -2,6 +2,9 @@ import sys import time + + + if sys.version_info < (3, ): # Python 2 text_type = unicode # flake8: noqa @@ -9,7 +12,6 @@ else: # Python 3 text_type = str - # Capture some time functions to allow monkeypatching them in tests _time = time.time _localtime = time.localtime @@ -141,8 +143,18 @@ class TeamcityServiceMessages(object): def testIgnored(self, testName, message='', flowId=None): self.message('testIgnored', name=testName, message=message, flowId=flowId) - def testFailed(self, testName, message='', details='', flowId=None): - self.message('testFailed', name=testName, message=message, details=details, flowId=flowId) + def testFailed(self, testName, message='', details='', flowId=None, comparison_failure=None): + if not comparison_failure: + self.message('testFailed', name=testName, message=message, details=details, flowId=flowId) + else: + self.message('testFailed', + name=testName, + message=message, + details=details, + flowId=flowId, + type="comparisonFailure", + actual=comparison_failure.actual, + expected=comparison_failure.expected) def testStdOut(self, testName, out, flowId=None): self.message('testStdOut', name=testName, out=out, flowId=flowId) diff --git a/python/helpers/pycharm/teamcity/nose_report.py b/python/helpers/pycharm/teamcity/nose_report.py index 7a3330891635..141a5938eed6 100644 --- a/python/helpers/pycharm/teamcity/nose_report.py +++ b/python/helpers/pycharm/teamcity/nose_report.py @@ -7,6 +7,7 @@ import inspect from teamcity import is_running_under_teamcity from teamcity.common import is_string, get_class_fullname, convert_error_to_string, dump_test_stdout, FlushingStringIO from teamcity.messages import TeamcityServiceMessages +from .diff_tools import EqualsAssertionError, patch_unittest_diff import nose # noinspection PyPackageRequirements @@ -14,6 +15,7 @@ from nose.exc import SkipTest, DeprecatedTest # noinspection PyPackageRequirements from nose.plugins import Plugin +patch_unittest_diff() CONTEXT_SUITE_FQN = "nose.suite.ContextSuite" @@ -144,6 +146,14 @@ class TeamcityReport(Plugin): # do not log test output twice, see report_finish for actual output handling details = details[:start_index] + details[end_index + len(_captured_output_end_marker):] + try: + error = err[1] + if isinstance(error, EqualsAssertionError): + details = convert_error_to_string(err, 2) + self.messages.testFailed(test_id, message=error.msg, details=details, flowId=test_id, comparison_failure=error) + return + except: + pass self.messages.testFailed(test_id, message=fail_type, details=details, flowId=test_id) def report_finish(self, test): diff --git a/python/helpers/pycharm/teamcity/pytest_plugin.py b/python/helpers/pycharm/teamcity/pytest_plugin.py index 05c6f3875a82..4e4065f7e6ac 100644 --- a/python/helpers/pycharm/teamcity/pytest_plugin.py +++ b/python/helpers/pycharm/teamcity/pytest_plugin.py @@ -20,6 +20,30 @@ from datetime import timedelta from teamcity.messages import TeamcityServiceMessages from teamcity.common import convert_error_to_string, dump_test_stderr, dump_test_stdout from teamcity import is_running_under_teamcity +from teamcity import diff_tools + +diff_tools.patch_unittest_diff() + + +def fetch_diff_error_from_message(err_message): + line_with_diff = None + diff_error_message = None + lines = err_message.split("\n") + if err_message.startswith("AssertionError: assert"): + # Everything in one line + line_with_diff = lines[0][len("AssertionError: assert "):] + elif len(err_message.split("\n")) > 1: + err_line = lines[1] + line_with_diff = err_line[len("assert "):] + diff_error_message = lines[0] + + if line_with_diff and line_with_diff.count("==") == 1: + parts = [x.strip() for x in line_with_diff.split("==")] + parts = [s[1:-1] if s.startswith("'") or s.startswith('"') else s for s in parts] + # Pytest cuts too long lines, no need to check is_too_big + return diff_tools.EqualsAssertionError(parts[0], parts[1], diff_error_message) + else: + return None def pytest_addoption(parser): @@ -191,7 +215,38 @@ class EchoTeamCityMessages(object): self.ensure_test_start_reported(test_id) if report_output: self.report_test_output(report, test_id) - self.teamcity.testFailed(test_id, message, str(report.longrepr), flowId=test_id) + + diff_error = None + try: + err_message = str(report.longrepr.reprcrash.message) + diff_name = diff_tools.EqualsAssertionError.__name__ + # There is a string like "foo.bar.DiffError: [serialized_data]" + if diff_name in err_message: + serialized_data = err_message[err_message.index(diff_name) + len(diff_name) + 1:] + diff_error = diff_tools.deserialize_error(serialized_data) + + # AssertionError is patched in py.test, we can try to fetch diff from it + # In general case message starts with "AssertionError: ", but can also starts with "assert" for top-level + # function. To support both cases we unify them + if err_message.startswith("assert"): + err_message = "AssertionError: " + err_message + if err_message.startswith("AssertionError:"): + diff_error = fetch_diff_error_from_message(err_message) + except: + pass + + if diff_error: + # Cut everything after postfix: it is internal view of DiffError + strace = str(report.longrepr) + data_postfix = "_ _ _ _ _" + if data_postfix in strace: + strace = strace[0:strace.index(data_postfix)] + self.teamcity.testFailed(test_id, diff_error.msg if diff_error.msg else message, strace, + flowId=test_id, + comparison_failure=diff_error + ) + else: + self.teamcity.testFailed(test_id, message, str(report.longrepr), flowId=test_id) self.report_test_finished(test_id, duration) def report_test_skip(self, test_id, report): diff --git a/python/helpers/pycharm/teamcity/unittestpy.py b/python/helpers/pycharm/teamcity/unittestpy.py index b593873eaca3..cb2223c76790 100644 --- a/python/helpers/pycharm/teamcity/unittestpy.py +++ b/python/helpers/pycharm/teamcity/unittestpy.py @@ -8,6 +8,8 @@ from teamcity.messages import TeamcityServiceMessages from teamcity.common import is_string, get_class_fullname, convert_error_to_string, \ dump_test_stdout, dump_test_stderr, get_exception_message, to_unicode, FlushingStringIO +from .diff_tools import EqualsAssertionError, patch_unittest_diff + _real_stdout = sys.stdout _real_stderr = sys.stderr _ERROR_HOLDERS_FQN = ("unittest.suite._ErrorHolder", "unittest2.suite._ErrorHolder") @@ -189,19 +191,35 @@ class TeamcityTestResult(TestResult): def report_fail(self, test, fail_type, err): test_id = self.get_test_id(test) + diff_failed = None + try: + error = err[1] + if isinstance(error, EqualsAssertionError): + diff_failed = error + except: + pass + if is_string(err): details = err elif get_class_fullname(err) == "twisted.python.failure.Failure": details = err.getTraceback() else: - details = convert_error_to_string(err) + frames_to_skip_from_tail = 2 if diff_failed else 0 + details = convert_error_to_string(err, frames_to_skip_from_tail) subtest_failures = self.get_subtest_failure(test_id) if subtest_failures: details = "Failed subtests list: " + subtest_failures + "\n\n" + details.strip() details = details.strip() - self.messages.testFailed(test_id, message=fail_type, details=details, flowId=test_id) + if diff_failed: + self.messages.testFailed(test_id, + message=diff_failed.msg, + details=details, + flowId=test_id, + comparison_failure=diff_failed) + else: + self.messages.testFailed(test_id, message=fail_type, details=details, flowId=test_id) self.failed_tests.add(test_id) def startTest(self, test): @@ -272,6 +290,7 @@ class TeamcityTestRunner(TextTestRunner): def run(self, test): # noinspection PyBroadException + patch_unittest_diff() try: total_tests = test.countTestCases() TeamcityServiceMessages(_real_stdout).testCount(total_tests)