From 1d400e409ab2c560ad5697732f8f6a14a9dd16f7 Mon Sep 17 00:00:00 2001 From: "Ilya.Kazakevich" Date: Tue, 7 Feb 2017 21:47:10 +0300 Subject: [PATCH] PY-22409: TC runners how report number of tests. PY-13833: dots in test names are escaped by TC runners now. * Blocks temporary disabled (see comments in _jb_runner_tools.py) * Generators/parametrized tests are always provided with parentheses --- python/helpers/pycharm/_jb_nosetest_runner.py | 13 +-- python/helpers/pycharm/_jb_pytest_runner.py | 13 +-- python/helpers/pycharm/_jb_runner_tools.py | 42 +++++---- python/helpers/pycharm/teamcity/messages.py | 4 + .../helpers/pycharm/teamcity/nose_report.py | 86 +++++++++++++------ .../helpers/pycharm/teamcity/pytest_plugin.py | 26 +++++- python/helpers/pycharm/teamcity/unittestpy.py | 51 +++++++---- .../universalTests/PyUniversalTests.kt | 27 ++++-- .../testRunner/env/pytest/test_escape_me.py | 11 +++ .../env/pytest/test_escape_me.tree.txt | 9 ++ .../python/testing/PythonPyTestingTest.java | 31 ++++++- 11 files changed, 222 insertions(+), 91 deletions(-) create mode 100644 python/testData/testRunner/env/pytest/test_escape_me.py create mode 100644 python/testData/testRunner/env/pytest/test_escape_me.tree.txt diff --git a/python/helpers/pycharm/_jb_nosetest_runner.py b/python/helpers/pycharm/_jb_nosetest_runner.py index 6af3b514ca91..0887c4b28b25 100644 --- a/python/helpers/pycharm/_jb_nosetest_runner.py +++ b/python/helpers/pycharm/_jb_nosetest_runner.py @@ -7,19 +7,8 @@ import sys from _jb_runner_tools import jb_start_tests, jb_patch_separator, jb_doc_args from teamcity.nose_report import TeamcityReport -def __parse_parametrized(part): - """ - - Support nose generators that provides names like foo(1,2) - """ - match = re.match("^(.+)\((.+)\)$", part) - if not match: - return [part] - else: - return [match.group(1), match.group(2)] - if __name__ == '__main__': - path, targets, additional_args = jb_start_tests(__parse_parametrized) + path, targets, additional_args = jb_start_tests() sys.argv += [path] if path else jb_patch_separator(targets, fs_glue=".", python_glue=".", fs_to_python_glue=":") sys.argv += additional_args jb_doc_args("Nosetest", sys.argv) diff --git a/python/helpers/pycharm/_jb_pytest_runner.py b/python/helpers/pycharm/_jb_pytest_runner.py index 81975ddecfbf..0f8d837817f6 100644 --- a/python/helpers/pycharm/_jb_pytest_runner.py +++ b/python/helpers/pycharm/_jb_pytest_runner.py @@ -9,20 +9,9 @@ from _jb_runner_tools import jb_start_tests, jb_patch_separator, jb_doc_args from teamcity import pytest_plugin -def __parse_parametrized(part): - """ - - Support pytest parametrized tests for cases like foo[1,2] - """ - match = re.match("^(.+)\[(.+)\]$", part) - if not match: - return [part] - else: - return [match.group(1), match.group(2)] - if __name__ == '__main__': - path, targets, additional_args = jb_start_tests(__parse_parametrized) + path, targets, additional_args = jb_start_tests() sys.argv += additional_args joined_targets = jb_patch_separator(targets, fs_glue="/", python_glue="::", fs_to_python_glue=".py::") # When file is launched in py.test it should be file.py: you can't provide it as bare module diff --git a/python/helpers/pycharm/_jb_runner_tools.py b/python/helpers/pycharm/_jb_runner_tools.py index a7c74ce31bac..8f74f51c8814 100644 --- a/python/helpers/pycharm/_jb_runner_tools.py +++ b/python/helpers/pycharm/_jb_runner_tools.py @@ -6,9 +6,8 @@ import argparse import atexit import imp import os +import re import sys -import time -import datetime from teamcity import teamcity_presence_env_var, messages @@ -17,6 +16,20 @@ if teamcity_presence_env_var not in os.environ: os.environ[teamcity_presence_env_var] = "LOCAL" +def _parse_parametrized(part): + """ + + Support nose generators / py.test parameters and other functions that provides names like foo(1,2) + Until https://github.com/JetBrains/teamcity-messages/issues/121, all such tests are provided + with parentheses + """ + match = re.match("^(.+)(\(.+\))$", part) + if not match: + return [part] + else: + return [match.group(1), match.group(2)] + + # Monkeypatching TC to pass location hint class _TreeManager(object): @@ -146,7 +159,7 @@ class NewTeamcityServiceMessages(_old_service_messages): # Intellij may fail to process message if it has char just before it. # Space before message has no visible affect, but saves from such cases print(" ") - if messageName == "enteredTheMatrix": + if messageName in {"enteredTheMatrix", "testCount"}: _old_service_messages.message(self, messageName, **properties) return properties["locationHint"] = "python://{0}".format(properties["name"]) @@ -169,12 +182,9 @@ class NewTeamcityServiceMessages(_old_service_messages): It most cases dot is used, but runner may provide custom function """ parts = test_name.split(".") - if not PARSE_FUNC: - return parts - result = [] for part in parts: - result += PARSE_FUNC(part) + result += _parse_parametrized(part) return result def _fix_setup_teardown_name(self, test_name): @@ -187,11 +197,15 @@ class NewTeamcityServiceMessages(_old_service_messages): except KeyError: return test_name - def blockOpened(self, name, flowId=None): - self.testStarted(".".join(TREE_MANAGER.current_branch + [self._fix_setup_teardown_name(name)])) - - def blockClosed(self, name, flowId=None): - self.testFinished(".".join(TREE_MANAGER.current_branch + [self._fix_setup_teardown_name(name)])) + # Blocks are used for 2 cases now: + # 1) Unittest subtests (broken, because failure can't be reported) + # 2) setup/teardown (does not work, see https://github.com/JetBrains/teamcity-messages/issues/114) + # So, temporary disabled + # def blockOpened(self, name, flowId=None): + # self.testStarted(".".join(TREE_MANAGER.current_branch + [self._fix_setup_teardown_name(name)])) + # + # def blockClosed(self, name, flowId=None): + # self.testFinished(".".join(TREE_MANAGER.current_branch + [self._fix_setup_teardown_name(name)])) def testStarted(self, testName, captureStandardOutput=None, flowId=None, is_suite=False): test_name_as_list = self._test_to_list(testName) @@ -304,7 +318,7 @@ def jb_patch_separator(targets, fs_glue, python_glue, fs_to_python_glue): return map(_patch_target, targets) -def jb_start_tests(func_to_parse=None): +def jb_start_tests(): """ Parses arguments, starts protocol and returns tuple of arguments @@ -326,8 +340,6 @@ def jb_start_tests(func_to_parse=None): parser.add_argument('--target', help='Python target to run', action="append") namespace = parser.parse_args() del sys.argv[1:] # Remove all args - global PARSE_FUNC - PARSE_FUNC = func_to_parse NewTeamcityServiceMessages().message('enteredTheMatrix') return namespace.path, namespace.target, additional_args diff --git a/python/helpers/pycharm/teamcity/messages.py b/python/helpers/pycharm/teamcity/messages.py index e358baf53139..5d77282fd94f 100644 --- a/python/helpers/pycharm/teamcity/messages.py +++ b/python/helpers/pycharm/teamcity/messages.py @@ -121,6 +121,10 @@ class TeamcityServiceMessages(object): import teamcity.context_managers as cm return cm.test(self, testName=testName, captureStandardOutput=captureStandardOutput, testDuration=testDuration, flowId=flowId) + # Unsupported in TeamCity, used in IntellIJ-based IDEs to predict number of tests to be run in the test session + def testCount(self, count, flowId=None): + self.message('testCount', count=str(count), flowId=flowId) + def testIgnored(self, testName, message='', flowId=None): self.message('testIgnored', name=testName, message=message, flowId=flowId) diff --git a/python/helpers/pycharm/teamcity/nose_report.py b/python/helpers/pycharm/teamcity/nose_report.py index 9ed71c85dd8d..78f692dec2ad 100644 --- a/python/helpers/pycharm/teamcity/nose_report.py +++ b/python/helpers/pycharm/teamcity/nose_report.py @@ -8,7 +8,11 @@ from teamcity import is_running_under_teamcity from teamcity.common import is_string, split_output, limit_output, get_class_fullname, convert_error_to_string from teamcity.messages import TeamcityServiceMessages +import nose +# noinspection PyPackageRequirements from nose.exc import SkipTest, DeprecatedTest +# noinspection PyPackageRequirements +from nose.plugins import Plugin CONTEXT_SUITE_FQN = "nose.suite.ContextSuite" @@ -21,7 +25,7 @@ def _ln(label): out = '%s %s %s' % ('-' * chunk, label, '-' * chunk) pad = 70 - len(out) if pad > 0: - out = out + ('-' * pad) + out += '-' * pad return out @@ -31,7 +35,8 @@ _captured_output_end_marker = "\n" + _ln('>> end captured stdout <<') _real_stdout = sys.stdout -class TeamcityReport(object): +# noinspection PyPep8Naming,PyMethodMayBeStatic +class TeamcityReport(Plugin): name = 'teamcity-report' score = 10000 @@ -40,6 +45,8 @@ class TeamcityReport(object): self.messages = TeamcityServiceMessages(_real_stdout) self.test_started_datetime_map = {} + self.config = None + self.total_tests = 0 self.enabled = False def get_test_id(self, test): @@ -79,41 +86,32 @@ class TeamcityReport(object): def configure(self, options, conf): self.enabled = is_running_under_teamcity() + self.config = conf def options(self, parser, env=os.environ): pass - def _get_capture_plugin(self, test): + def _get_capture_plugin(self): """ - :type test: nose.case.Test - :rtype: nose.plugins.base.Plugin + :rtype: nose.plugins.capture.Capture """ - for plugin in test.config.plugins.plugins: + for plugin in self.config.plugins.plugins: if plugin.name == "capture": return plugin return None - def _capture_plugin_enabled(self, test): - """ - :type test: nose.case.Test - """ - plugin = self._get_capture_plugin(test) + def _capture_plugin_enabled(self): + plugin = self._get_capture_plugin() return plugin is not None and plugin.enabled - def _capture_plugin_buffer(self, test): - """ - :type test: nose.case.Test - """ - plugin = self._get_capture_plugin(test) + def _capture_plugin_buffer(self): + plugin = self._get_capture_plugin() if plugin is None: return None return getattr(plugin, "buffer", None) - def _captureStandardOutput_value(self, test): - """ - :type test: nose.case.Test - """ - if self._capture_plugin_enabled(test): + def _captureStandardOutput_value(self): + if self._capture_plugin_enabled(): return 'false' else: return 'true' @@ -140,9 +138,9 @@ class TeamcityReport(object): test_id = self.get_test_id(test) captured_output = getattr(test, "capturedOutput", None) - if captured_output is None and self._capture_plugin_enabled(test): + if captured_output is None and self._capture_plugin_enabled(): # nose capture does not fill 'capturedOutput' property on successful tests - captured_output = self._capture_plugin_buffer(test) + captured_output = self._capture_plugin_buffer() if captured_output: for chunk in split_output(limit_output(captured_output)): self.messages.testStdOut(test_id, chunk, flowId=test_id) @@ -153,6 +151,44 @@ class TeamcityReport(object): else: self.messages.testFinished(test_id, flowId=test_id) + def prepareTestLoader(self, loader): + """Insert ourselves into loader calls to count tests. + The top-level loader call often returns lazy results, like a LazySuite. + This is a problem, as we would destroy the suite by iterating over it + to count the tests. Consequently, we monkey-patch the top-level loader + call to do the load twice: once for the actual test running and again + to yield something we can iterate over to do the count. + + from https://github.com/erikrose/nose-progressive/ + :type loader: nose.loader.TestLoader + """ + + # TODO: If there's ever a practical need, also patch loader.suiteClass + # or even TestProgram.createTests. createTests seems to be main top- + # level caller of loader methods, and nose.core.collector() (which + # isn't even called in nose) is an alternate one. + # + # nose 1.3.4 contains required fix: + # Another fix for Python 3.4: Call super in LazySuite to access _removed_tests variable + if hasattr(loader, 'loadTestsFromNames') and nose.__versioninfo__ >= (1, 3, 4): + old_loadTestsFromNames = loader.loadTestsFromNames + + def _loadTestsFromNames(*args, **kwargs): + suite = old_loadTestsFromNames(*args, **kwargs) + self.total_tests += suite.countTestCases() + + # Clear out the loader's cache. Otherwise, it never finds any tests + # for the actual test run: + loader._visitedPaths = set() + + return old_loadTestsFromNames(*args, **kwargs) + loader.loadTestsFromNames = _loadTestsFromNames + + # noinspection PyUnusedLocal + def prepareTestRunner(self, runner): + if self.total_tests: + self.messages.testCount(self.total_tests) + def addError(self, test, err): test_class_name = get_class_fullname(test) test_id = self.get_test_id(test) @@ -164,7 +200,7 @@ class TeamcityReport(object): self.messages.testIgnored(test_id, message="Deprecated", flowId=test_id) self.report_finish(test) elif test_class_name == CONTEXT_SUITE_FQN: - self.messages.testStarted(test_id, captureStandardOutput=self._captureStandardOutput_value(test), flowId=test_id) + self.messages.testStarted(test_id, captureStandardOutput=self._captureStandardOutput_value(), flowId=test_id) self.report_fail(test, 'error in ' + test.error_context + ' context', err) self.messages.testFinished(test_id, flowId=test_id) else: @@ -179,7 +215,7 @@ class TeamcityReport(object): test_id = self.get_test_id(test) self.test_started_datetime_map[test_id] = datetime.datetime.now() - self.messages.testStarted(test_id, captureStandardOutput=self._captureStandardOutput_value(test), flowId=test_id) + self.messages.testStarted(test_id, captureStandardOutput=self._captureStandardOutput_value(), flowId=test_id) def addSuccess(self, test): self.report_finish(test) diff --git a/python/helpers/pycharm/teamcity/pytest_plugin.py b/python/helpers/pycharm/teamcity/pytest_plugin.py index 4076d561b7f4..8b55641f4873 100644 --- a/python/helpers/pycharm/teamcity/pytest_plugin.py +++ b/python/helpers/pycharm/teamcity/pytest_plugin.py @@ -107,13 +107,32 @@ class EchoTeamCityMessages(object): test_id = nodeid - if test_id.find("::") < 0: - test_id += "::top_level" + if test_id: + if test_id.find("::") < 0: + test_id += "::top_level" + else: + test_id = "top_level" + + first_bracket = test_id.find("[") + if first_bracket > 0: + # [] -> (), make it look like nose parameterized tests + params = "(" + test_id[first_bracket + 1:] + if params.endswith("]"): + params = params[:-1] + ")" + test_id = test_id[:first_bracket] + if test_id.endswith("::"): + test_id = test_id[:-2] + else: + params = "" test_id = test_id.replace("::()::", "::") test_id = re.sub(r"\.pyc?::", r"::", test_id) test_id = test_id.replace(".", "_").replace(os.sep, ".").replace("/", ".").replace('::', '.') + if params: + params = params.replace(".", "_") + test_id += params + return test_id def format_location(self, location): @@ -121,6 +140,9 @@ class EchoTeamCityMessages(object): return "%s:%s (%s)" % (str(location[0]), str(location[1]), str(location[2])) return str(location) + def pytest_collection_modifyitems(self, session, config, items): + self.teamcity.testCount(len(items)) + def pytest_runtest_logstart(self, nodeid, location): self.ensure_test_start_reported(self.format_test_id(nodeid, location)) diff --git a/python/helpers/pycharm/teamcity/unittestpy.py b/python/helpers/pycharm/teamcity/unittestpy.py index c0e0a0260c66..424a7bea32e4 100644 --- a/python/helpers/pycharm/teamcity/unittestpy.py +++ b/python/helpers/pycharm/teamcity/unittestpy.py @@ -76,6 +76,7 @@ class TeamcityTestResult(TestResult): test_name = re.sub(r'^(.*) \((.*)\)$', r'\2.\1', test_name) self.messages.testStarted(test_name, flowId=test_name) + # noinspection PyTypeChecker self.report_fail(test_name, 'Failure', err) self.messages.testFinished(test_name, flowId=test_name) elif get_class_fullname(err[0]) == "unittest2.case.SkipTest": @@ -97,25 +98,38 @@ class TeamcityTestResult(TestResult): super(TeamcityTestResult, self).addSubTest(test, subtest, err) test_id = self.get_test_id(test) + subtest_id = self.get_test_id(subtest) + + if subtest_id.startswith(test_id): + block_id = subtest_id[len(test_id):].strip() + else: + block_id = subtest_id + if len(block_id) == 0: + block_id = subtest_id if err is not None: - if issubclass(err[0], test.failureException): - self.add_subtest_failure(test_id, self.get_test_id(subtest), err) - self.messages.testStdErr(test_id, out="%s: failure\n" % self.get_test_id(subtest), flowId=test_id) - else: - self.add_subtest_failure(test_id, self.get_test_id(subtest), err) - self.messages.testStdErr(test_id, out="%s: error\n" % self.get_test_id(subtest), flowId=test_id) - else: - self.messages.testStdOut(test_id, out="%s: ok\n" % self.get_test_id(subtest), flowId=test_id) + self.add_subtest_failure(test_id, block_id) - def add_subtest_failure(self, test_id, subtest_id, err): + if issubclass(err[0], test.failureException): + self.messages.blockOpened(block_id, flowId=test_id) + self.messages.testStdErr(test_id, out="SubTest failure: %s\n" % convert_error_to_string(err), flowId=test_id) + self.messages.blockClosed(block_id, flowId=test_id) + else: + self.messages.blockOpened(block_id, flowId=test_id) + self.messages.testStdErr(test_id, out="SubTest error: %s\n" % convert_error_to_string(err), flowId=test_id) + self.messages.blockClosed(block_id, flowId=test_id) + else: + self.messages.blockOpened(block_id, flowId=test_id) + self.messages.blockClosed(block_id, flowId=test_id) + + def add_subtest_failure(self, test_id, subtest_block_id): fail_array = self.subtest_failures.get(test_id, []) - fail_array.append("%s:\n%s" % (subtest_id, convert_error_to_string(err))) + fail_array.append(subtest_block_id) self.subtest_failures[test_id] = fail_array def get_subtest_failure(self, test_id): fail_array = self.subtest_failures.get(test_id, []) - return "\n".join(fail_array) + return ", ".join(fail_array) def report_fail(self, test, fail_type, err): test_id = self.get_test_id(test) @@ -127,9 +141,10 @@ class TeamcityTestResult(TestResult): else: details = convert_error_to_string(err) - subtest_failure = self.get_subtest_failure(test_id) - if subtest_failure: - details = subtest_failure + "\n" + details + subtest_failures = self.get_subtest_failure(test_id) + if subtest_failures: + details = "Failed subtests list: " + subtest_failures + "\n\n" + details.strip() + details = details.strip() self.messages.testFailed(test_id, message=fail_type, details=details, flowId=test_id) self.failed_tests.add(test_id) @@ -145,7 +160,7 @@ class TeamcityTestResult(TestResult): def stopTest(self, test): test_id = self.get_test_id(test) - if self.buffer: + if getattr(self, 'buffer', None): # Do not allow super() method to print output by itself self._mirrorOutput = False @@ -161,8 +176,10 @@ class TeamcityTestResult(TestResult): super(TeamcityTestResult, self).stopTest(test) - if test_id not in self.failed_tests and self.subtest_failures.get(test_id, []): - self.report_fail(test, "Subtest failed", "") + if test_id not in self.failed_tests: + subtest_failures = self.get_subtest_failure(test_id) + if subtest_failures: + self.report_fail(test, "One or more subtests failed", "") time_diff = datetime.datetime.now() - self.test_started_datetime_map[test_id] self.messages.testFinished(test_id, testDuration=time_diff, flowId=test_id) diff --git a/python/src/com/jetbrains/python/testing/universalTests/PyUniversalTests.kt b/python/src/com/jetbrains/python/testing/universalTests/PyUniversalTests.kt index 8181b9d74bab..334ee39c8328 100644 --- a/python/src/com/jetbrains/python/testing/universalTests/PyUniversalTests.kt +++ b/python/src/com/jetbrains/python/testing/universalTests/PyUniversalTests.kt @@ -86,7 +86,18 @@ internal fun getAdditionalArgumentsPropertyName() = PyUniversalTestConfiguration * Since runners report names of tests as qualified name, no need to convert it to PSI and back to string. * We just save its name and provide it again to rerun */ -private class PyTargetBasedPsiLocation(val target: ConfigurationTarget, element: PsiElement) : PsiLocation(element) +private class PyTargetBasedPsiLocation(val target: ConfigurationTarget, element: PsiElement) : PsiLocation(element) { + override fun equals(other: Any?): Boolean { + if (other is PyTargetBasedPsiLocation) { + return target == other.target + } + return false + } + + override fun hashCode(): Int { + return target.hashCode() + } +} /** @@ -105,11 +116,16 @@ private object PyUniversalTestsLocator : SMTestLocator { if (scope !is ModuleWithDependenciesScope) { return listOf() } - val element = QualifiedName.fromDottedString(path).toElement(scope.module, - TypeEvalContext.userInitiated(project, null)) + val qualifiedName = QualifiedName.fromDottedString(path) + val element = qualifiedName.toElement(scope.module, + TypeEvalContext.userInitiated(project, null)) if (element != null) { // Path is qualified name of python test according to runners protocol - return listOf(PyTargetBasedPsiLocation(ConfigurationTarget(path, TestTargetType.PYTHON), element)) + // Parentheses are part of generators / parametrized tests + // Until https://github.com/JetBrains/teamcity-messages/issues/121 they are disabled, + // so we cut them out of path not to provide unsupported targets to runners + val pathNoParentheses = QualifiedName.fromComponents(qualifiedName.components.filter { !it.contains('(') }).toString() + return listOf(PyTargetBasedPsiLocation(ConfigurationTarget(pathNoParentheses, TestTargetType.PYTHON), element)) } else { return listOf() @@ -325,7 +341,8 @@ abstract class PyUniversalTestConfiguration(project: Project, override fun getTestSpecsForRerun(scope: GlobalSearchScope, locations: MutableList, AbstractTestProxy>>): List { val result = ArrayList() - locations.map { getTestSpecForPythonTarget(it.first) }.filterNotNull().forEach { result.addAll(it) } + // Set used to remove duplicate targets + locations.map { it.first }.toSet().map { getTestSpecForPythonTarget(it) }.filterNotNull().forEach { result.addAll(it) } return result + generateRawArguments() } diff --git a/python/testData/testRunner/env/pytest/test_escape_me.py b/python/testData/testRunner/env/pytest/test_escape_me.py new file mode 100644 index 000000000000..5046830d07de --- /dev/null +++ b/python/testData/testRunner/env/pytest/test_escape_me.py @@ -0,0 +1,11 @@ +import pytest + +@pytest.mark.parametrize('username,url', [ + (None, 'https://facebook.com/'), + (None, 'https://facebook.com/share.php?http://foo.com/'), + (None, 'https://facebook.com/home.php'), + ('username', 'https://facebook.com/username'), + ('username', 'https://facebook.com/username/app_123'), +]) +def test_get_facebook_username(url, username): + pass \ No newline at end of file diff --git a/python/testData/testRunner/env/pytest/test_escape_me.tree.txt b/python/testData/testRunner/env/pytest/test_escape_me.tree.txt new file mode 100644 index 000000000000..6408af88b0de --- /dev/null +++ b/python/testData/testRunner/env/pytest/test_escape_me.tree.txt @@ -0,0 +1,9 @@ +Test tree: +[root] +.test_escape_me +..test_get_facebook_username +...(None-https://facebook_com/)(+) +...(None-https://facebook_com/share_php?http://foo_com/)(+) +...(None-https://facebook_com/home_php)(+) +...(username-https://facebook_com/username)(+) +...(username-https://facebook_com/username/app_123)(+) diff --git a/python/testSrc/com/jetbrains/env/python/testing/PythonPyTestingTest.java b/python/testSrc/com/jetbrains/env/python/testing/PythonPyTestingTest.java index 13f9fe59b3f5..4eacdda4e8d5 100644 --- a/python/testSrc/com/jetbrains/env/python/testing/PythonPyTestingTest.java +++ b/python/testSrc/com/jetbrains/env/python/testing/PythonPyTestingTest.java @@ -49,6 +49,30 @@ public class PythonPyTestingTest extends PyEnvTestCase { PyUniversalPyTestConfiguration.class)); } + /** + * Ensure dots in test names do not break anything (PY-13833) + */ + @Test + public void testEscape() throws Exception { + runPythonTest(new PyProcessWithConsoleTestTask("/testRunner/env/pytest/", SdkCreationType.EMPTY_SDK) { + @NotNull + @Override + protected PyTestTestProcessRunner createProcessRunner() throws Exception { + return new PyTestTestProcessRunner("test_escape_me.py", 0); + } + + @Override + protected void checkTestResults(@NotNull final PyTestTestProcessRunner runner, + @NotNull final String stdout, + @NotNull final String stderr, + @NotNull final String all) { + final String resultTree = runner.getFormattedTestTree().trim(); + final String expectedTree = myFixture.configureByFile("test_escape_me.tree.txt").getText().trim(); + Assert.assertEquals("Test result wrong tree", expectedTree, resultTree); + } + }); + } + // Import error should lead to test failure @Test public void testFailInCaseOfError() { @@ -147,9 +171,10 @@ public class PythonPyTestingTest extends PyEnvTestCase { @NotNull final String stderr, @NotNull final String all) { if (runner.getCurrentRerunStep() > 0) { - // Pytest supports [1] notation, so we only rerun failed - assertEquals(runner.getFormattedTestTree(), 4, runner.getAllTestsCount()); - assertEquals(runner.getFormattedTestTree(), 0, runner.getPassedTestsCount()); + // We rerun all tests, since running parametrized tests is broken until + // https://github.com/JetBrains/teamcity-messages/issues/121 + assertEquals(runner.getFormattedTestTree(), 7, runner.getAllTestsCount()); + assertEquals(runner.getFormattedTestTree(), 3, runner.getPassedTestsCount()); assertEquals(runner.getFormattedTestTree(), 4, runner.getFailedTestsCount()); return; }