New test runners: Splitting flat names to tree

* Tests are provided in format like "foo.bar", but we need to convert
them to parent-child

python/helpers/pycharm/_jb_nosetest_runner.py

@@ -1,12 +1,25 @@
 # coding=utf-8
+import re
+
 import nose
 import sys
 
 from _jb_runner_tools import jb_start_tests, jb_patch_separator, jb_doc_args
 from teamcity.nose_report import TeamcityReport
 
+def __parse_parametrized(part):
+    """
+
+    Support nose generators that provides names like foo(1,2)
+    """
+    match = re.match("^(.+)\((.+)\)$", part)
+    if not match:
+        return [part]
+    else:
+        return [match.group(1), match.group(2)]
+
 if __name__ == '__main__':
-    path, targets, additional_args = jb_start_tests()
+    path, targets, additional_args = jb_start_tests(__parse_parametrized)
     sys.argv += [path] if path else jb_patch_separator(targets, fs_glue=".", python_glue=".", fs_to_python_glue=":")
     sys.argv += additional_args
     jb_doc_args("Nosetest", sys.argv)

python/helpers/pycharm/_jb_pytest_runner.py

@@ -1,17 +1,38 @@
 # coding=utf-8
+import re
 import sys
 
 import pytest
+from _pytest.config import _prepareconfig
 
 from _jb_runner_tools import jb_start_tests, jb_patch_separator, jb_doc_args
 from teamcity import pytest_plugin
 
+
+def __parse_parametrized(part):
+    """
+
+    Support pytest parametrized tests for cases like foo[1,2]
+    """
+    match = re.match("^(.+)\[(.+)\]$", part)
+    if not match:
+        return [part]
+    else:
+        return [match.group(1), match.group(2)]
+
+
 if __name__ == '__main__':
-    path, targets, additional_args = jb_start_tests()
+    path, targets, additional_args = jb_start_tests(__parse_parametrized)
     sys.argv += additional_args
-    joined_targets = jb_patch_separator(targets, fs_glue=":", python_glue="::", fs_to_python_glue=".py::")
+    joined_targets = jb_patch_separator(targets, fs_glue="/", python_glue="::", fs_to_python_glue=".py::")
     # When file is launched in py.test it should be file.py: you can't provide it as bare module
     joined_targets = [t + ".py" if ":" not in t else t for t in joined_targets]
     sys.argv += [path] if path else joined_targets
     jb_doc_args("py.test", sys.argv[1:])
-    pytest.main(sys.argv[1:], [pytest_plugin])
+
+    # plugin is discovered automatically in 3, but not in 2
+    # to prevent "plugin already registered" problem we check it first
+    plugins_to_load = []
+    if not _prepareconfig().pluginmanager.hasplugin("pytest-teamcity"):
+        plugins_to_load.append(pytest_plugin)
+    pytest.main(sys.argv[1:], plugins_to_load)

python/helpers/pycharm/_jb_runner_tools.py

@@ -3,10 +3,12 @@
 Tools to implement runners (https://confluence.jetbrains.com/display/~link/PyCharm+test+runners+protocol)
 """
 import argparse
-
-import sys, os
-
+import atexit
 import imp
+import os
+import sys
+import time
+import datetime
 
 from teamcity import teamcity_presence_env_var, messages
 
@@ -14,20 +16,250 @@ from teamcity import teamcity_presence_env_var, messages
 if teamcity_presence_env_var not in os.environ:
     os.environ[teamcity_presence_env_var] = "LOCAL"
 
+
 # Monkeypatching TC to pass location hint
-old_started = messages.TeamcityServiceMessages.message
+
+class _TreeManager(object):
+    """
+    Manages output tree by building it from flat test names.
+    """
+
+    def __init__(self):
+        super(_TreeManager, self).__init__()
+        # Currently active branch as list. New nodes go to this branch
+        self.current_branch = []
+        # node unique name to its nodeId
+        self._node_ids_dict = {}
+        # Node id mast be incremented for each new branch
+        self._max_node_id = 0
+
+    def _calculate_relation(self, branch_as_list):
+        """
+        Get relation of branch_as_list to current branch.
+        :return: tuple. First argument could be: "same", "child", "parent" or "sibling"(need to start new tree)
+        Second argument is relative path from current branch to child if argument is child
+        """
+        if branch_as_list == self.current_branch:
+            return "same", None
+
+        hierarchy_name_len = len(branch_as_list)
+        current_branch_len = len(self.current_branch)
+
+        if hierarchy_name_len > current_branch_len and branch_as_list[0:current_branch_len] == self.current_branch:
+            return "child", branch_as_list[current_branch_len:]
+
+        if hierarchy_name_len < current_branch_len and self.current_branch[0:hierarchy_name_len] == branch_as_list:
+            return "parent", None
+
+        return "sibling", None
+
+    def _add_new_node(self, new_node_name):
+        """
+        Adds new node to branch
+        """
+        self.current_branch.append(new_node_name)
+        self._max_node_id += 1
+        self._node_ids_dict[".".join(self.current_branch)] = self._max_node_id
+
+    def level_opened(self, test_as_list, func_to_open):
+        """
+        To be called on test start.
+
+        :param test_as_list: test name splitted as list
+        :param func_to_open: func to be called if test can open new level
+        :return: None if new level opened, or tuple of command client should execute and try opening level again
+         Command is "open" (open provided level) or "close" (close it). Second item is test name as list
+        """
+        relation, relative_path = self._calculate_relation(test_as_list)
+        if relation == 'same':
+            return  # Opening same level?
+        if relation == 'child':
+            # If one level -- open new level gracefully
+            if len(relative_path) == 1:
+                self._add_new_node(relative_path[0])
+                func_to_open()
+                return None
+            else:
+                # Open previous level
+                return "open", self.current_branch + relative_path[0:1]
+        if relation == "sibling":
+            if self.current_branch:
+                # Different tree, close whole branch
+                return "close", self.current_branch
+            else:
+                return None
+        if relation == 'parent':
+            # Opening parent? Insane
+            pass
+
+    def level_closed(self, test_as_list, func_to_close):
+        """
+        To be called on test end or failure.
+
+        See level_opened doc.
+        """
+        relation, relative_path = self._calculate_relation(test_as_list)
+        if relation == 'same':
+            # Closing current level
+            func_to_close()
+            self.current_branch.pop()
+        if relation == 'child':
+            return None
+
+        if relation == 'sibling':
+            pass
+        if relation == 'parent':
+            return "close", self.current_branch
+
+    @property
+    def parent_branch(self):
+        return self.current_branch[:-1] if self.current_branch else None
+
+    def _get_node_id(self, branch):
+        return self._node_ids_dict[".".join(branch)]
+
+    @property
+    def node_ids(self):
+        """
+
+        :return: (current_node_id, parent_node_id)
+        """
+        current = self._get_node_id(self.current_branch)
+        parent = self._get_node_id(self.parent_branch) if self.parent_branch else "0"
+        return str(current), str(parent)
+
+    def close_all(self):
+        if not self.current_branch:
+            return None
+        return "close", self.current_branch
 
 
-def __jb_message(self, messageName, **properties):
-    try:
+TREE_MANAGER = _TreeManager()
+
+_old_service_messages = messages.TeamcityServiceMessages
+
+PARSE_FUNC = None
+
+
+class NewTeamcityServiceMessages(_old_service_messages):
+    def message(self, messageName, **properties):
+        if messageName == "enteredTheMatrix":
+            _old_service_messages.message(self, messageName, **properties)
+            return
         properties["locationHint"] = "python://{0}".format(properties["name"])
-    except KeyError:
-        pass
-    print("\n")
-    old_started(self, messageName, **properties)
+
+        # Shortcut for name
+        try:
+            properties["name"] = str(properties["name"]).split(".")[-1]
+        except IndexError:
+            pass
+
+        current, parent = TREE_MANAGER.node_ids
+        properties["nodeId"] = str(current)
+        properties["parentNodeId"] = str(parent)
+
+        # TODO: Dirty hack, to be fixed
+        # For some reason events are processed in background (invoke-n-wait) on Java side
+        # That leads to "test not opened" error sometimes
+        # Artificial sleep fixes this error until further investigation
+        time.sleep(0.05)
+        _old_service_messages.message(self, messageName, **properties)
+
+    def _test_to_list(self, test_name):
+        """
+        Splits test name to parts to use it as list.
+        It most cases dot is used, but runner may provide custom function
+        """
+        parts = test_name.split(".")
+        if not PARSE_FUNC:
+            return parts
+
+        result = []
+        for part in parts:
+            result += PARSE_FUNC(part)
+        return result
+
+    def _fix_setup_teardown_name(self, test_name):
+        """
+
+        Hack to rename setup and teardown methods to much real python signatures
+        """
+        try:
+            return {"test setup": "setUpClass", "test teardown": "tearDownClass"}[test_name]
+        except KeyError:
+            return test_name
+
+    def blockOpened(self, name, flowId=None):
+        self.testStarted(".".join(TREE_MANAGER.current_branch + [self._fix_setup_teardown_name(name)]))
+
+    def blockClosed(self, name, flowId=None):
+        self.testFinished(".".join(TREE_MANAGER.current_branch + [self._fix_setup_teardown_name(name)]))
+
+    def testStarted(self, testName, captureStandardOutput=None, flowId=None):
+        test_name_as_list = self._test_to_list(testName)
+        testName = ".".join(test_name_as_list)
+
+        def _write_start_message():
+            # testName, captureStandardOutput, flowId
+            args = {"name": testName, "captureStandardOutput": captureStandardOutput}
+            self.message("testStarted", **args)
+
+        commands = TREE_MANAGER.level_opened(self._test_to_list(testName), _write_start_message)
+        if commands:
+            self.do_command(commands[0], commands[1])
+            self.testStarted(testName, captureStandardOutput)
+
+    def testFailed(self, testName, message='', details='', flowId=None):
+        testName = ".".join(self._test_to_list(testName))
+        args = {"name": testName, "message": str(message),
+                     "details": details}
+        self.message("testFailed", **args)
+
+    def testFinished(self, testName, testDuration=None, flowId=None):
+        testName = ".".join(self._test_to_list(testName))
+
+        def _write_finished_message():
+            # testName, captureStandardOutput, flowId
+            current, parent = TREE_MANAGER.node_ids
+            args = {"nodeId": current, "parentNodeId": parent, "name": testName}
+
+            # TODO: Doc copy/paste with parent, extract
+            if testDuration is not None:
+                duration_ms = testDuration.days * 86400000 + \
+                              testDuration.seconds * 1000 + \
+                              int(testDuration.microseconds / 1000)
+                args["duration"] = str(duration_ms)
+
+            self.message("testFinished", **args)
+
+        commands = TREE_MANAGER.level_closed(self._test_to_list(testName), _write_finished_message)
+        if commands:
+            self.do_command(commands[0], commands[1])
+            self.testFinished(testName, testDuration)
+
+    def do_command(self, command, test):
+        """
+
+        Executes commands, returned by level_closed and level_opened
+        """
+        test_name = ".".join(test)
+        if command == "open":
+            self.testStarted(test_name)
+        else:
+            self.testFinished(test_name)
+
+    def close_all(self):
+        """
+
+        Closes all tests
+        """
+        commands = TREE_MANAGER.close_all()
+        if commands:
+            self.do_command(commands[0], commands[1])
+            self.close_all()
 
 
-messages.TeamcityServiceMessages.message = __jb_message
+messages.TeamcityServiceMessages = NewTeamcityServiceMessages
 
 
 # Monkeypatched
@@ -61,17 +293,19 @@ def jb_patch_separator(targets, fs_glue, python_glue, fs_to_python_glue):
                 return fs_part + fs_to_python_glue + python_path if python_path else fs_part
             if desc[2] == imp.PKG_DIRECTORY:
                 # Package
-                path = imp.load_module(m, fil, path, desc).__path__
+                path = imp.load_module(m, fil, path, desc).__path__[0]
         return target
 
     return map(_patch_target, targets)
 
 
-def jb_start_tests():
+def jb_start_tests(func_to_parse=None):
     """
     Parses arguments, starts protocol and returns tuple of arguments
 
     :return: (string with path or None, list of targets or None, list of additional arguments)
+    :param func_to_parse function that accepts each part of test name and returns list to be used instead of it.
+    It may return list with only one element (name itself) if name is the same or split names to several parts
     """
 
     # Handle additional args after --
@@ -87,10 +321,19 @@ def jb_start_tests():
     parser.add_argument('--target', help='Python target to run', action="append")
     namespace = parser.parse_args()
     del sys.argv[1:]  # Remove all args
-    messages.TeamcityServiceMessages().message('enteredTheMatrix')
+    global PARSE_FUNC
+    PARSE_FUNC = func_to_parse
+    NewTeamcityServiceMessages().message('enteredTheMatrix')
     return namespace.path, namespace.target, additional_args
 
 
+def _close_all_tests():
+    NewTeamcityServiceMessages().close_all()
+
+
+atexit.register(_close_all_tests)
+
+
 def jb_doc_args(framework_name, args):
     """
     Runner encouraged to report its arguments to user with aid of this function

python/src/com/jetbrains/python/run/AbstractPythonRunConfiguration.java

@@ -86,6 +86,10 @@ public abstract class AbstractPythonRunConfiguration<T extends AbstractPythonRun
     myMappingSettings = mappingSettings;
   }
 
+  public boolean isTestBased() {
+    return false;
+  }
+
   public static List<Module> getValidModules(Project project) {
     final Module[] modules = ModuleManager.getInstance(project).getModules();
     List<Module> result = Lists.newArrayList();
@@ -230,7 +234,7 @@ public abstract class AbstractPythonRunConfiguration<T extends AbstractPythonRun
     myAddContentRoots = addContentRoots == null || Boolean.parseBoolean(addContentRoots);
     final String addSourceRoots = JDOMExternalizerUtil.readField(element, "ADD_SOURCE_ROOTS");
     myAddSourceRoots = addSourceRoots == null || Boolean.parseBoolean(addSourceRoots);
-    if ( !mySkipModuleSerialization) {
+    if (!mySkipModuleSerialization) {
       getConfigurationModule().readExternal(element);
     }
 
@@ -247,7 +251,7 @@ public abstract class AbstractPythonRunConfiguration<T extends AbstractPythonRun
     EnvironmentVariablesComponent.readExternal(element, getEnvs());
   }
 
-  public void writeExternal(@NotNull  Element element) throws WriteExternalException {
+  public void writeExternal(@NotNull Element element) throws WriteExternalException {
     super.writeExternal(element);
     JDOMExternalizerUtil.writeField(element, "INTERPRETER_OPTIONS", myInterpreterOptions);
     writeEnvs(element);
@@ -256,7 +260,7 @@ public abstract class AbstractPythonRunConfiguration<T extends AbstractPythonRun
     JDOMExternalizerUtil.writeField(element, "IS_MODULE_SDK", Boolean.toString(myUseModuleSdk));
     JDOMExternalizerUtil.writeField(element, "ADD_CONTENT_ROOTS", Boolean.toString(myAddContentRoots));
     JDOMExternalizerUtil.writeField(element, "ADD_SOURCE_ROOTS", Boolean.toString(myAddSourceRoots));
-    if ( !mySkipModuleSerialization) {
+    if (!mySkipModuleSerialization) {
       getConfigurationModule().writeExternal(element);
     }
 

python/src/com/jetbrains/python/testing/PythonTRunnerConsoleProperties.java

@@ -17,8 +17,7 @@ package com.jetbrains.python.testing;
 
 import com.intellij.execution.Executor;
 import com.intellij.execution.configurations.ModuleRunConfiguration;
-import com.intellij.execution.testframework.sm.runner.SMTRunnerConsoleProperties;
-import com.intellij.execution.testframework.sm.runner.SMTestLocator;
+import com.intellij.execution.testframework.sm.runner.*;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
@@ -53,4 +52,23 @@ public class PythonTRunnerConsoleProperties extends SMTRunnerConsoleProperties {
   public SMTestLocator getTestLocator() {
     return myLocator;
   }
+
+  /**
+   * Makes configuration id-based,
+   * @see GeneralIdBasedToSMTRunnerEventsConvertor
+   */
+  final void makeIdTestBased() {
+    setIdBasedTestTree(true);
+    getProject().getMessageBus().connect(this)
+      .subscribe(SMTRunnerEventsListener.TEST_STATUS, new SMTRunnerEventsAdapter() {
+        @Override
+        public void onTestFailed(@NotNull final SMTestProxy test) {
+          SMTestProxy currentTest = test.getParent();
+          while (currentTest != null) {
+            currentTest.setTestFailed(" ", null, false);
+            currentTest = currentTest.getParent();
+          }
+        }
+      });
+  }
 }

python/src/com/jetbrains/python/testing/PythonTestCommandLineStateBase.java

@@ -85,7 +85,11 @@ public abstract class PythonTestCommandLineStateBase<T extends AbstractPythonRun
   }
 
   protected PythonTRunnerConsoleProperties createConsoleProperties(Executor executor) {
-    return new PythonTRunnerConsoleProperties(myConfiguration, executor, true, getTestLocator());
+    final PythonTRunnerConsoleProperties properties = new PythonTRunnerConsoleProperties(myConfiguration, executor, true, getTestLocator());
+    if (myConfiguration.isTestBased()) {
+      properties.makeIdTestBased();
+    }
+    return properties;
   }
 
   @Nullable

python/src/com/jetbrains/python/testing/PythonUnitTestUtil.java

@@ -118,8 +118,10 @@ public class PythonUnitTestUtil {
 
   public static boolean isTestCaseFunction(PyFunction function, boolean checkAssert) {
     final String name = function.getName();
-    if (name == null || !TEST_MATCH_PATTERN.matcher(name).find()) {
-      return false;
+    if (name != null && TEST_MATCH_PATTERN.matcher(name).find()) {
+      // Since there are a lot of ways to launch assert in modern frameworks,
+      // we assume any function with "test" word in name is test
+      return true;
     }
     if (function.getContainingClass() != null) {
       if (isTestCaseClass(function.getContainingClass(), null)) return true;

python/src/com/jetbrains/python/testing/universalTests/PyUniversalTests.kt

@@ -179,6 +179,8 @@ abstract class PyUniversalTestConfiguration(project: Project,
   val legacyConfigurationAdapter = PyUniversalTestLegacyConfigurationAdapter(this)
 
 
+  override fun isTestBased() = true
+
   private fun getTestSpecForPythonTarget(location: Location<*>): List<String> {
     if (location !is PsiLocation) {
       return emptyList()