New test runners: tests moved to new configuration:

* Some tests fail. Will fix.
* py files renamed to prevent them from unexpected discovering
* "prefixes" added to tests as a way to provide different targets
* verbosity improved
* suite changed to "non leaf" since there are no suites in new runners

python/src/com/jetbrains/python/testing/universalTests/PyUniversalTests.kt

@@ -316,6 +316,12 @@ object PyUniversalTestsConfigurationProducer : RunConfigurationProducer<PyUniver
     return cloneTemplateConfigurationStatic(context, findConfigurationFactoryFromSettings(context.module))
   }
 
+  override fun createConfigurationFromContext(context: ConfigurationContext?): ConfigurationFromContext? {
+    // Since we need module, no need to even try to create config with out of it
+    context?.module?: return null
+    return super.createConfigurationFromContext(context)
+  }
+
   override fun findOrCreateConfigurationFromContext(context: ConfigurationContext?): ConfigurationFromContext? {
     if (!isNewTestsModeEnabled()) {
       return null

python/testSrc/com/jetbrains/env/PyAbstractTestProcessRunner.java

@@ -33,6 +33,7 @@ import com.intellij.openapi.application.ModalityState;
 import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.openapi.project.Project;
 import com.intellij.openapi.util.Ref;
+import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.psi.search.GlobalSearchScope;
 import com.intellij.testFramework.EdtTestUtil;
 import com.jetbrains.python.run.AbstractPythonRunConfigurationParams;
@@ -99,8 +100,8 @@ public class PyAbstractTestProcessRunner<CONF_T extends AbstractPythonRunConfigu
    */
   public void assertAllTestsPassed() {
     final String consoleText = getAllConsoleText();
-    Assert.assertEquals(consoleText, 0, myProxyManager.getProxy().getChildren(Filter.NOT_PASSED).size());
-    Assert.assertEquals(consoleText, 0, getFailedTestsCount());
+    Assert.assertEquals(getFormattedTestTree() + consoleText, 0, myProxyManager.getProxy().getChildren(Filter.NOT_PASSED).size());
+    Assert.assertEquals(getFormattedTestTree() + consoleText, 0, getFailedTestsCount());
   }
 
   /**
@@ -125,6 +126,30 @@ public class PyAbstractTestProcessRunner<CONF_T extends AbstractPythonRunConfigu
     return myProxyManager.getProxy();
   }
 
+  /**
+   * @return Test tree using poorman's graphics
+   */
+  @NotNull
+  public final String getFormattedTestTree() {
+    final StringBuilder builder = new StringBuilder("Test tree:\n");
+
+    formatLevel(getTestProxy(), 0, builder);
+
+    return builder.toString();
+  }
+
+  private static void formatLevel(@NotNull final SMTestProxy test, final int level, @NotNull final StringBuilder builder) {
+    builder.append(StringUtil.repeat(".", level));
+    builder.append(test.getName());
+    if (test.isLeaf()) {
+      builder.append(test.isPassed() ? "(+)" : "(-)");
+    }
+    builder.append('\n');
+    for (SMTestProxy child : test.getChildren()) {
+      formatLevel(child, level + 1, builder);
+    }
+  }
+
 
   /**
    * Searches for test by its name recursevly in test, passed as arumuent.

python/testSrc/com/jetbrains/env/SMRootTestsCounter.java

@@ -31,7 +31,7 @@ final class SMRootTestsCounter {
   static final Filter<SMTestProxy> NOT_SUIT = new Filter<SMTestProxy>() {
     @Override
     public boolean shouldAccept(final SMTestProxy test) {
-      return !test.isSuite();
+      return test.isLeaf();
     }
   };
 

python/testSrc/com/jetbrains/env/python/testing/PyUnitTestProcessWithConsoleTestTask.java

@@ -40,4 +40,6 @@ abstract class PyUnitTestProcessWithConsoleTestTask extends PyProcessWithConsole
   protected PyUnitTestProcessRunner createProcessRunner() throws Exception {
     return new PyUnitTestProcessRunner(myScriptName, 0);
   }
+
+
 }

python/testSrc/com/jetbrains/env/python/testing/PythonPyTestingTest.java

@@ -10,7 +10,8 @@ import com.jetbrains.env.ut.PyTestTestProcessRunner;
 import com.jetbrains.python.sdkTools.SdkCreationType;
 import com.jetbrains.python.testing.PythonTestConfigurationsModel;
 import com.jetbrains.python.testing.pytest.PyTestConfigurationProducer;
-import com.jetbrains.python.testing.pytest.PyTestRunConfiguration;
+import com.jetbrains.python.testing.universalTests.PyUniversalPyTestConfiguration;
+import com.jetbrains.python.testing.universalTests.TestTargetType;
 import org.hamcrest.Matchers;
 import org.jetbrains.annotations.NotNull;
 import org.junit.Assert;
@@ -66,12 +67,13 @@ public class PythonPyTestingTest extends PyEnvTestCase {
       protected PyTestTestProcessRunner createProcessRunner() throws Exception {
         return new PyTestTestProcessRunner("", 0) {
           @Override
-          protected void configurationCreatedAndWillLaunch(@NotNull final PyTestRunConfiguration configuration) throws IOException {
+          protected void configurationCreatedAndWillLaunch(@NotNull final PyUniversalPyTestConfiguration configuration) throws IOException {
             super.configurationCreatedAndWillLaunch(configuration);
             configuration.setWorkingDirectory(null);
             final VirtualFile fullFilePath = myFixture.getTempDirFixture().getFile("dir_test.py");
             assert fullFilePath != null : String.format("No dir_test.py in %s", myFixture.getTempDirFixture().getTempDirPath());
-            configuration.setTestToRun(fullFilePath.getPath());
+            configuration.getTarget().setTarget(fullFilePath.getPath());
+            configuration.getTarget().setTargetType(TestTargetType.PATH);
           }
         };
       }

python/testSrc/com/jetbrains/env/python/testing/PythonUnitTestingTest.java

@@ -23,6 +23,7 @@ import com.intellij.testFramework.EdtTestUtil;
 import com.intellij.testFramework.fixtures.CodeInsightTestFixture;
 import com.jetbrains.env.PyEnvTestCase;
 import com.jetbrains.env.Staging;
+import com.jetbrains.env.ut.PyScriptTestProcessRunner;
 import com.jetbrains.env.ut.PyUnitTestProcessRunner;
 import com.jetbrains.python.PyBundle;
 import com.jetbrains.python.psi.LanguageLevel;
@@ -59,8 +60,8 @@ public final class PythonUnitTestingTest extends PyEnvTestCase {
                                       @NotNull final String stdout,
                                       @NotNull final String stderr,
                                       @NotNull final String all) {
-        assertEquals(2, runner.getAllTestsCount());
-        assertEquals(2, runner.getPassedTestsCount());
+        assertEquals(runner.getFormattedTestTree(), 2, runner.getAllTestsCount());
+        assertEquals(runner.getFormattedTestTree(), 2, runner.getPassedTestsCount());
         runner.assertAllTestsPassed();
       }
     });
@@ -91,6 +92,7 @@ public final class PythonUnitTestingTest extends PyEnvTestCase {
 
   /**
    * Ensure rerun test works even if test is declared in parent
+   * See https://github.com/JetBrains/teamcity-messages/issues/117
    */
   @Test
   public void testRerunDerivedClass() throws Exception {
@@ -201,9 +203,9 @@ public final class PythonUnitTestingTest extends PyEnvTestCase {
                                       @NotNull final String stdout,
                                       @NotNull final String stderr,
                                       @NotNull final String all) {
-        assertEquals(3, runner.getAllTestsCount());
-        assertEquals(1, runner.getPassedTestsCount());
-        assertEquals(2, runner.getFailedTestsCount());
+        assertEquals(runner.getFormattedTestTree(), 3, runner.getAllTestsCount());
+        assertEquals(runner.getFormattedTestTree(), 1, runner.getPassedTestsCount());
+        assertEquals(runner.getFormattedTestTree(), 2, runner.getFailedTestsCount());
       }
     });
   }
@@ -213,7 +215,7 @@ public final class PythonUnitTestingTest extends PyEnvTestCase {
    */
   @Test
   public void testUTRunnerByPattern() {
-    runPythonTest(new PyUnitTestProcessWithConsoleTestTask("/testRunner/env/unit", "./_args_separator_*pattern.py") {
+    runPythonTest(new PyUnitTestProcessWithConsoleTestTask("/testRunner/env/unit", PyUnitTestProcessRunner.TEST_PATTERN_PREFIX + "*pattern.py") {
 
 
       @Override
@@ -221,9 +223,9 @@ public final class PythonUnitTestingTest extends PyEnvTestCase {
                                       @NotNull final String stdout,
                                       @NotNull final String stderr,
                                       @NotNull final String all) {
-        assertEquals(4, runner.getAllTestsCount());
-        assertEquals(2, runner.getPassedTestsCount());
-        assertEquals(2, runner.getFailedTestsCount());
+        assertEquals(runner.getFormattedTestTree(),4, runner.getAllTestsCount());
+        assertEquals(runner.getFormattedTestTree(),2, runner.getPassedTestsCount());
+        assertEquals(runner.getFormattedTestTree(),2, runner.getFailedTestsCount());
       }
     });
   }
@@ -246,12 +248,12 @@ public final class PythonUnitTestingTest extends PyEnvTestCase {
         assertEquals(1, runner.getFailedTestsCount());
       }
     });
-
   }
 
   @Test
   public void testClass() {
-    runPythonTest(new PyUnitTestProcessWithConsoleTestTask("/testRunner/env/unit", "test_file.py::GoodTest") {
+    runPythonTest(new PyUnitTestProcessWithConsoleTestTask("/testRunner/env/unit",
+                                                           PyScriptTestProcessRunner.TEST_TARGET_PREFIX + "test_file.GoodTest") {
 
       @Override
       protected void checkTestResults(@NotNull final PyUnitTestProcessRunner runner,
@@ -266,7 +268,9 @@ public final class PythonUnitTestingTest extends PyEnvTestCase {
 
   @Test
   public void testMethod() {
-    runPythonTest(new PyUnitTestProcessWithConsoleTestTask("/testRunner/env/unit", "test_file.py::GoodTest::test_passes") {
+    runPythonTest(new PyUnitTestProcessWithConsoleTestTask("/testRunner/env/unit",
+                                                           PyScriptTestProcessRunner.TEST_TARGET_PREFIX +
+                                                           "test_file.GoodTest.test_passes") {
 
       @Override
       protected void checkTestResults(@NotNull final PyUnitTestProcessRunner runner,
@@ -313,7 +317,7 @@ public final class PythonUnitTestingTest extends PyEnvTestCase {
                                         runner.getAllConsoleText()),
                           fileNames.size() >= 3);
         // UnitTest highlights file name
-        Assert.assertThat("Bad line highlighted", fileNames, everyItem(endsWith(fileName)));
+        Assert.assertThat("Bad line highlighted", fileNames, hasItem(endsWith(fileName)));
       }
     });
   }
@@ -336,14 +340,14 @@ public final class PythonUnitTestingTest extends PyEnvTestCase {
   @Test
   @Staging
   public void testRelativeImports() {
-    runPythonTest(new PyUnitTestProcessWithConsoleTestTask("/testRunner/env/unit/relativeImports", "relative_imports/tests/test_imps.py") {
+    runPythonTest(new PyUnitTestProcessWithConsoleTestTask("/testRunner/env/unit/relativeImports", PyUnitTestProcessRunner.TEST_PATTERN_PREFIX + "test_imps.py") {
       @Override
       protected void checkTestResults(@NotNull final PyUnitTestProcessRunner runner,
                                       @NotNull final String stdout,
                                       @NotNull final String stderr,
                                       @NotNull final String all) {
-        assertEquals(1, runner.getAllTestsCount());
-        assertEquals(1, runner.getPassedTestsCount());
+        assertEquals(runner.getFormattedTestTree(),1, runner.getAllTestsCount());
+        assertEquals(runner.getFormattedTestTree(),1, runner.getPassedTestsCount());
       }
     });
   }

python/testSrc/com/jetbrains/env/ut/PyNoseTestProcessRunner.java

@@ -16,8 +16,8 @@
 package com.jetbrains.env.ut;
 
 import com.jetbrains.env.ProcessWithConsoleRunner;
-import com.jetbrains.python.testing.PythonTestConfigurationType;
-import com.jetbrains.python.testing.nosetest.PythonNoseTestRunConfiguration;
+import com.jetbrains.python.testing.universalTests.PyUniversalNoseTestConfiguration;
+import com.jetbrains.python.testing.universalTests.PyUniversalNoseTestFactory;
 import org.jetbrains.annotations.NotNull;
 
 /**
@@ -25,9 +25,9 @@ import org.jetbrains.annotations.NotNull;
  *
  * @author Ilya.Kazakevich
  */
-public class PyNoseTestProcessRunner extends PyScriptTestProcessRunner<PythonNoseTestRunConfiguration> {
+public class PyNoseTestProcessRunner extends PyScriptTestProcessRunner<PyUniversalNoseTestConfiguration> {
   public PyNoseTestProcessRunner(@NotNull final String scriptName, final int timesToRerunFailedTests) {
-    super(PythonTestConfigurationType.getInstance().LEGACY_NOSETEST_FACTORY,
-          PythonNoseTestRunConfiguration.class, scriptName, timesToRerunFailedTests);
+    super(PyUniversalNoseTestFactory.INSTANCE,
+          PyUniversalNoseTestConfiguration.class, scriptName, timesToRerunFailedTests);
   }
 }

python/testSrc/com/jetbrains/env/ut/PyScriptTestProcessRunner.java

@@ -20,19 +20,30 @@ import com.jetbrains.env.ConfigurationBasedProcessRunner;
 import com.jetbrains.env.PyAbstractTestProcessRunner;
 import com.jetbrains.python.run.AbstractPythonRunConfigurationParams;
 import com.jetbrains.python.testing.AbstractPythonTestRunConfigurationParams;
+import com.jetbrains.python.testing.universalTests.ConfigurationTarget;
+import com.jetbrains.python.testing.universalTests.PyUniversalTestConfiguration;
+import com.jetbrains.python.testing.universalTests.TestTargetType;
 import org.jetbrains.annotations.NotNull;
 
 import java.io.IOException;
 
 /**
- * {@link PyAbstractTestProcessRunner} to run script-bases tests
+ * {@link PyAbstractTestProcessRunner} to run script-bases tests.
+ *
+ * Since this class only supports scripts {@link #myScriptName}, one may use
+ * {@link #TEST_TARGET_PREFIX} as prefix to provide python qname for new test runners
  *
  * @author Ilya.Kazakevich
  */
-public class PyScriptTestProcessRunner<CONF_T extends AbstractPythonRunConfigurationParams & AbstractPythonTestRunConfigurationParams>
+public class PyScriptTestProcessRunner<CONF_T extends AbstractPythonRunConfigurationParams>
   extends PyAbstractTestProcessRunner<CONF_T> {
+
+  /**
+   * Prepent script name with it if you use python test qname but not script name
+   */
+  public static final String TEST_TARGET_PREFIX = "test:";
   @NotNull
-  private final String myScriptName;
+  protected final String myScriptName;
 
   /**
    * @param scriptName name of script to run
@@ -50,6 +61,18 @@ public class PyScriptTestProcessRunner<CONF_T extends AbstractPythonRunConfigura
   @Override
   protected void configurationCreatedAndWillLaunch(@NotNull final CONF_T configuration) throws IOException {
     super.configurationCreatedAndWillLaunch(configuration);
-    configuration.setScriptName(myScriptName);
+    if (configuration instanceof AbstractPythonTestRunConfigurationParams) {
+      ((AbstractPythonTestRunConfigurationParams)configuration).setScriptName(myScriptName);
+    }
+    if (configuration instanceof PyUniversalTestConfiguration) {
+      final ConfigurationTarget target = ((PyUniversalTestConfiguration)configuration).getTarget();
+      if (myScriptName.startsWith(TEST_TARGET_PREFIX)) {
+        target.setTarget(myScriptName.substring(TEST_TARGET_PREFIX.length()));
+        target.setTargetType(TestTargetType.PYTHON);
+      } else {
+        target.setTarget(myScriptName);
+        target.setTargetType(TestTargetType.PATH);
+      }
+    }
   }
 }

python/testSrc/com/jetbrains/env/ut/PyTestTestProcessRunner.java

@@ -16,26 +16,19 @@
 package com.jetbrains.env.ut;
 
 import com.jetbrains.env.ProcessWithConsoleRunner;
-import com.jetbrains.python.testing.PythonTestConfigurationType;
-import com.jetbrains.python.testing.pytest.PyTestRunConfiguration;
+import com.jetbrains.python.testing.universalTests.PyUniversalPyTestConfiguration;
+import com.jetbrains.python.testing.universalTests.PyUniversalPyTestFactory;
 import org.jetbrains.annotations.NotNull;
 
-import java.io.IOException;
 
 /**
  * {@link ProcessWithConsoleRunner} to pytest
  *
  * @author Ilya.Kazakevich
  */
-public class PyTestTestProcessRunner extends PyScriptTestProcessRunner<PyTestRunConfiguration> {
+public class PyTestTestProcessRunner extends PyScriptTestProcessRunner<PyUniversalPyTestConfiguration> {
   public PyTestTestProcessRunner(@NotNull final String scriptName, final int timesToRerunFailedTests) {
-    super(PythonTestConfigurationType.getInstance().LEGACY_PYTEST_FACTORY,
-          PyTestRunConfiguration.class, scriptName, timesToRerunFailedTests);
-  }
-
-  @Override
-  protected void configurationCreatedAndWillLaunch(@NotNull final PyTestRunConfiguration configuration) throws IOException {
-    super.configurationCreatedAndWillLaunch(configuration);
-    configuration.setTestToRun(configuration.getScriptName());
+    super(PyUniversalPyTestFactory.INSTANCE,
+          PyUniversalPyTestConfiguration.class, scriptName, timesToRerunFailedTests);
   }
 }

python/testSrc/com/jetbrains/env/ut/PyUnitTestProcessRunner.java

@@ -16,18 +16,36 @@
 package com.jetbrains.env.ut;
 
 import com.jetbrains.env.ProcessWithConsoleRunner;
-import com.jetbrains.python.testing.PythonTestConfigurationType;
-import com.jetbrains.python.testing.unittest.PythonUnitTestRunConfiguration;
+import com.jetbrains.python.testing.universalTests.PyUniversalUnitTestConfiguration;
+import com.jetbrains.python.testing.universalTests.PyUniversalUnitTestFactory;
+import com.jetbrains.python.testing.universalTests.TestTargetType;
 import org.jetbrains.annotations.NotNull;
 
+import java.io.IOException;
+
 /**
  * {@link ProcessWithConsoleRunner} to run unittest
  *
  * @author Ilya.Kazakevich
  */
-public class PyUnitTestProcessRunner extends PyScriptTestProcessRunner<PythonUnitTestRunConfiguration> {
+public class PyUnitTestProcessRunner extends PyScriptTestProcessRunner<PyUniversalUnitTestConfiguration> {
+  /**
+   * Prefix to use test pattern. See {@link #TEST_PATTERN_PREFIX} doc because it is similar
+   */
+  public static final String TEST_PATTERN_PREFIX = "pattern:";
+
   public PyUnitTestProcessRunner(@NotNull final String scriptName, final int timesToRerunFailedTests) {
-    super(PythonTestConfigurationType.getInstance().LEGACY_UNITTEST_FACTORY,
-          PythonUnitTestRunConfiguration.class, scriptName, timesToRerunFailedTests);
+    super(PyUniversalUnitTestFactory.INSTANCE,
+          PyUniversalUnitTestConfiguration.class, scriptName, timesToRerunFailedTests);
+  }
+
+  @Override
+  protected void configurationCreatedAndWillLaunch(@NotNull PyUniversalUnitTestConfiguration configuration) throws IOException {
+    super.configurationCreatedAndWillLaunch(configuration);
+    if (myScriptName.startsWith(TEST_PATTERN_PREFIX)) {
+      configuration.getTarget().setTargetType(TestTargetType.PATH);
+      configuration.getTarget().setTarget(".");
+      configuration.setPattern(myScriptName.substring(TEST_PATTERN_PREFIX.length()));
+    }
   }
 }