PY-19015: fix for "rerun" for failed tox tests

Tox fetches env id (like py27) and provides it to configuration.

python/src/com/jetbrains/python/run/AbstractPythonRunConfiguration.java

@@ -469,4 +469,12 @@ public abstract class AbstractPythonRunConfiguration<T extends AbstractPythonRun
   public boolean isCompileBeforeLaunchAddedByDefault() {
     return false;
   }
+
+  /**
+   * Adds test specs (like method, class, script, etc) to list of runner parameters.
+   */
+  public void addTestSpecsAsParameters(@NotNull final ParamsGroup paramsGroup, @NotNull final List<String> testSpecs) {
+    // By default we simply add them as arguments
+    paramsGroup.addParameters(testSpecs);
+  }
 }

python/src/com/jetbrains/python/testing/PythonTestCommandLineStateBase.java

@@ -158,7 +158,7 @@ public abstract class PythonTestCommandLineStateBase extends PythonCommandLineSt
     assert scriptParams != null;
     getRunner().addToGroup(scriptParams, cmd);
     addBeforeParameters(cmd);
-    scriptParams.addParameters(getTestSpecs());
+    myConfiguration.addTestSpecsAsParameters(scriptParams, getTestSpecs());
     addAfterParameters(cmd);
   }
 

python/src/com/jetbrains/python/testing/tox/PyToxCommandLineState.java

@@ -15,16 +15,13 @@
  */
 package com.jetbrains.python.testing.tox;
 
-import com.intellij.execution.configurations.GeneralCommandLine;
-import com.intellij.execution.configurations.ParamsGroup;
 import com.intellij.execution.runners.ExecutionEnvironment;
-import com.intellij.openapi.util.text.StringUtil;
 import com.jetbrains.python.HelperPackage;
 import com.jetbrains.python.PythonHelper;
 import com.jetbrains.python.testing.PythonTestCommandLineStateBase;
 import org.jetbrains.annotations.NotNull;
 
-import java.util.Collections;
+import java.util.Arrays;
 import java.util.List;
 
 /**
@@ -47,23 +44,9 @@ class PyToxCommandLineState extends PythonTestCommandLineStateBase {
     return PythonHelper.TOX;
   }
 
-
-  @Override
-  public GeneralCommandLine generateCommandLine() {
-    final GeneralCommandLine line = super.generateCommandLine();
-    final ParamsGroup group = line.getParametersList().getParamsGroup(GROUP_SCRIPT);
-    assert group != null : "No group " + GROUP_SCRIPT;
-    final String[] envs = myConfiguration.getRunOnlyEnvs();
-    if (envs.length > 0) {
-      group.addParameter(String.format("-e %s", StringUtil.join(envs, ",")));
-    }
-    group.addParameters(myConfiguration.getArguments());
-    return line;
-  }
-
   @NotNull
   @Override
   protected List<String> getTestSpecs() {
-    return Collections.emptyList();
+    return Arrays.asList(myConfiguration.getRunOnlyEnvs());
   }
 }

python/src/com/jetbrains/python/testing/tox/PyToxConfiguration.java

@@ -16,12 +16,17 @@
 package com.jetbrains.python.testing.tox;
 
 import com.intellij.execution.Executor;
+import com.intellij.execution.Location;
+import com.intellij.execution.configurations.ParamsGroup;
 import com.intellij.execution.configurations.RunProfileState;
 import com.intellij.execution.runners.ExecutionEnvironment;
+import com.intellij.execution.testframework.AbstractTestProxy;
 import com.intellij.openapi.options.SettingsEditor;
 import com.intellij.openapi.project.Project;
 import com.intellij.openapi.util.InvalidDataException;
 import com.intellij.openapi.util.WriteExternalException;
+import com.intellij.openapi.util.text.StringUtil;
+import com.intellij.openapi.vfs.VirtualFileManager;
 import com.intellij.util.ArrayUtil;
 import com.intellij.util.xmlb.SkipEmptySerializationFilter;
 import com.intellij.util.xmlb.XmlSerializer;
@@ -33,6 +38,8 @@ import org.jdom.Element;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
+import java.util.List;
+
 /**
  * @author Ilya.Kazakevich
  */
@@ -99,4 +106,30 @@ public final class PyToxConfiguration extends AbstractPythonRunConfiguration<PyT
   public RunProfileState getState(@NotNull final Executor executor, @NotNull final ExecutionEnvironment environment) {
     return new PyToxCommandLineState(this, environment);
   }
+
+  @Nullable
+  @Override
+  public String getTestSpec(@NotNull final Location<?> location, @NotNull final AbstractTestProxy failedTest) {
+
+    AbstractTestProxy test = failedTest;
+    while (test != null) {
+      final String url = test.getLocationUrl();
+      if (url == null) {
+        continue;
+      }
+      final String protocol = VirtualFileManager.extractProtocol(url);
+      if (PyToxTestLocator.PROTOCOL_ID.equals(protocol)) {
+        return VirtualFileManager.extractPath(url);
+      }
+      test = test.getParent();
+    }
+    return null;
+  }
+
+  @Override
+  public void addTestSpecsAsParameters(@NotNull final ParamsGroup paramsGroup, @NotNull final List<String> testSpecs) {
+    if (!testSpecs.isEmpty()) {
+      paramsGroup.addParameter(String.format("-e %s", StringUtil.join(testSpecs, ",")));
+    }
+  }
 }

python/src/com/jetbrains/python/testing/tox/PyToxTestLocator.java

@@ -39,11 +39,12 @@ public final class PyToxTestLocator implements PythonTestLocator {
 
   private static final String DUMMY_FILE_PADDING = "#env";
   private static final Key<String> ENV_NAME_KEY = Key.create("ENV_NAME");
+  static final String PROTOCOL_ID = "tox_env";
 
   @NotNull
   @Override
   public String getProtocolId() {
-    return "tox_env";
+    return PROTOCOL_ID;
   }
 
   @NotNull

python/testData/toxtest/toxConcreteEnv/tox.ini

@@ -0,0 +1,5 @@
+[tox]
+skipsdist=True
+envlist = py26, py27, py32, py34
+[testenv]
+commands=python -c "print 1"

python/testData/toxtest/toxRerun/tox.ini

@@ -0,0 +1,5 @@
+[tox]
+skipsdist=True
+envlist = py26, py27, py32, py34
+[testenv]
+commands=python -c "print 1"

python/testSrc/com/jetbrains/env/ConfigurationBasedProcessRunner.java

@@ -93,6 +93,8 @@ public abstract class ConfigurationBasedProcessRunner<CONF_T extends AbstractPyt
                         @NotNull final ProcessListener processListener,
                         @NotNull final String tempWorkingPath)
     throws ExecutionException {
+    ensureConsoleOk(myConsole);
+
     // Do not create new environment from factory, if child provided environment to rerun
     final ExecutionEnvironment executionEnvironment =
       // TODO: RENAME
@@ -136,6 +138,21 @@ public abstract class ConfigurationBasedProcessRunner<CONF_T extends AbstractPyt
     });
   }
 
+  /**
+   * {@link PyProcessWithConsoleTestTask#createProcessRunner()} should always return new runner.
+   * But some buggy code returns runner from prev. rerun with stale project in console.
+   * This code checks it.
+   */
+  private static void ensureConsoleOk(@Nullable final ConsoleViewImpl console) {
+    if (console == null) { // Not set yet
+      return;
+    }
+    if (console.getProject().isDisposed()) {
+      throw new AssertionError(
+        "=== Console is stale. Make sure you did not cache and reuse runner from prev. run. See this method doc for more info === ");
+    }
+  }
+
   @NotNull
   private ExecutionEnvironment createExecutionEnvironment(@NotNull final String sdkPath, @NotNull final Project project, @NotNull final String workingDir)
     throws ExecutionException {

python/testSrc/com/jetbrains/env/PyProcessWithConsoleTestTask.java

@@ -181,7 +181,8 @@ public abstract class PyProcessWithConsoleTestTask<T extends ProcessWithConsoleR
 
 
   /**
-   * @return process runner to be used to run process and fetch console
+   * @return process runner to be used to run process and fetch console.
+   * <strong>Always</strong> create new runner, to prevent stale artifacts on reruns.
    */
   @NotNull
   protected abstract T createProcessRunner() throws Exception;

python/testSrc/com/jetbrains/env/PyToxTest.java

@@ -23,13 +23,17 @@ import com.intellij.openapi.util.Pair;
 import com.jetbrains.python.sdkTools.SdkCreationType;
 import com.jetbrains.python.testing.tox.PyToxConfiguration;
 import com.jetbrains.python.testing.tox.PyToxConfigurationFactory;
+import com.jetbrains.python.testing.tox.PyToxTestTools;
 import org.hamcrest.Matchers;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.junit.Assert;
 import org.junit.Test;
 
+import java.io.IOException;
 import java.util.*;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
 
 /**
  * Ensure tox runner works
@@ -47,7 +51,7 @@ public final class PyToxTest extends PyEnvTestCase {
   @Test
   public void testToxSimpleRun() {
     runPythonTest(new MyPyProcessWithConsoleTestTask("/toxtest/toxSimpleRun/", 2,
-                                                     new MyTestProcessRunner(),
+                                                     () -> new MyTestProcessRunner(),
                                                      Arrays.asList(
                                                        // Should fail, no skip in 26
                                                        Pair.create("py26", new InterpreterExpectations(
@@ -63,7 +67,7 @@ public final class PyToxTest extends PyEnvTestCase {
   @Test
   public void testToxNose() {
     runPythonTest(new MyPyProcessWithConsoleTestTask("/toxtest/toxNose/", 1,
-                                                     new MyTestProcessRunner(),
+                                                     () -> new MyTestProcessRunner(),
                                                      Arrays.asList(
                                                        Pair.create("py26", new InterpreterExpectations("", true)),
                                                        Pair.create("py27", new InterpreterExpectations("", true)),
@@ -81,7 +85,7 @@ public final class PyToxTest extends PyEnvTestCase {
   @Test
   public void testToxPyTest() {
     runPythonTest(new MyPyProcessWithConsoleTestTask("/toxtest/toxPyTest/", 1,
-                                                     new MyTestProcessRunner(),
+                                                     () -> new MyTestProcessRunner(),
                                                      Arrays.asList(
                                                        Pair.create("py26", new InterpreterExpectations("", true)),
                                                        Pair.create("py27", new InterpreterExpectations("", true)),
@@ -99,7 +103,7 @@ public final class PyToxTest extends PyEnvTestCase {
   @Test
   public void testToxUnitTest() {
     runPythonTest(new MyPyProcessWithConsoleTestTask("/toxtest/toxUnitTest/", 1,
-                                                     new MyTestProcessRunner(),
+                                                     () -> new MyTestProcessRunner(),
                                                      Arrays.asList(
                                                        Pair.create("py26", new InterpreterExpectations("", true)),
                                                        Pair.create("py27", new InterpreterExpectations("", true)),
@@ -117,7 +121,7 @@ public final class PyToxTest extends PyEnvTestCase {
   @Test
   public void textToxOneInterpreter() throws Exception {
     runPythonTest(new MyPyProcessWithConsoleTestTask("/toxtest/toxOneInterpreter/", 0,
-                                                     new MyTestProcessRunner(),
+                                                     () -> new MyTestProcessRunner(),
                                                      Arrays.asList(
                                                        Pair.create("py26", new InterpreterExpectations("", true)),
                                                        Pair.create("py27", new InterpreterExpectations("ython 2.7", true)),
@@ -135,7 +139,7 @@ public final class PyToxTest extends PyEnvTestCase {
   @Test
   public void testDoubleRun() throws Exception {
     runPythonTest(new MyPyProcessWithConsoleTestTask("/toxtest/toxDoubleRun/", 1,
-                                                     new MyTestProcessRunner(),
+                                                     () -> new MyTestProcessRunner(),
                                                      Collections.singletonList(
                                                        Pair.create("py27", new InterpreterExpectations("", true))
                                                      ),
@@ -149,7 +153,7 @@ public final class PyToxTest extends PyEnvTestCase {
   @Test
   public void testToxSuccessTest() {
     runPythonTest(new MyPyProcessWithConsoleTestTask("/toxtest/toxSuccess/", 1,
-                                                     new MyTestProcessRunner(),
+                                                     () -> new MyTestProcessRunner(),
                                                      Arrays.asList(
                                                        Pair.create("py26", new InterpreterExpectations("I am 2.6", true)),
                                                        Pair.create("py27", new InterpreterExpectations("I am 2.7", true)),
@@ -161,29 +165,84 @@ public final class PyToxTest extends PyEnvTestCase {
     );
   }
 
+  /**
+   * Ensures rerun works for tox
+   */
+  @Test
+  public void testEnvRerun() throws Exception {
+    runPythonTest(new MyPyProcessWithConsoleTestTask("/toxtest/toxConcreteEnv/", 0,
+                                                     () -> new MyTestProcessRunner(1),
+                                                     Arrays.asList(
+                                                       Pair.create("py32", new InterpreterExpectations("", false)),
+                                                       Pair.create("py34", new InterpreterExpectations("", false))
+                                                     ),
+                                                     Integer.MAX_VALUE)
+    );
+  }
+
+  /**
+   * Provide certain env and check it is launched
+   */
+  @Test
+  public void testConcreteEnv() throws Exception {
+    final String[] envsToRun = {"py27", "py34"};
+    runPythonTest(
+      new PyProcessWithConsoleTestTask<PyAbstractTestProcessRunner<PyToxConfiguration>>("/toxtest/toxSuccess/", SdkCreationType.EMPTY_SDK) {
+        @NotNull
+        @Override
+        protected PyAbstractTestProcessRunner<PyToxConfiguration> createProcessRunner() throws Exception {
+          return new PyAbstractTestProcessRunner<PyToxConfiguration>(PyToxConfigurationFactory.INSTANCE, PyToxConfiguration.class, 0) {
+            @Override
+            protected void configurationCreatedAndWillLaunch(@NotNull final PyToxConfiguration configuration) throws IOException {
+              super.configurationCreatedAndWillLaunch(configuration);
+              PyToxTestTools.setRunOnlyEnvs(configuration, envsToRun);
+            }
+          };
+        }
+
+        @Override
+        protected void checkTestResults(@NotNull final PyAbstractTestProcessRunner<PyToxConfiguration> runner,
+                                        @NotNull final String stdout,
+                                        @NotNull final String stderr,
+                                        @NotNull final String all) {
+          final Set<String> environments = runner.getTestProxy().getChildren().stream().map(t -> t.getName()).collect(Collectors.toSet());
+          Assert.assertThat("Wrong environments launched", environments, Matchers.equalTo(Sets.newHashSet(envsToRun)));
+        }
+
+        @NotNull
+        @Override
+        public Set<String> getTags() {
+          return Sets.newHashSet("tox");
+        }
+      });
+  }
+
 
   private static final class MyPyProcessWithConsoleTestTask extends PyProcessWithConsoleTestTask<MyTestProcessRunner> {
+    private static final Logger LOGGER = Logger.getInstance(MyPyProcessWithConsoleTestTask.class);
     @NotNull
     private final Map<String, InterpreterExpectations> myInterpreters = new HashMap<>();
     private final int myMinimumSuccessTestCount;
     private final int myMaximumSuccessTestCount;
     @NotNull
-    private final MyTestProcessRunner myRunner;
+    private final Supplier<MyTestProcessRunner> myRunnerSupplier;
 
     /**
      * @param minimumSuccessTestCount how many success tests should be
      * @param interpreterExpectations interpreter_name -] expected result
+     * @param runnerSupplier          Lambda to create runner (can't reuse one runner several times,
+     *                                see {@link PyProcessWithConsoleTestTask#createProcessRunner()}
      * @param maximumTestCount        max number of success tests
      */
     private MyPyProcessWithConsoleTestTask(@Nullable final String relativeTestDataPath,
                                            final int minimumSuccessTestCount,
-                                           @NotNull final MyTestProcessRunner runner,
+                                           @NotNull final Supplier<MyTestProcessRunner> runnerSupplier,
                                            @NotNull final Iterable<Pair<String, InterpreterExpectations>> interpreterExpectations,
                                            final int maximumTestCount) {
       super(relativeTestDataPath, SdkCreationType.EMPTY_SDK);
       myMinimumSuccessTestCount = minimumSuccessTestCount;
       myMaximumSuccessTestCount = maximumTestCount;
-      myRunner = runner;
+      myRunnerSupplier = runnerSupplier;
       for (final Pair<String, InterpreterExpectations> interpreterExpectation : interpreterExpectations) {
         myInterpreters.put(interpreterExpectation.first, interpreterExpectation.second);
       }
@@ -195,8 +254,12 @@ public final class PyToxTest extends PyEnvTestCase {
                                     @NotNull final String stderr,
                                     @NotNull final String all) {
 
+      final Set<String> expectedInterpreters =
+        myInterpreters.entrySet().stream().filter(intAndExp -> intAndExp.getValue() != null).map(intAndExp -> intAndExp.getKey()).collect(
+          Collectors.toSet());
+
       // Interpreters are used in tox.ini, so there should be such text
-      for (final String interpreterName : myInterpreters.keySet()) {
+      for (final String interpreterName : expectedInterpreters) {
         Assert.assertThat(String.format("No %s used from tox.ini", interpreterName), all, Matchers.containsString(interpreterName));
       }
 
@@ -213,6 +276,13 @@ public final class PyToxTest extends PyEnvTestCase {
         final String interpreterName = interpreterSuite.getName();
         checkedInterpreters.add(interpreterName);
 
+        final InterpreterExpectations expectations = myInterpreters.get(interpreterName);
+        if (expectations == null) {
+          LOGGER.warn(String.format("Launched %s, but no expectation provided, skipping", interpreterName));
+          continue;
+        }
+
+
         if (interpreterSuite.getChildren().size() == 1 && interpreterSuite.getChildren().get(0).getName().endsWith("ERROR")) {
           // Interpreter failed to run
           final String testOutput = getTestOutput(interpreterSuite.getChildren().get(0));
@@ -223,7 +293,6 @@ public final class PyToxTest extends PyEnvTestCase {
             continue;
           }
           // Some other error?
-          final InterpreterExpectations expectations = myInterpreters.get(interpreterName);
           Assert
             .assertFalse(String.format("Interpreter %s should not fail, but failed: %s", interpreterName, getTestOutput(interpreterSuite)),
                          expectations.myExpectedSuccess);
@@ -256,12 +325,13 @@ public final class PyToxTest extends PyEnvTestCase {
 
         Assert
           .assertThat(message,
-                      getTestOutput(interpreterSuite), Matchers.containsString(myInterpreters.get(interpreterName).myExpectedOutput));
+                      getTestOutput(interpreterSuite), Matchers.containsString(expectations.myExpectedOutput));
       }
 
-      Assert.assertThat("No all interpreters from tox.ini used", checkedInterpreters, Matchers.equalTo(myInterpreters.keySet()));
-      assert !skippedInterpreters.equals(myInterpreters.keySet()) : "All interpreters skipped (they do not exist on platform), " +
-                                                                    "we test nothing";
+      Assert
+        .assertThat("No all interpreters from tox.ini used", expectedInterpreters, Matchers.everyItem(Matchers.isIn(checkedInterpreters)));
+      assert !skippedInterpreters.equals(expectedInterpreters) : "All interpreters skipped (they do not exist on platform), " +
+                                                                 "we test nothing";
     }
 
     @NotNull
@@ -280,13 +350,17 @@ public final class PyToxTest extends PyEnvTestCase {
     @NotNull
     @Override
     protected MyTestProcessRunner createProcessRunner() throws Exception {
-      return myRunner;
+      return myRunnerSupplier.get();
     }
   }
 
   private static final class MyTestProcessRunner extends PyAbstractTestProcessRunner<PyToxConfiguration> {
     private MyTestProcessRunner() {
-      super(PyToxConfigurationFactory.INSTANCE, PyToxConfiguration.class, 0);
+      this(0);
+    }
+
+    private MyTestProcessRunner(final int timesToRerunFailedTests) {
+      super(PyToxConfigurationFactory.INSTANCE, PyToxConfiguration.class, timesToRerunFailedTests);
     }
   }
 

python/testSrc/com/jetbrains/python/testing/tox/PyToxTestTools.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2000-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.jetbrains.python.testing.tox;
+
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * Tools to test tox
+ * @author Ilya.Kazakevich
+ */
+public final class PyToxTestTools {
+  private PyToxTestTools() {
+  }
+
+
+  public static void setRunOnlyEnvs(@NotNull final PyToxConfiguration configuration, @NotNull final String... tests) {
+    configuration.setRunOnlyEnvs(tests);
+  }
+}