From 9dedfb18c1576cf9ff41892508cc4fb1084a135a Mon Sep 17 00:00:00 2001 From: "Ilya.Kazakevich" Date: Thu, 30 Jun 2016 23:28:59 +0300 Subject: [PATCH] PY-19015: fix for "rerun" for failed tox tests Tox fetches env id (like py27) and provides it to configuration. --- .../run/AbstractPythonRunConfiguration.java | 8 ++ .../PythonTestCommandLineStateBase.java | 2 +- .../testing/tox/PyToxCommandLineState.java | 21 +--- .../testing/tox/PyToxConfiguration.java | 33 ++++++ .../python/testing/tox/PyToxTestLocator.java | 3 +- .../testData/toxtest/toxConcreteEnv/tox.ini | 5 + python/testData/toxtest/toxRerun/tox.ini | 5 + .../env/ConfigurationBasedProcessRunner.java | 17 +++ .../env/PyProcessWithConsoleTestTask.java | 3 +- .../testSrc/com/jetbrains/env/PyToxTest.java | 110 +++++++++++++++--- .../python/testing/tox/PyToxTestTools.java | 32 +++++ 11 files changed, 199 insertions(+), 40 deletions(-) create mode 100644 python/testData/toxtest/toxConcreteEnv/tox.ini create mode 100644 python/testData/toxtest/toxRerun/tox.ini create mode 100644 python/testSrc/com/jetbrains/python/testing/tox/PyToxTestTools.java diff --git a/python/src/com/jetbrains/python/run/AbstractPythonRunConfiguration.java b/python/src/com/jetbrains/python/run/AbstractPythonRunConfiguration.java index 866ad1f49471..d4e16f7008c7 100644 --- a/python/src/com/jetbrains/python/run/AbstractPythonRunConfiguration.java +++ b/python/src/com/jetbrains/python/run/AbstractPythonRunConfiguration.java @@ -469,4 +469,12 @@ public abstract class AbstractPythonRunConfiguration testSpecs) { + // By default we simply add them as arguments + paramsGroup.addParameters(testSpecs); + } } diff --git a/python/src/com/jetbrains/python/testing/PythonTestCommandLineStateBase.java b/python/src/com/jetbrains/python/testing/PythonTestCommandLineStateBase.java index f20daa18eb04..4bf3a730fd5b 100644 --- a/python/src/com/jetbrains/python/testing/PythonTestCommandLineStateBase.java +++ b/python/src/com/jetbrains/python/testing/PythonTestCommandLineStateBase.java @@ -158,7 +158,7 @@ public abstract class PythonTestCommandLineStateBase extends PythonCommandLineSt assert scriptParams != null; getRunner().addToGroup(scriptParams, cmd); addBeforeParameters(cmd); - scriptParams.addParameters(getTestSpecs()); + myConfiguration.addTestSpecsAsParameters(scriptParams, getTestSpecs()); addAfterParameters(cmd); } diff --git a/python/src/com/jetbrains/python/testing/tox/PyToxCommandLineState.java b/python/src/com/jetbrains/python/testing/tox/PyToxCommandLineState.java index bc300e4cb950..1c1d33ee3ec8 100644 --- a/python/src/com/jetbrains/python/testing/tox/PyToxCommandLineState.java +++ b/python/src/com/jetbrains/python/testing/tox/PyToxCommandLineState.java @@ -15,16 +15,13 @@ */ package com.jetbrains.python.testing.tox; -import com.intellij.execution.configurations.GeneralCommandLine; -import com.intellij.execution.configurations.ParamsGroup; import com.intellij.execution.runners.ExecutionEnvironment; -import com.intellij.openapi.util.text.StringUtil; import com.jetbrains.python.HelperPackage; import com.jetbrains.python.PythonHelper; import com.jetbrains.python.testing.PythonTestCommandLineStateBase; import org.jetbrains.annotations.NotNull; -import java.util.Collections; +import java.util.Arrays; import java.util.List; /** @@ -47,23 +44,9 @@ class PyToxCommandLineState extends PythonTestCommandLineStateBase { return PythonHelper.TOX; } - - @Override - public GeneralCommandLine generateCommandLine() { - final GeneralCommandLine line = super.generateCommandLine(); - final ParamsGroup group = line.getParametersList().getParamsGroup(GROUP_SCRIPT); - assert group != null : "No group " + GROUP_SCRIPT; - final String[] envs = myConfiguration.getRunOnlyEnvs(); - if (envs.length > 0) { - group.addParameter(String.format("-e %s", StringUtil.join(envs, ","))); - } - group.addParameters(myConfiguration.getArguments()); - return line; - } - @NotNull @Override protected List getTestSpecs() { - return Collections.emptyList(); + return Arrays.asList(myConfiguration.getRunOnlyEnvs()); } } diff --git a/python/src/com/jetbrains/python/testing/tox/PyToxConfiguration.java b/python/src/com/jetbrains/python/testing/tox/PyToxConfiguration.java index e62e82ea4117..dbd7e3ae995f 100644 --- a/python/src/com/jetbrains/python/testing/tox/PyToxConfiguration.java +++ b/python/src/com/jetbrains/python/testing/tox/PyToxConfiguration.java @@ -16,12 +16,17 @@ package com.jetbrains.python.testing.tox; import com.intellij.execution.Executor; +import com.intellij.execution.Location; +import com.intellij.execution.configurations.ParamsGroup; import com.intellij.execution.configurations.RunProfileState; import com.intellij.execution.runners.ExecutionEnvironment; +import com.intellij.execution.testframework.AbstractTestProxy; import com.intellij.openapi.options.SettingsEditor; import com.intellij.openapi.project.Project; import com.intellij.openapi.util.InvalidDataException; import com.intellij.openapi.util.WriteExternalException; +import com.intellij.openapi.util.text.StringUtil; +import com.intellij.openapi.vfs.VirtualFileManager; import com.intellij.util.ArrayUtil; import com.intellij.util.xmlb.SkipEmptySerializationFilter; import com.intellij.util.xmlb.XmlSerializer; @@ -33,6 +38,8 @@ import org.jdom.Element; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import java.util.List; + /** * @author Ilya.Kazakevich */ @@ -99,4 +106,30 @@ public final class PyToxConfiguration extends AbstractPythonRunConfiguration location, @NotNull final AbstractTestProxy failedTest) { + + AbstractTestProxy test = failedTest; + while (test != null) { + final String url = test.getLocationUrl(); + if (url == null) { + continue; + } + final String protocol = VirtualFileManager.extractProtocol(url); + if (PyToxTestLocator.PROTOCOL_ID.equals(protocol)) { + return VirtualFileManager.extractPath(url); + } + test = test.getParent(); + } + return null; + } + + @Override + public void addTestSpecsAsParameters(@NotNull final ParamsGroup paramsGroup, @NotNull final List testSpecs) { + if (!testSpecs.isEmpty()) { + paramsGroup.addParameter(String.format("-e %s", StringUtil.join(testSpecs, ","))); + } + } } diff --git a/python/src/com/jetbrains/python/testing/tox/PyToxTestLocator.java b/python/src/com/jetbrains/python/testing/tox/PyToxTestLocator.java index f6ecea9a3e97..415cd0b9f932 100644 --- a/python/src/com/jetbrains/python/testing/tox/PyToxTestLocator.java +++ b/python/src/com/jetbrains/python/testing/tox/PyToxTestLocator.java @@ -39,11 +39,12 @@ public final class PyToxTestLocator implements PythonTestLocator { private static final String DUMMY_FILE_PADDING = "#env"; private static final Key ENV_NAME_KEY = Key.create("ENV_NAME"); + static final String PROTOCOL_ID = "tox_env"; @NotNull @Override public String getProtocolId() { - return "tox_env"; + return PROTOCOL_ID; } @NotNull diff --git a/python/testData/toxtest/toxConcreteEnv/tox.ini b/python/testData/toxtest/toxConcreteEnv/tox.ini new file mode 100644 index 000000000000..68d81edafd85 --- /dev/null +++ b/python/testData/toxtest/toxConcreteEnv/tox.ini @@ -0,0 +1,5 @@ +[tox] +skipsdist=True +envlist = py26, py27, py32, py34 +[testenv] +commands=python -c "print 1" \ No newline at end of file diff --git a/python/testData/toxtest/toxRerun/tox.ini b/python/testData/toxtest/toxRerun/tox.ini new file mode 100644 index 000000000000..68d81edafd85 --- /dev/null +++ b/python/testData/toxtest/toxRerun/tox.ini @@ -0,0 +1,5 @@ +[tox] +skipsdist=True +envlist = py26, py27, py32, py34 +[testenv] +commands=python -c "print 1" \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/env/ConfigurationBasedProcessRunner.java b/python/testSrc/com/jetbrains/env/ConfigurationBasedProcessRunner.java index a9f2f328752e..8d86fd222cd6 100644 --- a/python/testSrc/com/jetbrains/env/ConfigurationBasedProcessRunner.java +++ b/python/testSrc/com/jetbrains/env/ConfigurationBasedProcessRunner.java @@ -93,6 +93,8 @@ public abstract class ConfigurationBasedProcessRunnerAlways create new runner, to prevent stale artifacts on reruns. */ @NotNull protected abstract T createProcessRunner() throws Exception; diff --git a/python/testSrc/com/jetbrains/env/PyToxTest.java b/python/testSrc/com/jetbrains/env/PyToxTest.java index ab4fec20c309..5c4ee346bcb4 100644 --- a/python/testSrc/com/jetbrains/env/PyToxTest.java +++ b/python/testSrc/com/jetbrains/env/PyToxTest.java @@ -23,13 +23,17 @@ import com.intellij.openapi.util.Pair; import com.jetbrains.python.sdkTools.SdkCreationType; import com.jetbrains.python.testing.tox.PyToxConfiguration; import com.jetbrains.python.testing.tox.PyToxConfigurationFactory; +import com.jetbrains.python.testing.tox.PyToxTestTools; import org.hamcrest.Matchers; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.junit.Assert; import org.junit.Test; +import java.io.IOException; import java.util.*; +import java.util.function.Supplier; +import java.util.stream.Collectors; /** * Ensure tox runner works @@ -47,7 +51,7 @@ public final class PyToxTest extends PyEnvTestCase { @Test public void testToxSimpleRun() { runPythonTest(new MyPyProcessWithConsoleTestTask("/toxtest/toxSimpleRun/", 2, - new MyTestProcessRunner(), + () -> new MyTestProcessRunner(), Arrays.asList( // Should fail, no skip in 26 Pair.create("py26", new InterpreterExpectations( @@ -63,7 +67,7 @@ public final class PyToxTest extends PyEnvTestCase { @Test public void testToxNose() { runPythonTest(new MyPyProcessWithConsoleTestTask("/toxtest/toxNose/", 1, - new MyTestProcessRunner(), + () -> new MyTestProcessRunner(), Arrays.asList( Pair.create("py26", new InterpreterExpectations("", true)), Pair.create("py27", new InterpreterExpectations("", true)), @@ -81,7 +85,7 @@ public final class PyToxTest extends PyEnvTestCase { @Test public void testToxPyTest() { runPythonTest(new MyPyProcessWithConsoleTestTask("/toxtest/toxPyTest/", 1, - new MyTestProcessRunner(), + () -> new MyTestProcessRunner(), Arrays.asList( Pair.create("py26", new InterpreterExpectations("", true)), Pair.create("py27", new InterpreterExpectations("", true)), @@ -99,7 +103,7 @@ public final class PyToxTest extends PyEnvTestCase { @Test public void testToxUnitTest() { runPythonTest(new MyPyProcessWithConsoleTestTask("/toxtest/toxUnitTest/", 1, - new MyTestProcessRunner(), + () -> new MyTestProcessRunner(), Arrays.asList( Pair.create("py26", new InterpreterExpectations("", true)), Pair.create("py27", new InterpreterExpectations("", true)), @@ -117,7 +121,7 @@ public final class PyToxTest extends PyEnvTestCase { @Test public void textToxOneInterpreter() throws Exception { runPythonTest(new MyPyProcessWithConsoleTestTask("/toxtest/toxOneInterpreter/", 0, - new MyTestProcessRunner(), + () -> new MyTestProcessRunner(), Arrays.asList( Pair.create("py26", new InterpreterExpectations("", true)), Pair.create("py27", new InterpreterExpectations("ython 2.7", true)), @@ -135,7 +139,7 @@ public final class PyToxTest extends PyEnvTestCase { @Test public void testDoubleRun() throws Exception { runPythonTest(new MyPyProcessWithConsoleTestTask("/toxtest/toxDoubleRun/", 1, - new MyTestProcessRunner(), + () -> new MyTestProcessRunner(), Collections.singletonList( Pair.create("py27", new InterpreterExpectations("", true)) ), @@ -149,7 +153,7 @@ public final class PyToxTest extends PyEnvTestCase { @Test public void testToxSuccessTest() { runPythonTest(new MyPyProcessWithConsoleTestTask("/toxtest/toxSuccess/", 1, - new MyTestProcessRunner(), + () -> new MyTestProcessRunner(), Arrays.asList( Pair.create("py26", new InterpreterExpectations("I am 2.6", true)), Pair.create("py27", new InterpreterExpectations("I am 2.7", true)), @@ -161,29 +165,84 @@ public final class PyToxTest extends PyEnvTestCase { ); } + /** + * Ensures rerun works for tox + */ + @Test + public void testEnvRerun() throws Exception { + runPythonTest(new MyPyProcessWithConsoleTestTask("/toxtest/toxConcreteEnv/", 0, + () -> new MyTestProcessRunner(1), + Arrays.asList( + Pair.create("py32", new InterpreterExpectations("", false)), + Pair.create("py34", new InterpreterExpectations("", false)) + ), + Integer.MAX_VALUE) + ); + } + + /** + * Provide certain env and check it is launched + */ + @Test + public void testConcreteEnv() throws Exception { + final String[] envsToRun = {"py27", "py34"}; + runPythonTest( + new PyProcessWithConsoleTestTask>("/toxtest/toxSuccess/", SdkCreationType.EMPTY_SDK) { + @NotNull + @Override + protected PyAbstractTestProcessRunner createProcessRunner() throws Exception { + return new PyAbstractTestProcessRunner(PyToxConfigurationFactory.INSTANCE, PyToxConfiguration.class, 0) { + @Override + protected void configurationCreatedAndWillLaunch(@NotNull final PyToxConfiguration configuration) throws IOException { + super.configurationCreatedAndWillLaunch(configuration); + PyToxTestTools.setRunOnlyEnvs(configuration, envsToRun); + } + }; + } + + @Override + protected void checkTestResults(@NotNull final PyAbstractTestProcessRunner runner, + @NotNull final String stdout, + @NotNull final String stderr, + @NotNull final String all) { + final Set environments = runner.getTestProxy().getChildren().stream().map(t -> t.getName()).collect(Collectors.toSet()); + Assert.assertThat("Wrong environments launched", environments, Matchers.equalTo(Sets.newHashSet(envsToRun))); + } + + @NotNull + @Override + public Set getTags() { + return Sets.newHashSet("tox"); + } + }); + } + private static final class MyPyProcessWithConsoleTestTask extends PyProcessWithConsoleTestTask { + private static final Logger LOGGER = Logger.getInstance(MyPyProcessWithConsoleTestTask.class); @NotNull private final Map myInterpreters = new HashMap<>(); private final int myMinimumSuccessTestCount; private final int myMaximumSuccessTestCount; @NotNull - private final MyTestProcessRunner myRunner; + private final Supplier myRunnerSupplier; /** * @param minimumSuccessTestCount how many success tests should be * @param interpreterExpectations interpreter_name -] expected result + * @param runnerSupplier Lambda to create runner (can't reuse one runner several times, + * see {@link PyProcessWithConsoleTestTask#createProcessRunner()} * @param maximumTestCount max number of success tests */ private MyPyProcessWithConsoleTestTask(@Nullable final String relativeTestDataPath, final int minimumSuccessTestCount, - @NotNull final MyTestProcessRunner runner, + @NotNull final Supplier runnerSupplier, @NotNull final Iterable> interpreterExpectations, final int maximumTestCount) { super(relativeTestDataPath, SdkCreationType.EMPTY_SDK); myMinimumSuccessTestCount = minimumSuccessTestCount; myMaximumSuccessTestCount = maximumTestCount; - myRunner = runner; + myRunnerSupplier = runnerSupplier; for (final Pair interpreterExpectation : interpreterExpectations) { myInterpreters.put(interpreterExpectation.first, interpreterExpectation.second); } @@ -195,8 +254,12 @@ public final class PyToxTest extends PyEnvTestCase { @NotNull final String stderr, @NotNull final String all) { + final Set expectedInterpreters = + myInterpreters.entrySet().stream().filter(intAndExp -> intAndExp.getValue() != null).map(intAndExp -> intAndExp.getKey()).collect( + Collectors.toSet()); + // Interpreters are used in tox.ini, so there should be such text - for (final String interpreterName : myInterpreters.keySet()) { + for (final String interpreterName : expectedInterpreters) { Assert.assertThat(String.format("No %s used from tox.ini", interpreterName), all, Matchers.containsString(interpreterName)); } @@ -213,6 +276,13 @@ public final class PyToxTest extends PyEnvTestCase { final String interpreterName = interpreterSuite.getName(); checkedInterpreters.add(interpreterName); + final InterpreterExpectations expectations = myInterpreters.get(interpreterName); + if (expectations == null) { + LOGGER.warn(String.format("Launched %s, but no expectation provided, skipping", interpreterName)); + continue; + } + + if (interpreterSuite.getChildren().size() == 1 && interpreterSuite.getChildren().get(0).getName().endsWith("ERROR")) { // Interpreter failed to run final String testOutput = getTestOutput(interpreterSuite.getChildren().get(0)); @@ -223,7 +293,6 @@ public final class PyToxTest extends PyEnvTestCase { continue; } // Some other error? - final InterpreterExpectations expectations = myInterpreters.get(interpreterName); Assert .assertFalse(String.format("Interpreter %s should not fail, but failed: %s", interpreterName, getTestOutput(interpreterSuite)), expectations.myExpectedSuccess); @@ -256,12 +325,13 @@ public final class PyToxTest extends PyEnvTestCase { Assert .assertThat(message, - getTestOutput(interpreterSuite), Matchers.containsString(myInterpreters.get(interpreterName).myExpectedOutput)); + getTestOutput(interpreterSuite), Matchers.containsString(expectations.myExpectedOutput)); } - Assert.assertThat("No all interpreters from tox.ini used", checkedInterpreters, Matchers.equalTo(myInterpreters.keySet())); - assert !skippedInterpreters.equals(myInterpreters.keySet()) : "All interpreters skipped (they do not exist on platform), " + - "we test nothing"; + Assert + .assertThat("No all interpreters from tox.ini used", expectedInterpreters, Matchers.everyItem(Matchers.isIn(checkedInterpreters))); + assert !skippedInterpreters.equals(expectedInterpreters) : "All interpreters skipped (they do not exist on platform), " + + "we test nothing"; } @NotNull @@ -280,13 +350,17 @@ public final class PyToxTest extends PyEnvTestCase { @NotNull @Override protected MyTestProcessRunner createProcessRunner() throws Exception { - return myRunner; + return myRunnerSupplier.get(); } } private static final class MyTestProcessRunner extends PyAbstractTestProcessRunner { private MyTestProcessRunner() { - super(PyToxConfigurationFactory.INSTANCE, PyToxConfiguration.class, 0); + this(0); + } + + private MyTestProcessRunner(final int timesToRerunFailedTests) { + super(PyToxConfigurationFactory.INSTANCE, PyToxConfiguration.class, timesToRerunFailedTests); } } diff --git a/python/testSrc/com/jetbrains/python/testing/tox/PyToxTestTools.java b/python/testSrc/com/jetbrains/python/testing/tox/PyToxTestTools.java new file mode 100644 index 000000000000..61b5315a5212 --- /dev/null +++ b/python/testSrc/com/jetbrains/python/testing/tox/PyToxTestTools.java @@ -0,0 +1,32 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.jetbrains.python.testing.tox; + +import org.jetbrains.annotations.NotNull; + +/** + * Tools to test tox + * @author Ilya.Kazakevich + */ +public final class PyToxTestTools { + private PyToxTestTools() { + } + + + public static void setRunOnlyEnvs(@NotNull final PyToxConfiguration configuration, @NotNull final String... tests) { + configuration.setRunOnlyEnvs(tests); + } +}