From be9ddbb56c64d6f34e0066e5531664f45cdaf66c Mon Sep 17 00:00:00 2001 From: Vladislav Rassokhin Date: Mon, 25 Sep 2023 14:30:34 +0200 Subject: [PATCH] [test framework] IJI-1588 Run every test class or package in a dedicated runtime For now, without retries. First pass collects all test classes without executing them. Then they run either one by one, or grouped by package. GitOrigin-RevId: cbdea0a64313cdc1dd0bbc6a281b9cd679c2a512 --- .../intellij/build/TestingOptions.kt | 10 +- .../intellij/build/impl/TestingTasksImpl.kt | 198 ++++++++++++------ .../JUnit5TeamCityRunnerForTestAllSuite.java | 36 +++- ...nit5TeamCityRunnerForTestsOnClasspath.java | 34 ++- .../core/src/com/intellij/TestAll.java | 36 +++- 5 files changed, 227 insertions(+), 87 deletions(-) diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/TestingOptions.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/TestingOptions.kt index 8c6c5f2bcac7..4ef95691418c 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/TestingOptions.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/TestingOptions.kt @@ -1,4 +1,4 @@ -// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.intellij.build import com.intellij.TestCaseLoader @@ -132,9 +132,13 @@ open class TestingOptions { var batchTestIncludes: String? = System.getProperty("intellij.build.test.batchTest.includes") /** - * Run only whole classes in forked Runtime in case if [batchTestIncludes] mode enabled + * Run only whole classes/packages in forked Runtime + * Allowed values: + * * `false` + * * `class` + * * `package` */ - var isDedicatedRuntimePerClassEnabled: Boolean = SystemProperties.getBooleanProperty("intellij.build.test.dedicated.runtime.per.class.enabled", false) + var isDedicatedTestRuntime: String = System.getProperty("intellij.build.test.dedicated.runtime", "false") var isPerformanceTestsOnly = SystemProperties.getBooleanProperty(PERFORMANCE_TESTS_ONLY_FLAG, false) diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/TestingTasksImpl.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/TestingTasksImpl.kt index 320364537ae6..34a1f9156917 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/TestingTasksImpl.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/TestingTasksImpl.kt @@ -36,6 +36,7 @@ import java.nio.file.Path import java.util.concurrent.Callable import java.util.regex.Pattern import java.util.stream.Stream +import kotlin.io.path.absolutePathString import kotlin.io.path.readLines internal class TestingTasksImpl(private val context: CompilationContext, private val options: TestingOptions) : TestingTasks { @@ -613,7 +614,7 @@ internal class TestingTasksImpl(private val context: CompilationContext, private .toList() } - private fun getTestClassesForModule(mainModule: String, filteringPattern: Pattern = Pattern.compile(".*.class")): List { + private fun getTestClassesForModule(mainModule: String, filteringPattern: Pattern = Pattern.compile(".*\\.class")): List { val mainModuleTestsOutput = context.getModuleTestsOutputPath(context.findRequiredModule(mainModule)) val root = Path.of(mainModuleTestsOutput) @@ -661,7 +662,7 @@ internal class TestingTasksImpl(private val context: CompilationContext, private val jUnit4And5TestMethods = getAnnotatedTestMethods(aClass, testAnnotation4, testAnnotation5, testFactoryAnnotation5) // Run JUnit 4 and 5 whole test classes separately - if (options.isDedicatedRuntimePerClassEnabled && jUnit4And5TestMethods.isNotEmpty()) { + if (options.isDedicatedTestRuntime != "false" && jUnit4And5TestMethods.isNotEmpty()) { val exitCode = runJUnit5Engine(systemProperties = systemProperties, jvmArgs = jvmArgs, envVariables = envVariables, @@ -735,6 +736,111 @@ internal class TestingTasksImpl(private val context: CompilationContext, private runInBatchMode(mainModule, systemProperties, jvmArgs, envVariables, bootstrapClasspath, testClasspath) } } + else if (options.isDedicatedTestRuntime != "false") { + if (options.isDedicatedTestRuntime != "class" && options.isDedicatedTestRuntime != "package") { + context.messages.error("Unsupported 'intellij.build.test.dedicated.runtime' value: ${options.isDedicatedTestRuntime}. Expected 'class', 'package' or 'false'") + } + context.messages.info("Will run tests in dedicated runtimes ('${options.isDedicatedTestRuntime}')") + // First, collect all tests for both JUnit5 and JUnit3+4 + val testClassesJUnit5 = blockAndSpan("collect junit 5 tests") { + val testClassesListFile = Files.createTempFile("tests-to-run-", ".list").apply { Files.delete(this) } + runJUnit5Engine( + systemProperties = systemProperties + ("intellij.build.test.list.classes" to testClassesListFile.absolutePathString()), + jvmArgs = jvmArgs, + envVariables = envVariables, + bootstrapClasspath = bootstrapClasspath, + modulePath = modulePath, + testClasspath = testClasspath, + suiteName = null, + methodName = null) + return@blockAndSpan testClassesListFile.let { if (Files.exists(it)) it.readLines() else emptyList() } + } + val testClassesJUnit34 = blockAndSpan("collect junit 3+4 tests") { + val testClassesListFile = Files.createTempFile("tests-to-run-", ".list").apply { Files.delete(this) } + runJUnit5Engine( + systemProperties = systemProperties + ("intellij.build.test.list.classes" to testClassesListFile.absolutePathString()), + jvmArgs = jvmArgs, + envVariables = envVariables, + bootstrapClasspath = bootstrapClasspath, + modulePath = modulePath, + testClasspath = testClasspath, + suiteName = options.bootstrapSuite, + methodName = null) + return@blockAndSpan testClassesListFile.let { if (Files.exists(it)) it.readLines() else emptyList() } + } + + if (testClassesJUnit5.isEmpty() && testClassesJUnit34.isEmpty() && + // a bucket might be empty for run configurations with too few tests due to imperfect tests balancing + options.bucketsCount < 2) { + throw NoTestsFound() + } + + if (options.isDedicatedTestRuntime == "class") { + fun runOneClass(testClassName: String) { + blockAndSpan("running test class '$testClassName'") { + runJUnit5Engine( + systemProperties = systemProperties, + jvmArgs = jvmArgs, + envVariables = envVariables, + bootstrapClasspath = bootstrapClasspath, + modulePath = modulePath, + testClasspath = testClasspath, + suiteName = testClassName, + methodName = null + ) + } + } + + if (testClassesJUnit5.isNotEmpty()) { + context.messages.info("Will run JUnit 5 tests:\n${testClassesJUnit5.joinToString("\n")}") + testClassesJUnit5.forEach(::runOneClass) + } + if (testClassesJUnit34.isNotEmpty()) { + context.messages.info("Will run JUnit 3+4 tests:\n${testClassesJUnit34.joinToString("\n")}") + testClassesJUnit34.forEach(::runOneClass) + } + } + else if (options.isDedicatedTestRuntime == "package") { + fun groupByPackages(tests: List): Map> { + return tests.groupBy { + val i = it.lastIndexOf('.') + if (i != -1) it.substring(0, i) else "" + } + } + fun runOnePackage(entry: Map.Entry>) { + val packageName = entry.key + val classes = entry.value + + blockAndSpan("running tests in package '$packageName'") { + runJUnit5Engine( + systemProperties = systemProperties, + jvmArgs = jvmArgs, + envVariables = envVariables, + bootstrapClasspath = bootstrapClasspath, + modulePath = modulePath, + testClasspath = testClasspath, + suiteName = "__classes__", + methodName = classes.joinToString(";") + ) + } + } + + if (testClassesJUnit5.isNotEmpty()) { + val packages = groupByPackages(testClassesJUnit5) + context.messages.info(packages.entries.joinToString(prefix = "Will run JUnit 5 packages:\n", separator = "\n") { e -> + e.value.joinToString(prefix = "${e.key}\n ", separator = "\n ") + }) + packages.forEach(::runOnePackage) + } + if (testClassesJUnit34.isNotEmpty()) { + val packages = groupByPackages(testClassesJUnit34) + context.messages.info(packages.entries.joinToString(prefix = "Will run JUnit 3+4 packages:\n", separator = "\n") { e -> + e.value.joinToString(prefix = "${e.key}\n ", separator = "\n ") + }) + packages.forEach(::runOnePackage) + } + } + } else { val failedClassesJUnit5List = Files.createTempFile("failed-classes-junit5-", ".list").apply { Files.delete(this) } val failedClassesJUnit34List = Files.createTempFile("failed-classes-junit34-", ".list").apply { Files.delete(this) } @@ -754,74 +860,30 @@ internal class TestingTasksImpl(private val context: CompilationContext, private val additionalProperties: Map = if (attempt > 1) mapOf("intellij.build.test.ignoreFirstAndLastTests" to "true") else emptyMap() val exitCode5: Int = if (runJUnit5) blockAndSpan("run junit 5 tests${spanNameSuffix}") { - if (options.isDedicatedRuntimePerClassEnabled) { - context.messages.info("Creation of a dedicated runtime for each class is enabled") - val testClasses = getTestClassesForModule(mainModule = mainModule) - - var batchExitCode: Int = NO_TESTS_ERROR - for (testClass in testClasses) { - val testClassName = FileUtilRt.getNameWithoutExtension(testClass).replace('/', '.') - - val exitCode = runJUnit5Engine(systemProperties = systemProperties + additionalProperties + additionalPropertiesJUnit5, - jvmArgs = jvmArgs, - envVariables = envVariables, - bootstrapClasspath = bootstrapClasspath, - modulePath = modulePath, - testClasspath = testClasspath, - suiteName = testClassName, - methodName = null) - - if (exitCode == NO_TESTS_ERROR) batchExitCode = NO_TESTS_ERROR - } - - batchExitCode - } - else { - runJUnit5Engine(systemProperties = systemProperties + additionalProperties + additionalPropertiesJUnit5, - jvmArgs = jvmArgs, - envVariables = envVariables, - bootstrapClasspath = bootstrapClasspath, - modulePath = modulePath, - testClasspath = testClasspath, - suiteName = null, - methodName = null) - } + runJUnit5Engine( + systemProperties = systemProperties + additionalProperties + additionalPropertiesJUnit5, + jvmArgs = jvmArgs, + envVariables = envVariables, + bootstrapClasspath = bootstrapClasspath, + modulePath = modulePath, + testClasspath = testClasspath, + suiteName = null, + methodName = null + ) } else 0 val exitCode34: Int = if (runJUnit34) blockAndSpan("run junit 3+4 tests${spanNameSuffix}") { - if (options.isDedicatedRuntimePerClassEnabled) { - context.messages.info("Creation of a dedicated runtime for each class is enabled") - val testClasses = getTestClassesForModule(mainModule = mainModule) - - var batchExitCode: Int = NO_TESTS_ERROR - for (testClass in testClasses) { - val testClassName = FileUtilRt.getNameWithoutExtension(testClass).replace('/', '.') - - val exitCode = runJUnit5Engine(systemProperties = systemProperties + additionalProperties + additionalPropertiesJUnit34, - jvmArgs = jvmArgs, - envVariables = envVariables, - bootstrapClasspath = bootstrapClasspath, - modulePath = modulePath, - testClasspath = testClasspath, - suiteName = testClassName, - methodName = null) - - if (exitCode == NO_TESTS_ERROR) batchExitCode = NO_TESTS_ERROR - } - - batchExitCode - } - else { - runJUnit5Engine(systemProperties = systemProperties + additionalProperties + additionalPropertiesJUnit34, - jvmArgs = jvmArgs, - envVariables = envVariables, - bootstrapClasspath = bootstrapClasspath, - modulePath = modulePath, - testClasspath = testClasspath, - suiteName = options.bootstrapSuite, - methodName = null) - } + runJUnit5Engine( + systemProperties = systemProperties + additionalProperties + additionalPropertiesJUnit34, + jvmArgs = jvmArgs, + envVariables = envVariables, + bootstrapClasspath = bootstrapClasspath, + modulePath = modulePath, + testClasspath = testClasspath, + suiteName = options.bootstrapSuite, + methodName = null + ) } else 0 @@ -874,7 +936,9 @@ internal class TestingTasksImpl(private val context: CompilationContext, private if (modulePath == null) { appendJUnitStarter(classpath) } - if (!isBootstrapSuiteDefault || isRunningInBatchMode || options.isDedicatedRuntimePerClassEnabled || suiteName == null) { + if (!isBootstrapSuiteDefault || isRunningInBatchMode + || options.isDedicatedTestRuntime != "false" + || suiteName == null) { classpath += testClasspath } args += "-classpath" diff --git a/platform/testFramework/bootstrap/src/com/intellij/tests/JUnit5TeamCityRunnerForTestAllSuite.java b/platform/testFramework/bootstrap/src/com/intellij/tests/JUnit5TeamCityRunnerForTestAllSuite.java index 0a7b223d7c00..bcd0f193d8a2 100644 --- a/platform/testFramework/bootstrap/src/com/intellij/tests/JUnit5TeamCityRunnerForTestAllSuite.java +++ b/platform/testFramework/bootstrap/src/com/intellij/tests/JUnit5TeamCityRunnerForTestAllSuite.java @@ -9,15 +9,14 @@ import junit.framework.JUnit4TestAdapterCache; import junit.framework.TestResult; import junit.framework.TestSuite; import org.junit.platform.engine.DiscoverySelector; +import org.junit.platform.engine.Filter; import org.junit.platform.engine.TestExecutionResult; +import org.junit.platform.engine.discovery.ClassNameFilter; import org.junit.platform.engine.discovery.DiscoverySelectors; import org.junit.platform.engine.reporting.ReportEntry; import org.junit.platform.engine.support.descriptor.ClassSource; import org.junit.platform.engine.support.descriptor.MethodSource; -import org.junit.platform.launcher.Launcher; -import org.junit.platform.launcher.TestExecutionListener; -import org.junit.platform.launcher.TestIdentifier; -import org.junit.platform.launcher.TestPlan; +import org.junit.platform.launcher.*; import org.junit.platform.launcher.core.LauncherConfig; import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder; import org.junit.platform.launcher.core.LauncherFactory; @@ -34,19 +33,39 @@ import java.io.StringWriter; import java.util.*; // Used to run JUnit 3/4 tests via JUnit 5 runtime +@SuppressWarnings("UseOfSystemOutOrSystemErr") public final class JUnit5TeamCityRunnerForTestAllSuite { public static void main(String[] args) throws ClassNotFoundException { + if (args.length != 1 && args.length != 2) { + System.err.printf("Expected one or two arguments, got %d: %s%n", args.length, Arrays.toString(args)); + System.exit(1); + } try { Launcher launcher = LauncherFactory.create(LauncherConfig.builder().enableLauncherSessionListenerAutoRegistration(false).build()); - DiscoverySelector selector; + List selectors; + List> filters = new ArrayList<>(0); if (args.length == 1) { - selector = DiscoverySelectors.selectClass(args[0]); + selectors = Collections.singletonList(DiscoverySelectors.selectClass(args[0])); + } + else if (args[0].equals("__package__")) { + selectors = Collections.singletonList(DiscoverySelectors.selectPackage(args[1])); + // exclude subpackages + filters.add(ClassNameFilter.excludeClassNamePatterns("\\Q" + args[1] + "\\E\\.[^.]+\\..*")); + } + else if (args[0].equals("__classes__")) { + String[] classes = args[1].split(";"); + selectors = Arrays.stream(classes).map(DiscoverySelectors::selectClass).toList(); } else { - selector = DiscoverySelectors.selectMethod(args[0], args[1]); + selectors = Collections.singletonList(DiscoverySelectors.selectMethod(args[0], args[1])); } + LauncherDiscoveryRequest discoveryRequest = LauncherDiscoveryRequestBuilder.request() + .selectors(selectors) + .filters(filters.toArray(new Filter[0])) + .build(); TCExecutionListener listener = new TCExecutionListener(); - launcher.execute(LauncherDiscoveryRequestBuilder.request().selectors(selector).build(), listener); + TestPlan testPlan = launcher.discover(discoveryRequest); + launcher.execute(testPlan, listener); if (!listener.smthExecuted()) { //see org.jetbrains.intellij.build.impl.TestingTasksImpl.NO_TESTS_ERROR System.exit(42); @@ -97,7 +116,6 @@ public final class JUnit5TeamCityRunnerForTestAllSuite { }; } - @SuppressWarnings("UseOfSystemOutOrSystemErr") public static class TCExecutionListener implements TestExecutionListener { /** * The same constant as com.intellij.rt.execution.TestListenerProtocol.CLASS_CONFIGURATION diff --git a/platform/testFramework/bootstrap/src/com/intellij/tests/JUnit5TeamCityRunnerForTestsOnClasspath.java b/platform/testFramework/bootstrap/src/com/intellij/tests/JUnit5TeamCityRunnerForTestsOnClasspath.java index 263ef2fa7279..f176b9a34e16 100644 --- a/platform/testFramework/bootstrap/src/com/intellij/tests/JUnit5TeamCityRunnerForTestsOnClasspath.java +++ b/platform/testFramework/bootstrap/src/com/intellij/tests/JUnit5TeamCityRunnerForTestsOnClasspath.java @@ -12,20 +12,19 @@ import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder; import org.junit.platform.launcher.core.LauncherFactory; import org.junit.vintage.engine.descriptor.VintageTestDescriptor; +import java.io.IOException; import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodType; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Collections; -import java.util.List; -import java.util.ServiceLoader; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; // Used to run JUnit 5 tests via JUnit 5 runtime @SuppressWarnings({"UseOfSystemOutOrSystemErr", "CallToPrintStackTrace"}) public final class JUnit5TeamCityRunnerForTestsOnClasspath { + private static final String ourCollectTestsFile = System.getProperty("intellij.build.test.list.classes"); public static void main(String[] args) { try { @@ -67,6 +66,10 @@ public final class JUnit5TeamCityRunnerForTestsOnClasspath { .filters(nameFilter, postDiscoveryFilter, EngineFilter.excludeEngines(VintageTestDescriptor.ENGINE_ID)).build(); TestPlan testPlan = launcher.discover(discoveryRequest); if (testPlan.containsTests()) { + if (ourCollectTestsFile != null) { + saveListOfTestClasses(testPlan); + return; + } launcher.execute(testPlan, new JUnit5TeamCityRunnerForTestAllSuite.TCExecutionListener()); } else { @@ -161,4 +164,27 @@ public final class JUnit5TeamCityRunnerForTestsOnClasspath { } }; } + + private static void saveListOfTestClasses(TestPlan testPlan) { + ArrayList testClasses = new ArrayList<>(0); + for (TestIdentifier root : testPlan.getRoots()) { + Set firstLevel = testPlan.getChildren(root); + for (TestIdentifier identifier : firstLevel) { + identifier.getSource() + .filter(source -> source instanceof ClassSource) + .map(source -> ((ClassSource)source).getClassName()) + .ifPresent(name -> testClasses.add(name)); + } + } + Path path = Path.of(ourCollectTestsFile); + try { + Files.createDirectories(path.getParent()); + Files.write(path, testClasses); + } + catch (IOException e) { + System.err.printf("Cannot save list of test classes to '%s': %s%n", path.toAbsolutePath(), e); + e.printStackTrace(); + System.exit(1); + } + } } diff --git a/platform/testFramework/core/src/com/intellij/TestAll.java b/platform/testFramework/core/src/com/intellij/TestAll.java index cbb1d7bdf73f..7d9e1000bb25 100644 --- a/platform/testFramework/core/src/com/intellij/TestAll.java +++ b/platform/testFramework/core/src/com/intellij/TestAll.java @@ -101,6 +101,7 @@ public class TestAll implements Test { private static final List ourClassLoadingProblems = new ArrayList<>(); private static JUnit4TestAdapterCache ourUnit4TestAdapterCache; + private static final String ourCollectTestsFile = System.getProperty("intellij.build.test.list.classes", null); public TestAll(String rootPackage) throws Throwable { this(rootPackage, getClassRoots()); @@ -274,8 +275,9 @@ public class TestAll implements Test { System.out.println("------"); int totalTests = classes.size(); + final List collectedTests = ourCollectTestsFile != null ? new ArrayList<>(totalTests) : null; for (Class aClass : testsToRun) { - runNextTest(testResult, totalTests, aClass); + runOrCollectNextTest(testResult, totalTests, aClass, collectedTests); if (testResult.shouldStop()) break; } @@ -296,6 +298,20 @@ public class TestAll implements Test { } } + if (collectedTests != null) { + Path path = Path.of(ourCollectTestsFile); + try { + collectedTests.remove("_FirstInSuiteTest"); + collectedTests.remove("_LastInSuiteTest"); + Files.createDirectories(path.getParent()); + Files.write(path, collectedTests); + } + catch (IOException e) { + System.err.printf("Cannot save list of test classes to '%s': %s%n", path.toAbsolutePath(), e); + e.printStackTrace(); + } + } + TestCaseLoader.sendTestRunResultsToNastradamus(); } @@ -351,7 +367,10 @@ public class TestAll implements Test { return !"true".equals(System.getProperty("intellij.build.test.ignoreFirstAndLastTests")); } - private void runNextTest(final TestResult testResult, int totalTests, Class testCaseClass) { + private void runOrCollectNextTest(@NotNull final TestResult testResult, + int totalTests, + @NotNull Class testCaseClass, + @Nullable List collectedTests) { myRunTests++; int errorCount = testResult.errorCount(); @@ -363,9 +382,18 @@ public class TestAll implements Test { return; } - log("\nRunning " + testCaseClass.getName()); + String caseClassName = testCaseClass.getName(); Test test = getTest(testCaseClass); - if (test == null) return; + if (test == null) { + log("\nSkipping " + caseClassName + ": no Test detected"); + return; + } + log("\nRunning " + caseClassName); + + if (collectedTests != null) { + collectedTests.add(caseClassName); + return; + } try { test.run(testResult);