MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

[test framework] IJI-1588 Run every test class or package in a dedicated runtime

Browse Source 
For now, without retries.
First pass collects all test classes without executing them.
Then they run either one by one, or grouped by package.

GitOrigin-RevId: cbdea0a64313cdc1dd0bbc6a281b9cd679c2a512
This commit is contained in:
Vladislav Rassokhin
2024-03-27 15:30:47 +00:00
committed by intellij-monorepo-bot
parent 2c5a3e9498
commit be9ddbb56c
5 changed files with 227 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/build-scripts/src/org/jetbrains/intellij/build/TestingOptions.kt
+7 -3
View File
@@ -1,4 +1,4 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build
import com.intellij.TestCaseLoader
@@ -132,9 +132,13 @@ open class TestingOptions {
var batchTestIncludes: String? = System.getProperty("intellij.build.test.batchTest.includes")
/**
* Run only whole classes in forked Runtime in case if [batchTestIncludes] mode enabled
* Run only whole classes/packages in forked Runtime
* Allowed values:
* * `false`
* * `class`
* * `package`
*/
var isDedicatedRuntimePerClassEnabled: Boolean = SystemProperties.getBooleanProperty("intellij.build.test.dedicated.runtime.per.class.enabled", false)
var isDedicatedTestRuntime: String = System.getProperty("intellij.build.test.dedicated.runtime", "false")
var isPerformanceTestsOnly = SystemProperties.getBooleanProperty(PERFORMANCE_TESTS_ONLY_FLAG, false)
platform/build-scripts/src/org/jetbrains/intellij/build/impl/TestingTasksImpl.kt
+131 -67
View File
@@ -36,6 +36,7 @@ import java.nio.file.Path
import java.util.concurrent.Callable
import java.util.regex.Pattern
import java.util.stream.Stream
import kotlin.io.path.absolutePathString
import kotlin.io.path.readLines
internal class TestingTasksImpl(private val context: CompilationContext, private val options: TestingOptions) : TestingTasks {
@@ -613,7 +614,7 @@ internal class TestingTasksImpl(private val context: CompilationContext, private
.toList()
}
private fun getTestClassesForModule(mainModule: String, filteringPattern: Pattern = Pattern.compile(".*.class")): List<String> {
private fun getTestClassesForModule(mainModule: String, filteringPattern: Pattern = Pattern.compile(".*\\.class")): List<String> {
val mainModuleTestsOutput = context.getModuleTestsOutputPath(context.findRequiredModule(mainModule))
val root = Path.of(mainModuleTestsOutput)
@@ -661,7 +662,7 @@ internal class TestingTasksImpl(private val context: CompilationContext, private
val jUnit4And5TestMethods = getAnnotatedTestMethods(aClass, testAnnotation4, testAnnotation5, testFactoryAnnotation5)
// Run JUnit 4 and 5 whole test classes separately
if (options.isDedicatedRuntimePerClassEnabled && jUnit4And5TestMethods.isNotEmpty()) {
if (options.isDedicatedTestRuntime != "false" && jUnit4And5TestMethods.isNotEmpty()) {
val exitCode = runJUnit5Engine(systemProperties = systemProperties,
jvmArgs = jvmArgs,
envVariables = envVariables,
@@ -735,6 +736,111 @@ internal class TestingTasksImpl(private val context: CompilationContext, private
runInBatchMode(mainModule, systemProperties, jvmArgs, envVariables, bootstrapClasspath, testClasspath)
}
}
else if (options.isDedicatedTestRuntime != "false") {
if (options.isDedicatedTestRuntime != "class" && options.isDedicatedTestRuntime != "package") {
context.messages.error("Unsupported 'intellij.build.test.dedicated.runtime' value: ${options.isDedicatedTestRuntime}. Expected 'class', 'package' or 'false'")
}
context.messages.info("Will run tests in dedicated runtimes ('${options.isDedicatedTestRuntime}')")
// First, collect all tests for both JUnit5 and JUnit3+4
val testClassesJUnit5 = blockAndSpan("collect junit 5 tests") {
val testClassesListFile = Files.createTempFile("tests-to-run-", ".list").apply { Files.delete(this) }
runJUnit5Engine(
systemProperties = systemProperties + ("intellij.build.test.list.classes" to testClassesListFile.absolutePathString()),
jvmArgs = jvmArgs,
envVariables = envVariables,
bootstrapClasspath = bootstrapClasspath,
modulePath = modulePath,
testClasspath = testClasspath,
suiteName = null,
methodName = null)
return@blockAndSpan testClassesListFile.let { if (Files.exists(it)) it.readLines() else emptyList() }
}
val testClassesJUnit34 = blockAndSpan("collect junit 3+4 tests") {
val testClassesListFile = Files.createTempFile("tests-to-run-", ".list").apply { Files.delete(this) }
runJUnit5Engine(
systemProperties = systemProperties + ("intellij.build.test.list.classes" to testClassesListFile.absolutePathString()),
jvmArgs = jvmArgs,
envVariables = envVariables,
bootstrapClasspath = bootstrapClasspath,
modulePath = modulePath,
testClasspath = testClasspath,
suiteName = options.bootstrapSuite,
methodName = null)
return@blockAndSpan testClassesListFile.let { if (Files.exists(it)) it.readLines() else emptyList() }
}
if (testClassesJUnit5.isEmpty() && testClassesJUnit34.isEmpty() &&
// a bucket might be empty for run configurations with too few tests due to imperfect tests balancing
options.bucketsCount < 2) {
throw NoTestsFound()
}
if (options.isDedicatedTestRuntime == "class") {
fun runOneClass(testClassName: String) {
blockAndSpan("running test class '$testClassName'") {
runJUnit5Engine(
systemProperties = systemProperties,
jvmArgs = jvmArgs,
envVariables = envVariables,
bootstrapClasspath = bootstrapClasspath,
modulePath = modulePath,
testClasspath = testClasspath,
suiteName = testClassName,
methodName = null
)
}
}
if (testClassesJUnit5.isNotEmpty()) {
context.messages.info("Will run JUnit 5 tests:\n${testClassesJUnit5.joinToString("\n")}")
testClassesJUnit5.forEach(::runOneClass)
}
if (testClassesJUnit34.isNotEmpty()) {
context.messages.info("Will run JUnit 3+4 tests:\n${testClassesJUnit34.joinToString("\n")}")
testClassesJUnit34.forEach(::runOneClass)
}
}
else if (options.isDedicatedTestRuntime == "package") {
fun groupByPackages(tests: List<String>): Map<String, List<String>> {
return tests.groupBy {
val i = it.lastIndexOf('.')
if (i != -1) it.substring(0, i) else ""
}
}
fun runOnePackage(entry: Map.Entry<String, List<String>>) {
val packageName = entry.key
val classes = entry.value
blockAndSpan("running tests in package '$packageName'") {
runJUnit5Engine(
systemProperties = systemProperties,
jvmArgs = jvmArgs,
envVariables = envVariables,
bootstrapClasspath = bootstrapClasspath,
modulePath = modulePath,
testClasspath = testClasspath,
suiteName = "__classes__",
methodName = classes.joinToString(";")
)
}
}
if (testClassesJUnit5.isNotEmpty()) {
val packages = groupByPackages(testClassesJUnit5)
context.messages.info(packages.entries.joinToString(prefix = "Will run JUnit 5 packages:\n", separator = "\n") { e ->
e.value.joinToString(prefix = "${e.key}\n ", separator = "\n ")
})
packages.forEach(::runOnePackage)
}
if (testClassesJUnit34.isNotEmpty()) {
val packages = groupByPackages(testClassesJUnit34)
context.messages.info(packages.entries.joinToString(prefix = "Will run JUnit 3+4 packages:\n", separator = "\n") { e ->
e.value.joinToString(prefix = "${e.key}\n ", separator = "\n ")
})
packages.forEach(::runOnePackage)
}
}
}
else {
val failedClassesJUnit5List = Files.createTempFile("failed-classes-junit5-", ".list").apply { Files.delete(this) }
val failedClassesJUnit34List = Files.createTempFile("failed-classes-junit34-", ".list").apply { Files.delete(this) }
@@ -754,74 +860,30 @@ internal class TestingTasksImpl(private val context: CompilationContext, private
val additionalProperties: Map<String, String> = if (attempt > 1) mapOf("intellij.build.test.ignoreFirstAndLastTests" to "true") else emptyMap()
val exitCode5: Int = if (runJUnit5) blockAndSpan("run junit 5 tests${spanNameSuffix}") {
if (options.isDedicatedRuntimePerClassEnabled) {
context.messages.info("Creation of a dedicated runtime for each class is enabled")
val testClasses = getTestClassesForModule(mainModule = mainModule)
var batchExitCode: Int = NO_TESTS_ERROR
for (testClass in testClasses) {
val testClassName = FileUtilRt.getNameWithoutExtension(testClass).replace('/', '.')
val exitCode = runJUnit5Engine(systemProperties = systemProperties + additionalProperties + additionalPropertiesJUnit5,
jvmArgs = jvmArgs,
envVariables = envVariables,
bootstrapClasspath = bootstrapClasspath,
modulePath = modulePath,
testClasspath = testClasspath,
suiteName = testClassName,
methodName = null)
if (exitCode == NO_TESTS_ERROR) batchExitCode = NO_TESTS_ERROR
}
batchExitCode
}
else {
runJUnit5Engine(systemProperties = systemProperties + additionalProperties + additionalPropertiesJUnit5,
jvmArgs = jvmArgs,
envVariables = envVariables,
bootstrapClasspath = bootstrapClasspath,
modulePath = modulePath,
testClasspath = testClasspath,
suiteName = null,
methodName = null)
}
runJUnit5Engine(
systemProperties = systemProperties + additionalProperties + additionalPropertiesJUnit5,
jvmArgs = jvmArgs,
envVariables = envVariables,
bootstrapClasspath = bootstrapClasspath,
modulePath = modulePath,
testClasspath = testClasspath,
suiteName = null,
methodName = null
)
}
else 0
val exitCode34: Int = if (runJUnit34) blockAndSpan("run junit 3+4 tests${spanNameSuffix}") {
if (options.isDedicatedRuntimePerClassEnabled) {
context.messages.info("Creation of a dedicated runtime for each class is enabled")
val testClasses = getTestClassesForModule(mainModule = mainModule)
var batchExitCode: Int = NO_TESTS_ERROR
for (testClass in testClasses) {
val testClassName = FileUtilRt.getNameWithoutExtension(testClass).replace('/', '.')
val exitCode = runJUnit5Engine(systemProperties = systemProperties + additionalProperties + additionalPropertiesJUnit34,
jvmArgs = jvmArgs,
envVariables = envVariables,
bootstrapClasspath = bootstrapClasspath,
modulePath = modulePath,
testClasspath = testClasspath,
suiteName = testClassName,
methodName = null)
if (exitCode == NO_TESTS_ERROR) batchExitCode = NO_TESTS_ERROR
}
batchExitCode
}
else {
runJUnit5Engine(systemProperties = systemProperties + additionalProperties + additionalPropertiesJUnit34,
jvmArgs = jvmArgs,
envVariables = envVariables,
bootstrapClasspath = bootstrapClasspath,
modulePath = modulePath,
testClasspath = testClasspath,
suiteName = options.bootstrapSuite,
methodName = null)
}
runJUnit5Engine(
systemProperties = systemProperties + additionalProperties + additionalPropertiesJUnit34,
jvmArgs = jvmArgs,
envVariables = envVariables,
bootstrapClasspath = bootstrapClasspath,
modulePath = modulePath,
testClasspath = testClasspath,
suiteName = options.bootstrapSuite,
methodName = null
)
}
else 0
@@ -874,7 +936,9 @@ internal class TestingTasksImpl(private val context: CompilationContext, private
if (modulePath == null) {
appendJUnitStarter(classpath)
}
if (!isBootstrapSuiteDefault || isRunningInBatchMode || options.isDedicatedRuntimePerClassEnabled || suiteName == null) {
if (!isBootstrapSuiteDefault || isRunningInBatchMode
|| options.isDedicatedTestRuntime != "false"
|| suiteName == null) {
classpath += testClasspath
}
args += "-classpath"
platform/testFramework/bootstrap/src/com/intellij/tests/JUnit5TeamCityRunnerForTestAllSuite.java
+27 -9
View File
@@ -9,15 +9,14 @@ import junit.framework.JUnit4TestAdapterCache;
import junit.framework.TestResult;
import junit.framework.TestSuite;
import org.junit.platform.engine.DiscoverySelector;
import org.junit.platform.engine.Filter;
import org.junit.platform.engine.TestExecutionResult;
import org.junit.platform.engine.discovery.ClassNameFilter;
import org.junit.platform.engine.discovery.DiscoverySelectors;
import org.junit.platform.engine.reporting.ReportEntry;
import org.junit.platform.engine.support.descriptor.ClassSource;
import org.junit.platform.engine.support.descriptor.MethodSource;
import org.junit.platform.launcher.Launcher;
import org.junit.platform.launcher.TestExecutionListener;
import org.junit.platform.launcher.TestIdentifier;
import org.junit.platform.launcher.TestPlan;
import org.junit.platform.launcher.*;
import org.junit.platform.launcher.core.LauncherConfig;
import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder;
import org.junit.platform.launcher.core.LauncherFactory;
@@ -34,19 +33,39 @@ import java.io.StringWriter;
import java.util.*;
// Used to run JUnit 3/4 tests via JUnit 5 runtime
@SuppressWarnings("UseOfSystemOutOrSystemErr")
public final class JUnit5TeamCityRunnerForTestAllSuite {
public static void main(String[] args) throws ClassNotFoundException {
if (args.length != 1 && args.length != 2) {
System.err.printf("Expected one or two arguments, got %d: %s%n", args.length, Arrays.toString(args));
System.exit(1);
}
try {
Launcher launcher = LauncherFactory.create(LauncherConfig.builder().enableLauncherSessionListenerAutoRegistration(false).build());
DiscoverySelector selector;
List<? extends DiscoverySelector> selectors;
List<Filter<?>> filters = new ArrayList<>(0);
if (args.length == 1) {
selector = DiscoverySelectors.selectClass(args[0]);
selectors = Collections.singletonList(DiscoverySelectors.selectClass(args[0]));
}
else if (args[0].equals("__package__")) {
selectors = Collections.singletonList(DiscoverySelectors.selectPackage(args[1]));
// exclude subpackages
filters.add(ClassNameFilter.excludeClassNamePatterns("\\Q" + args[1] + "\\E\\.[^.]+\\..*"));
}
else if (args[0].equals("__classes__")) {
String[] classes = args[1].split(";");
selectors = Arrays.stream(classes).map(DiscoverySelectors::selectClass).toList();
}
else {
selector = DiscoverySelectors.selectMethod(args[0], args[1]);
selectors = Collections.singletonList(DiscoverySelectors.selectMethod(args[0], args[1]));
}
LauncherDiscoveryRequest discoveryRequest = LauncherDiscoveryRequestBuilder.request()
.selectors(selectors)
.filters(filters.toArray(new Filter[0]))
.build();
TCExecutionListener listener = new TCExecutionListener();
launcher.execute(LauncherDiscoveryRequestBuilder.request().selectors(selector).build(), listener);
TestPlan testPlan = launcher.discover(discoveryRequest);
launcher.execute(testPlan, listener);
if (!listener.smthExecuted()) {
//see org.jetbrains.intellij.build.impl.TestingTasksImpl.NO_TESTS_ERROR
System.exit(42);
@@ -97,7 +116,6 @@ public final class JUnit5TeamCityRunnerForTestAllSuite {
};
}
@SuppressWarnings("UseOfSystemOutOrSystemErr")
public static class TCExecutionListener implements TestExecutionListener {
/**
* The same constant as com.intellij.rt.execution.TestListenerProtocol.CLASS_CONFIGURATION
platform/testFramework/bootstrap/src/com/intellij/tests/JUnit5TeamCityRunnerForTestsOnClasspath.java
+30 -4
View File
@@ -12,20 +12,19 @@ import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder;
import org.junit.platform.launcher.core.LauncherFactory;
import org.junit.vintage.engine.descriptor.VintageTestDescriptor;
import java.io.IOException;
import java.lang.invoke.MethodHandle;
import java.lang.invoke.MethodHandles;
import java.lang.invoke.MethodType;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;
import java.util.List;
import java.util.ServiceLoader;
import java.util.Set;
import java.util.*;
import java.util.stream.Collectors;
// Used to run JUnit 5 tests via JUnit 5 runtime
@SuppressWarnings({"UseOfSystemOutOrSystemErr", "CallToPrintStackTrace"})
public final class JUnit5TeamCityRunnerForTestsOnClasspath {
private static final String ourCollectTestsFile = System.getProperty("intellij.build.test.list.classes");
public static void main(String[] args) {
try {
@@ -67,6 +66,10 @@ public final class JUnit5TeamCityRunnerForTestsOnClasspath {
.filters(nameFilter, postDiscoveryFilter, EngineFilter.excludeEngines(VintageTestDescriptor.ENGINE_ID)).build();
TestPlan testPlan = launcher.discover(discoveryRequest);
if (testPlan.containsTests()) {
if (ourCollectTestsFile != null) {
saveListOfTestClasses(testPlan);
return;
}
launcher.execute(testPlan, new JUnit5TeamCityRunnerForTestAllSuite.TCExecutionListener());
}
else {
@@ -161,4 +164,27 @@ public final class JUnit5TeamCityRunnerForTestsOnClasspath {
}
};
}
private static void saveListOfTestClasses(TestPlan testPlan) {
ArrayList<String> testClasses = new ArrayList<>(0);
for (TestIdentifier root : testPlan.getRoots()) {
Set<TestIdentifier> firstLevel = testPlan.getChildren(root);
for (TestIdentifier identifier : firstLevel) {
identifier.getSource()
.filter(source -> source instanceof ClassSource)
.map(source -> ((ClassSource)source).getClassName())
.ifPresent(name -> testClasses.add(name));
}
}
Path path = Path.of(ourCollectTestsFile);
try {
Files.createDirectories(path.getParent());
Files.write(path, testClasses);
}
catch (IOException e) {
System.err.printf("Cannot save list of test classes to '%s': %s%n", path.toAbsolutePath(), e);
e.printStackTrace();
System.exit(1);
}
}
}
platform/testFramework/core/src/com/intellij/TestAll.java
+32 -4
View File
@@ -101,6 +101,7 @@ public class TestAll implements Test {
private static final List<Throwable> ourClassLoadingProblems = new ArrayList<>();
private static JUnit4TestAdapterCache ourUnit4TestAdapterCache;
private static final String ourCollectTestsFile = System.getProperty("intellij.build.test.list.classes", null);
public TestAll(String rootPackage) throws Throwable {
this(rootPackage, getClassRoots());
@@ -274,8 +275,9 @@ public class TestAll implements Test {
System.out.println("------");
int totalTests = classes.size();
final List<String> collectedTests = ourCollectTestsFile != null ? new ArrayList<>(totalTests) : null;
for (Class<?> aClass : testsToRun) {
runNextTest(testResult, totalTests, aClass);
runOrCollectNextTest(testResult, totalTests, aClass, collectedTests);
if (testResult.shouldStop()) break;
}
@@ -296,6 +298,20 @@ public class TestAll implements Test {
}
}
if (collectedTests != null) {
Path path = Path.of(ourCollectTestsFile);
try {
collectedTests.remove("_FirstInSuiteTest");
collectedTests.remove("_LastInSuiteTest");
Files.createDirectories(path.getParent());
Files.write(path, collectedTests);
}
catch (IOException e) {
System.err.printf("Cannot save list of test classes to '%s': %s%n", path.toAbsolutePath(), e);
e.printStackTrace();
}
}
TestCaseLoader.sendTestRunResultsToNastradamus();
}
@@ -351,7 +367,10 @@ public class TestAll implements Test {
return !"true".equals(System.getProperty("intellij.build.test.ignoreFirstAndLastTests"));
}
private void runNextTest(final TestResult testResult, int totalTests, Class<?> testCaseClass) {
private void runOrCollectNextTest(@NotNull final TestResult testResult,
int totalTests,
@NotNull Class<?> testCaseClass,
@Nullable List<String> collectedTests) {
myRunTests++;
int errorCount = testResult.errorCount();
@@ -363,9 +382,18 @@ public class TestAll implements Test {
return;
}
log("\nRunning " + testCaseClass.getName());
String caseClassName = testCaseClass.getName();
Test test = getTest(testCaseClass);
if (test == null) return;
if (test == null) {
log("\nSkipping " + caseClassName + ": no Test detected");
return;
}
log("\nRunning " + caseClassName);
if (collectedTests != null) {
collectedTests.add(caseClassName);
return;
}
try {
test.run(testResult);