diff --git a/platform/platform-tests/testSrc/com/intellij/openapi/diagnostic/JUnit3AndAHalfLoggedErrorTest.kt b/platform/platform-tests/testSrc/com/intellij/openapi/diagnostic/JUnit3AndAHalfLoggedErrorTest.kt index 16c6f0cfffc5..fb32ab20c54b 100644 --- a/platform/platform-tests/testSrc/com/intellij/openapi/diagnostic/JUnit3AndAHalfLoggedErrorTest.kt +++ b/platform/platform-tests/testSrc/com/intellij/openapi/diagnostic/JUnit3AndAHalfLoggedErrorTest.kt @@ -18,9 +18,9 @@ class JUnit3AndAHalfLoggedErrorTest : UsefulTestCase() { private val LOG = Logger.getInstance(JUnit3AndAHalfLoggedErrorTest::class.java) } - // It is expected that this test fails and all 4 logged errors are visible in the test failure. + // It is expected that this test does not fail, and all 4 logged errors are reported as separate test failures. @Test - fun `logged error fails the test`() { + fun `logged error does not fail the test`() { LOG.error(Throwable()) LOG.error(Throwable("throwable message 1")) LOG.error("error with message", Throwable()) diff --git a/platform/platform-tests/testSrc/com/intellij/openapi/diagnostic/JUnit3LoggedErrorTest.kt b/platform/platform-tests/testSrc/com/intellij/openapi/diagnostic/JUnit3LoggedErrorTest.kt index 65fd251fc95c..d7579ab268e4 100644 --- a/platform/platform-tests/testSrc/com/intellij/openapi/diagnostic/JUnit3LoggedErrorTest.kt +++ b/platform/platform-tests/testSrc/com/intellij/openapi/diagnostic/JUnit3LoggedErrorTest.kt @@ -14,8 +14,8 @@ class JUnit3LoggedErrorTest : UsefulTestCase() { private val LOG = Logger.getInstance(JUnit3LoggedErrorTest::class.java) } - // It is expected that this test fails and all 4 logged errors are visible in the test failure. - fun `test logged error fails the test`() { + // It is expected that this test does not fail, and all 4 logged errors are reported as separate test failures. + fun `test logged error does not fail the test`() { LOG.error(Throwable()) LOG.error(Throwable("throwable message 1")) LOG.error("error with message", Throwable()) diff --git a/platform/platform-tests/testSrc/com/intellij/openapi/diagnostic/JUnit4LoggedErrorTest.kt b/platform/platform-tests/testSrc/com/intellij/openapi/diagnostic/JUnit4LoggedErrorTest.kt index 72bf1b44819f..31931b7345b1 100644 --- a/platform/platform-tests/testSrc/com/intellij/openapi/diagnostic/JUnit4LoggedErrorTest.kt +++ b/platform/platform-tests/testSrc/com/intellij/openapi/diagnostic/JUnit4LoggedErrorTest.kt @@ -27,9 +27,9 @@ class JUnit4LoggedErrorTest { @JvmField val testLoggerWatcher: TestRule = TestLoggerFactory.createTestWatcher() - // It is expected that this test fails and all 4 logged errors are visible in the test failure. + // It is expected that this test does not fail, and all 4 logged errors are reported as separate test failures. @Test - fun `logged error fails the test`() { + fun `logged error does not fail the test`() { LOG.error(Throwable()) LOG.error(Throwable("throwable message 1")) LOG.error("error with message", Throwable()) diff --git a/platform/testFramework/core/intellij.platform.testFramework.core.iml b/platform/testFramework/core/intellij.platform.testFramework.core.iml index 2414fcbd6322..175164803eeb 100644 --- a/platform/testFramework/core/intellij.platform.testFramework.core.iml +++ b/platform/testFramework/core/intellij.platform.testFramework.core.iml @@ -23,5 +23,6 @@ + \ No newline at end of file diff --git a/platform/testFramework/core/src/com/intellij/testFramework/TestLoggerFactory.java b/platform/testFramework/core/src/com/intellij/testFramework/TestLoggerFactory.java index f9d42716a8b6..40a985b0cc86 100644 --- a/platform/testFramework/core/src/com/intellij/testFramework/TestLoggerFactory.java +++ b/platform/testFramework/core/src/com/intellij/testFramework/TestLoggerFactory.java @@ -39,7 +39,7 @@ import java.util.logging.StreamHandler; import java.util.stream.Stream; import static com.intellij.openapi.application.PathManager.PROPERTY_LOG_PATH; -import static com.intellij.testFramework.TestLoggerKt.rethrowErrorsLoggedInTheCurrentThread; +import static com.intellij.testFramework.TestLoggerKt.recordErrorsLoggedInTheCurrentThreadAndReportThemAsFailures; import static java.util.Objects.requireNonNullElse; @SuppressWarnings({"CallToPrintStackTrace", "UseOfSystemOutOrSystemErr"}) @@ -405,7 +405,7 @@ public final class TestLoggerFactory implements Logger.Factory { .around((base, description) -> new Statement() { @Override public void evaluate() { - rethrowErrorsLoggedInTheCurrentThread(() -> base.evaluate()); + recordErrorsLoggedInTheCurrentThreadAndReportThemAsFailures(() -> base.evaluate()); } }); } diff --git a/platform/testFramework/core/src/com/intellij/testFramework/errorLogReporting.kt b/platform/testFramework/core/src/com/intellij/testFramework/errorLogReporting.kt new file mode 100644 index 000000000000..ffd005df97f6 --- /dev/null +++ b/platform/testFramework/core/src/com/intellij/testFramework/errorLogReporting.kt @@ -0,0 +1,43 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.testFramework + +import com.intellij.platform.testFramework.teamCity.convertToHashCodeWithOnlyLetters +import com.intellij.platform.testFramework.teamCity.generifyErrorMessage +import com.intellij.platform.testFramework.teamCity.reportTestFailure + +internal fun ErrorLog.reportAsFailures() { + val errors = takeLoggedErrors() + for (error in errors) { + logAsTeamcityTestFailure(error) + } +} + +// Avoid changing the test name! +// TeamCity remembers failures by the test name. +// Changing the test name results in effectively new failed tests being reported, +// so all saved TC data about previous failures will not apply, including muted state and investigations. +// Some exception messages include file names, system hash codes (Object.toString), etc. +// To make the test name stable between different test runs, such data is stripped out before computing the test name. +private fun logAsTeamcityTestFailure(error: LoggedError) { + val message = findMessage(error) + val stackTraceContent = error.stackTraceToString() + val stackTraceHash = convertToHashCodeWithOnlyLetters(generifyErrorMessage(stackTraceContent).hashCode()) + val generifiedMessage = if (message == null) "Error logged without message" else generifyErrorMessage(message) + val testName = "$stackTraceHash ($generifiedMessage)" + System.out.reportTestFailure(testName, message ?: "", stackTraceContent) +} + +private fun findMessage(t: Throwable): String? { + var current: Throwable = t + while (true) { + val message = current.message + if (!message.isNullOrBlank()) { + return message + } + val cause = current.cause + if (cause == null || cause == current) { + return null + } + current = cause + } +} diff --git a/platform/testFramework/core/src/com/intellij/testFramework/testLogger.kt b/platform/testFramework/core/src/com/intellij/testFramework/testLogger.kt index ec66474a7593..1a08b5a80360 100644 --- a/platform/testFramework/core/src/com/intellij/testFramework/testLogger.kt +++ b/platform/testFramework/core/src/com/intellij/testFramework/testLogger.kt @@ -67,36 +67,27 @@ private fun collectErrorsLoggedInTheCurrentThread(executable: () -> Unit): List< * to a fresh [TestLoggerAssertionError], which is then thrown. */ @Internal -fun rethrowErrorsLoggedInTheCurrentThread(executable: () -> T): T { +fun recordErrorsLoggedInTheCurrentThreadAndReportThemAsFailures(executable: () -> T): T { if (System.getProperty("intellij.testFramework.rethrow.logged.errors") == "true") { return executable() } val errorLog = ErrorLog() - val result: T = try { + try { withErrorLog(errorLog).use { _ -> - executable() + return executable() } } - catch (t: Throwable) { - val loggedErrors = errorLog.takeLoggedErrors() - if (loggedErrors.isNotEmpty()) { - rethrowLoggedErrors(testFailure = t, loggedErrors) - } - throw t + finally { + errorLog.reportAsFailures() } - val loggedErrors = errorLog.takeLoggedErrors() - if (loggedErrors.isNotEmpty()) { - rethrowLoggedErrors(testFailure = null, loggedErrors) - } - return result } /** * An overload for Java. */ @Internal -fun rethrowErrorsLoggedInTheCurrentThread(executable: ThrowableRunnable<*>) { - rethrowErrorsLoggedInTheCurrentThread(executable::run) +fun recordErrorsLoggedInTheCurrentThreadAndReportThemAsFailures(executable: ThrowableRunnable<*>) { + recordErrorsLoggedInTheCurrentThreadAndReportThemAsFailures(executable::run) } private fun rethrowLoggedErrors( diff --git a/platform/testFramework/junit5/src/impl/TestLoggerInterceptor.kt b/platform/testFramework/junit5/src/impl/TestLoggerInterceptor.kt index 01f4f83905fd..af21763cb128 100644 --- a/platform/testFramework/junit5/src/impl/TestLoggerInterceptor.kt +++ b/platform/testFramework/junit5/src/impl/TestLoggerInterceptor.kt @@ -1,14 +1,14 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.testFramework.junit5.impl -import com.intellij.testFramework.rethrowErrorsLoggedInTheCurrentThread +import com.intellij.testFramework.recordErrorsLoggedInTheCurrentThreadAndReportThemAsFailures import org.junit.jupiter.api.extension.ExtensionContext import org.junit.jupiter.api.extension.InvocationInterceptor internal class TestLoggerInterceptor : AbstractInvocationInterceptor() { override fun intercept(invocation: InvocationInterceptor.Invocation, context: ExtensionContext): T { - return rethrowErrorsLoggedInTheCurrentThread { + return recordErrorsLoggedInTheCurrentThreadAndReportThemAsFailures { invocation.proceed() } } diff --git a/platform/testFramework/junit5/test/showcase/JUnit5LoggedErrorTest.kt b/platform/testFramework/junit5/test/showcase/JUnit5LoggedErrorTest.kt index 41a4d774bda3..90cae9c0c096 100644 --- a/platform/testFramework/junit5/test/showcase/JUnit5LoggedErrorTest.kt +++ b/platform/testFramework/junit5/test/showcase/JUnit5LoggedErrorTest.kt @@ -15,9 +15,9 @@ class JUnit5LoggedErrorTest { private val LOG = Logger.getInstance(JUnit5ApplicationTest::class.java) } - // It is expected that this test fails and all 4 logged errors are visible in the test failure. + // It is expected that this test does not fail, and all 4 logged errors are reported as separate test failures. @Test - fun `logged error fails the test`() { + fun `logged error does not fail the test`() { LOG.error(Throwable()) LOG.error(Throwable("throwable message 1")) LOG.error("error with message", Throwable()) diff --git a/platform/testFramework/src/com/intellij/testFramework/UsefulTestCase.java b/platform/testFramework/src/com/intellij/testFramework/UsefulTestCase.java index df384ccd1cef..3a36b5f4192f 100644 --- a/platform/testFramework/src/com/intellij/testFramework/UsefulTestCase.java +++ b/platform/testFramework/src/com/intellij/testFramework/UsefulTestCase.java @@ -69,7 +69,7 @@ import java.util.concurrent.TimeUnit; import java.util.function.BiPredicate; import java.util.function.Supplier; -import static com.intellij.testFramework.TestLoggerKt.rethrowErrorsLoggedInTheCurrentThread; +import static com.intellij.testFramework.TestLoggerKt.recordErrorsLoggedInTheCurrentThreadAndReportThemAsFailures; import static com.intellij.testFramework.common.Cleanup.cleanupSwingDataStructures; import static com.intellij.testFramework.common.TestEnvironmentKt.initializeTestEnvironment; import static org.junit.Assume.assumeTrue; @@ -523,7 +523,7 @@ Most likely there was an uncaught exception in asynchronous execution that resul boolean success = false; TestLoggerFactory.onTestStarted(); try { - rethrowErrorsLoggedInTheCurrentThread(() -> testRunnable.run()); + recordErrorsLoggedInTheCurrentThreadAndReportThemAsFailures(testRunnable); success = true; } catch (AssumptionViolatedException e) {