IJPL-453 report logged errors as separate test failures

GitOrigin-RevId: 427e5e77a493457d90060c236c114ea92be247b5

platform/platform-tests/testSrc/com/intellij/openapi/diagnostic/JUnit3AndAHalfLoggedErrorTest.kt

@@ -18,9 +18,9 @@ class JUnit3AndAHalfLoggedErrorTest : UsefulTestCase() {
     private val LOG = Logger.getInstance(JUnit3AndAHalfLoggedErrorTest::class.java)
   }
 
-  // It is expected that this test fails and all 4 logged errors are visible in the test failure.
+  // It is expected that this test does not fail, and all 4 logged errors are reported as separate test failures.
   @Test
-  fun `logged error fails the test`() {
+  fun `logged error does not fail the test`() {
     LOG.error(Throwable())
     LOG.error(Throwable("throwable message 1"))
     LOG.error("error with message", Throwable())

platform/platform-tests/testSrc/com/intellij/openapi/diagnostic/JUnit3LoggedErrorTest.kt

@@ -14,8 +14,8 @@ class JUnit3LoggedErrorTest : UsefulTestCase() {
     private val LOG = Logger.getInstance(JUnit3LoggedErrorTest::class.java)
   }
 
-  // It is expected that this test fails and all 4 logged errors are visible in the test failure.
-  fun `test logged error fails the test`() {
+  // It is expected that this test does not fail, and all 4 logged errors are reported as separate test failures.
+  fun `test logged error does not fail the test`() {
     LOG.error(Throwable())
     LOG.error(Throwable("throwable message 1"))
     LOG.error("error with message", Throwable())

platform/platform-tests/testSrc/com/intellij/openapi/diagnostic/JUnit4LoggedErrorTest.kt

@@ -27,9 +27,9 @@ class JUnit4LoggedErrorTest {
   @JvmField
   val testLoggerWatcher: TestRule = TestLoggerFactory.createTestWatcher()
 
-  // It is expected that this test fails and all 4 logged errors are visible in the test failure.
+  // It is expected that this test does not fail, and all 4 logged errors are reported as separate test failures.
   @Test
-  fun `logged error fails the test`() {
+  fun `logged error does not fail the test`() {
     LOG.error(Throwable())
     LOG.error(Throwable("throwable message 1"))
     LOG.error("error with message", Throwable())

platform/testFramework/core/intellij.platform.testFramework.core.iml

@@ -23,5 +23,6 @@
     <orderEntry type="library" name="jackson-module-kotlin" level="project" />
     <orderEntry type="library" name="http-client" level="project" />
     <orderEntry type="library" name="kotlinx-collections-immutable" level="project" />
+    <orderEntry type="module" module-name="intellij.platform.testFramework.teamCity" />
   </component>
 </module>

platform/testFramework/core/src/com/intellij/testFramework/TestLoggerFactory.java

@@ -39,7 +39,7 @@ import java.util.logging.StreamHandler;
 import java.util.stream.Stream;
 
 import static com.intellij.openapi.application.PathManager.PROPERTY_LOG_PATH;
-import static com.intellij.testFramework.TestLoggerKt.rethrowErrorsLoggedInTheCurrentThread;
+import static com.intellij.testFramework.TestLoggerKt.recordErrorsLoggedInTheCurrentThreadAndReportThemAsFailures;
 import static java.util.Objects.requireNonNullElse;
 
 @SuppressWarnings({"CallToPrintStackTrace", "UseOfSystemOutOrSystemErr"})
@@ -405,7 +405,7 @@ public final class TestLoggerFactory implements Logger.Factory {
       .around((base, description) -> new Statement() {
         @Override
         public void evaluate() {
-          rethrowErrorsLoggedInTheCurrentThread(() -> base.evaluate());
+          recordErrorsLoggedInTheCurrentThreadAndReportThemAsFailures(() -> base.evaluate());
         }
       });
   }

platform/testFramework/core/src/com/intellij/testFramework/errorLogReporting.kt

@@ -0,0 +1,43 @@
+// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package com.intellij.testFramework
+
+import com.intellij.platform.testFramework.teamCity.convertToHashCodeWithOnlyLetters
+import com.intellij.platform.testFramework.teamCity.generifyErrorMessage
+import com.intellij.platform.testFramework.teamCity.reportTestFailure
+
+internal fun ErrorLog.reportAsFailures() {
+  val errors = takeLoggedErrors()
+  for (error in errors) {
+    logAsTeamcityTestFailure(error)
+  }
+}
+
+// Avoid changing the test name!
+// TeamCity remembers failures by the test name.
+// Changing the test name results in effectively new failed tests being reported,
+// so all saved TC data about previous failures will not apply, including muted state and investigations.
+// Some exception messages include file names, system hash codes (Object.toString), etc.
+// To make the test name stable between different test runs, such data is stripped out before computing the test name.
+private fun logAsTeamcityTestFailure(error: LoggedError) {
+  val message = findMessage(error)
+  val stackTraceContent = error.stackTraceToString()
+  val stackTraceHash = convertToHashCodeWithOnlyLetters(generifyErrorMessage(stackTraceContent).hashCode())
+  val generifiedMessage = if (message == null) "Error logged without message" else generifyErrorMessage(message)
+  val testName = "$stackTraceHash ($generifiedMessage)"
+  System.out.reportTestFailure(testName, message ?: "", stackTraceContent)
+}
+
+private fun findMessage(t: Throwable): String? {
+  var current: Throwable = t
+  while (true) {
+    val message = current.message
+    if (!message.isNullOrBlank()) {
+      return message
+    }
+    val cause = current.cause
+    if (cause == null || cause == current) {
+      return null
+    }
+    current = cause
+  }
+}

platform/testFramework/core/src/com/intellij/testFramework/testLogger.kt

@@ -67,36 +67,27 @@ private fun collectErrorsLoggedInTheCurrentThread(executable: () -> Unit): List<
  * to a fresh [TestLoggerAssertionError], which is then thrown.
  */
 @Internal
-fun <T> rethrowErrorsLoggedInTheCurrentThread(executable: () -> T): T {
+fun <T> recordErrorsLoggedInTheCurrentThreadAndReportThemAsFailures(executable: () -> T): T {
   if (System.getProperty("intellij.testFramework.rethrow.logged.errors") == "true") {
     return executable()
   }
   val errorLog = ErrorLog()
-  val result: T = try {
+  try {
     withErrorLog(errorLog).use { _ ->
-      executable()
+      return executable()
     }
   }
-  catch (t: Throwable) {
-    val loggedErrors = errorLog.takeLoggedErrors()
-    if (loggedErrors.isNotEmpty()) {
-      rethrowLoggedErrors(testFailure = t, loggedErrors)
-    }
-    throw t
+  finally {
+    errorLog.reportAsFailures()
   }
-  val loggedErrors = errorLog.takeLoggedErrors()
-  if (loggedErrors.isNotEmpty()) {
-    rethrowLoggedErrors(testFailure = null, loggedErrors)
-  }
-  return result
 }
 
 /**
  * An overload for Java.
  */
 @Internal
-fun rethrowErrorsLoggedInTheCurrentThread(executable: ThrowableRunnable<*>) {
-  rethrowErrorsLoggedInTheCurrentThread(executable::run)
+fun recordErrorsLoggedInTheCurrentThreadAndReportThemAsFailures(executable: ThrowableRunnable<*>) {
+  recordErrorsLoggedInTheCurrentThreadAndReportThemAsFailures(executable::run)
 }
 
 private fun rethrowLoggedErrors(

platform/testFramework/junit5/src/impl/TestLoggerInterceptor.kt

@@ -1,14 +1,14 @@
 // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package com.intellij.testFramework.junit5.impl
 
-import com.intellij.testFramework.rethrowErrorsLoggedInTheCurrentThread
+import com.intellij.testFramework.recordErrorsLoggedInTheCurrentThreadAndReportThemAsFailures
 import org.junit.jupiter.api.extension.ExtensionContext
 import org.junit.jupiter.api.extension.InvocationInterceptor
 
 internal class TestLoggerInterceptor : AbstractInvocationInterceptor() {
 
   override fun <T> intercept(invocation: InvocationInterceptor.Invocation<T>, context: ExtensionContext): T {
-    return rethrowErrorsLoggedInTheCurrentThread {
+    return recordErrorsLoggedInTheCurrentThreadAndReportThemAsFailures {
       invocation.proceed()
     }
   }

platform/testFramework/junit5/test/showcase/JUnit5LoggedErrorTest.kt

@@ -15,9 +15,9 @@ class JUnit5LoggedErrorTest {
     private val LOG = Logger.getInstance(JUnit5ApplicationTest::class.java)
   }
 
-  // It is expected that this test fails and all 4 logged errors are visible in the test failure.
+  // It is expected that this test does not fail, and all 4 logged errors are reported as separate test failures.
   @Test
-  fun `logged error fails the test`() {
+  fun `logged error does not fail the test`() {
     LOG.error(Throwable())
     LOG.error(Throwable("throwable message 1"))
     LOG.error("error with message", Throwable())

platform/testFramework/src/com/intellij/testFramework/UsefulTestCase.java

@@ -69,7 +69,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.function.BiPredicate;
 import java.util.function.Supplier;
 
-import static com.intellij.testFramework.TestLoggerKt.rethrowErrorsLoggedInTheCurrentThread;
+import static com.intellij.testFramework.TestLoggerKt.recordErrorsLoggedInTheCurrentThreadAndReportThemAsFailures;
 import static com.intellij.testFramework.common.Cleanup.cleanupSwingDataStructures;
 import static com.intellij.testFramework.common.TestEnvironmentKt.initializeTestEnvironment;
 import static org.junit.Assume.assumeTrue;
@@ -523,7 +523,7 @@ Most likely there was an uncaught exception in asynchronous execution that resul
       boolean success = false;
       TestLoggerFactory.onTestStarted();
       try {
-        rethrowErrorsLoggedInTheCurrentThread(() -> testRunnable.run());
+        recordErrorsLoggedInTheCurrentThreadAndReportThemAsFailures(testRunnable);
         success = true;
       }
       catch (AssumptionViolatedException e) {