[ai-ij-exp] LLM-27517 Add 'artifacts' to eval report

(cherry picked from commit 13ae52e1ac006eb51498222b176927f4cff288c7)

IJ-MR-210595

(cherry picked from commit 329e2882ba497289e1ec07031a32a90a3ad9303e)

IJ-MR-211001

GitOrigin-RevId: 1c4c894d15789d4c5a1c30d2a1f8bca68ef3c30a

plugins/evaluation-plugin/core/src/com/intellij/cce/evaluable/AiaConstants.kt

@@ -40,4 +40,5 @@ const val AIA_CORRECT_ATTACHMENT: String = "aia_correct_attachment"
 const val AIA_RETRIEVED_ATTACHMENTS: String = "aia_retrieved_attachments"
 const val AIA_IS_CORRECT_ATTACHMENT_AMONG_RETRIEVED: String = "aia_is_correct_attachment_among_retrieved"
 const val AIA_EVAL_ARTIFACT: String = "aia_eval_artifact"
-const val AIA_EVAL_ARTIFACT_NAME: String = "aia_eval_artifact_name"
+const val AIA_EVAL_ARTIFACT_NAME: String = "aia_eval_artifact_name"
+const val AIA_EVAL_ARTIFACT_FILES: String = "aia_eval_artifact_files"

plugins/evaluation-plugin/core/src/com/intellij/cce/evaluation/data/DataPlacement.kt

@@ -132,6 +132,22 @@ sealed interface DataPlacement<In, Out> {
     }
   }
 
+  data class ArtifactFiles(val propertyKey: String) : DataPlacement<List<ArtifactFile>, ArtifactFile> {
+    override val serialName: String = "artifact_files"
+
+    override fun dump(lookup: Lookup, t: List<ArtifactFile>): Lookup {
+      return lookup.copy(
+        additionalInfo = lookup.additionalInfo + Pair(propertyKey, gson.toJsonTree(t))
+      )
+    }
+
+    override fun restore(props: DataProps): List<ArtifactFile> {
+      val files = props.lookup.additionalInfo[propertyKey] ?: return emptyList()
+      val filesJson = files as? JsonElement ?: gson.toJsonTree(files)
+      return gson.fromJson(filesJson, Array<ArtifactFile>::class.java).toList()
+    }
+  }
+
   data class AdditionalCodeCommentRanges(val propertyKey: String) : DataPlacement<List<CodeCommentRange>, List<CodeCommentRange>> {
     override val serialName: String = "code_comment_range"
 
@@ -185,6 +201,7 @@ sealed interface DataPlacement<In, Out> {
         "latency" -> Latency
         "current_file_update" -> CurrentFileUpdate
         "file_updates" -> context?.deserialize(json, FileUpdates::class.java)
+        "artifact_files" -> context?.deserialize(json, ArtifactFiles::class.java)
         "colored_insights_placement" -> context?.deserialize(json, ColoredInsightsPlacement::class.java)
         else -> throw IllegalArgumentException("Unknown type: $type")
       }
@@ -205,4 +222,4 @@ data class DataProps(
   init {
     check(session.lookups.contains(lookup))
   }
-}
+}

plugins/evaluation-plugin/core/src/com/intellij/cce/evaluation/data/EvalDataConstants.kt

@@ -11,6 +11,7 @@ import com.intellij.cce.evaluable.AIA_CORRECT_ATTACHMENT
 import com.intellij.cce.evaluable.AIA_DESCRIPTION
 import com.intellij.cce.evaluable.AIA_ERASED_APIS
 import com.intellij.cce.evaluable.AIA_EVAL_ARTIFACT
+import com.intellij.cce.evaluable.AIA_EVAL_ARTIFACT_FILES
 import com.intellij.cce.evaluable.AIA_EVAL_ARTIFACT_NAME
 import com.intellij.cce.evaluable.AIA_EXACT_MATCH
 import com.intellij.cce.evaluable.AIA_EXPECTED_FUNCTION_CALLS
@@ -288,8 +289,19 @@ object Execution {
       ignoreMissingData = true,
     )
   )
+
+  val ARTIFACTS_FILES: EvalDataDescription<List<ArtifactFile>, ArtifactFile> = EvalDataDescription(
+    name = "Artifact files",
+    description = "Files that should be copied to report artifacts. Source is the path to copy, target is the relative report path",
+    placement = DataPlacement.ArtifactFiles(AIA_EVAL_ARTIFACT_FILES),
+  )
 }
 
+data class ArtifactFile(
+  val target: String,
+  val source: String,
+)
+
 object Analysis {
   val HAS_SYNTAX_ERRORS: TrivialEvalData<Boolean> = EvalDataDescription(
     name = "Has syntax errors",

plugins/evaluation-plugin/core/src/com/intellij/cce/report/ArtifactsToReport.kt

@@ -2,9 +2,13 @@
 package com.intellij.cce.report
 
 import com.intellij.cce.core.Lookup
+import com.intellij.cce.core.Session
 import com.intellij.cce.evaluable.AIA_EVAL_ARTIFACT
 import com.intellij.cce.evaluable.AIA_EVAL_ARTIFACT_NAME
 import com.intellij.cce.evaluable.AIA_NAME
+import com.intellij.cce.evaluation.data.ArtifactFile
+import com.intellij.cce.evaluation.data.DataProps
+import com.intellij.cce.evaluation.data.Execution
 import com.intellij.cce.metric.MetricInfo
 import com.intellij.cce.workspace.info.FileErrorInfo
 import com.intellij.cce.workspace.info.FileEvaluationInfo
@@ -12,6 +16,7 @@ import com.intellij.openapi.diagnostic.Logger
 import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.Paths
+import java.nio.file.StandardCopyOption
 
 class ArtifactsToReport(
   outputDir: String,
@@ -21,7 +26,7 @@ class ArtifactsToReport(
 
   override val type: String = "artifacts"
 
-  private val dir: Path = Paths.get(outputDir, comparisonFilterName, type, filterName).also { Files.createDirectories(it) }
+  private val dir: Path = Paths.get(outputDir, comparisonFilterName, type, filterName).normalize().also { Files.createDirectories(it) }
 
   override fun generateFileReport(sessions: List<FileEvaluationInfo>) {
     if (sessions.isEmpty()) return
@@ -30,6 +35,7 @@ class ArtifactsToReport(
         for (session in fileEvaluation.sessionsInfo.sessions) {
           for (lookup in session.lookups) {
             saveArtifact(lookup)
+            saveArtifactFiles(session, lookup)
           }
         }
       }
@@ -54,6 +60,39 @@ class ArtifactsToReport(
     Files.writeString(artifactPath, artifact)
   }
 
+  private fun saveArtifactFiles(session: Session, lookup: Lookup) {
+    val props = DataProps(null, null, session, lookup)
+    for (artifactFile in Execution.ARTIFACTS_FILES.placement.restore(props)) {
+      copyArtifactFile(artifactFile)
+    }
+  }
+
+  private fun copyArtifactFile(artifactFile: ArtifactFile) {
+    val targetPath = resolveTarget(artifactFile.target) ?: return
+    targetPath.parent?.let { Files.createDirectories(it) }
+    Files.copy(Paths.get(artifactFile.source), targetPath, StandardCopyOption.REPLACE_EXISTING)
+  }
+
+  private fun resolveTarget(target: String): Path? {
+    if (target.isBlank()) {
+      LOG.warn("Failed to save artifact file: target path is blank")
+      return null
+    }
+
+    val relativeTarget = Paths.get(target)
+    if (relativeTarget.isAbsolute) {
+      LOG.warn("Failed to save artifact file: target path should be relative: $target")
+      return null
+    }
+
+    val targetPath = dir.resolve(relativeTarget).normalize()
+    if (!targetPath.startsWith(dir)) {
+      LOG.warn("Failed to save artifact file: target path escapes report directory: $target")
+      return null
+    }
+    return targetPath
+  }
+
   override fun generateErrorReports(errors: List<FileErrorInfo>): Unit = Unit
 
   override fun generateGlobalReport(globalMetrics: List<MetricInfo>): Path {

plugins/evaluation-plugin/core/src/com/intellij/cce/report/CardReportGenerator.kt

@@ -430,6 +430,10 @@ private data class PropertyValue(
           """sessions["${sessionId}"]["_lookups"][${lookupIndex}]["additionalInfo"]["${placement.propertyKey}"][${placementIndex}].originalText""",
           """sessions["${sessionId}"]["_lookups"][${lookupIndex}]["additionalInfo"]["${placement.propertyKey}"][${placementIndex}].updatedText"""
         )
+        is DataPlacement.ArtifactFiles -> listOf(
+          """sessions["${sessionId}"]["_lookups"][${lookupIndex}]["additionalInfo"]["${placement.propertyKey}"][${placementIndex}].target""",
+          """sessions["${sessionId}"]["_lookups"][${lookupIndex}]["additionalInfo"]["${placement.propertyKey}"][${placementIndex}].source"""
+        )
         is DataPlacement.AdditionalCodeCommentRanges -> listOf(
           """sessions["${sessionId}"]["_lookups"][${lookupIndex}]["additionalInfo"]["${placement.propertyKey}"][${placementIndex}].start""",
           """sessions["${sessionId}"]["_lookups"][${lookupIndex}]["additionalInfo"]["${placement.propertyKey}"][${placementIndex}].end""",
@@ -487,4 +491,4 @@ data class CardLayout(
       builder.create()
     }
   }
-}
+}

plugins/evaluation-plugin/test/com/intellij/cce/report/ArtifactsToReportTest.kt

@@ -0,0 +1,69 @@
+// Copyright 2000-2026 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package com.intellij.cce.report
+
+import com.intellij.cce.core.Lookup
+import com.intellij.cce.core.Session
+import com.intellij.cce.core.TokenProperties
+import com.intellij.cce.evaluation.data.ArtifactFile
+import com.intellij.cce.evaluation.data.Execution
+import com.intellij.cce.workspace.info.FileEvaluationInfo
+import com.intellij.cce.workspace.info.FileSessionsInfo
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Assertions.assertFalse
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.io.TempDir
+import java.nio.file.Files
+import java.nio.file.Path
+
+class ArtifactsToReportTest {
+  @Test
+  fun `copies artifact files to report directory`(@TempDir tempDir: Path) {
+    val source = tempDir.resolve("source.txt")
+    Files.writeString(source, "artifact file content")
+
+    val lookup = Execution.ARTIFACTS_FILES.placement.dump(
+      lookup(),
+      listOf(ArtifactFile(target = "nested/result.txt", source = source.toString()))
+    )
+
+    ArtifactsToReport(tempDir.resolve("report").toString(), "filter", "comparison")
+      .generateFileReport(listOf(fileEvaluationInfo(lookup)))
+
+    val copied = tempDir.resolve("report/comparison/artifacts/filter/nested/result.txt")
+    assertEquals("artifact file content", Files.readString(copied))
+  }
+
+  @Test
+  fun `does not copy artifact files outside report directory`(@TempDir tempDir: Path) {
+    val source = tempDir.resolve("source.txt")
+    Files.writeString(source, "artifact file content")
+
+    val lookup = Execution.ARTIFACTS_FILES.placement.dump(
+      lookup(),
+      listOf(ArtifactFile(target = "../outside.txt", source = source.toString()))
+    )
+
+    ArtifactsToReport(tempDir.resolve("report").toString(), "filter", "comparison")
+      .generateFileReport(listOf(fileEvaluationInfo(lookup)))
+
+    assertFalse(Files.exists(tempDir.resolve("report/comparison/artifacts/outside.txt")))
+  }
+
+  private fun fileEvaluationInfo(lookup: Lookup): FileEvaluationInfo {
+    val session = Session(0, "", 0, TokenProperties.UNKNOWN)
+    session.addLookup(lookup)
+    return FileEvaluationInfo(
+      sessionsInfo = FileSessionsInfo("project", "file.kt", "", listOf(session)),
+      metrics = emptyList(),
+      evaluationType = "evaluation"
+    )
+  }
+
+  private fun lookup(): Lookup = Lookup(
+    prefix = "",
+    offset = 0,
+    suggestions = emptyList(),
+    latency = 0,
+    isNew = true
+  )
+}