[daemon] LLM-17397 Add evaluation metrics

GitOrigin-RevId: 3cf08d9d0ad756bae1263d13c5d1b713a0cc67ca

plugins/evaluation-plugin/core/src/com/intellij/cce/evaluable/Constants.kt

@@ -16,8 +16,10 @@ const val METHOD_BODY_PROPERTY = "method_body"
 const val REFERENCE_PROPERTY = "reference"
 const val CATEGORY_PROPERTY = "category"
 const val PATH_PROPERTY = "path"
-const val START_LINE_PROPERTY = "start_line"
-const val END_LINE_PROPERTY = "end_line"
+const val START_LINES_PROPERTY = "start_lines"
+const val END_LINES_PROPERTY = "end_lines"
+const val REFERENCE_START_LINE_PROPERTY = "reference_start_line"
+const val REFERENCE_END_LINE_PROPERTY = "reference_end_line"
 
 const val FILE_PATTERN_PROPERTY_PREFIX: String = "file_property_regex"
 const val FILE_UNCHANGED_LINES_PREFIX: String = "file_unchanged_lines"

plugins/evaluation-plugin/core/src/com/intellij/cce/evaluation/data/DataPlacement.kt

@@ -61,6 +61,15 @@ sealed interface DataPlacement<In, Out> {
     override fun restore(props: DataProps): List<Int> = listOfNotNull(props.lookup.additionalInfo[propertyKey] as? Double).map { it.toInt() }
   }
 
+  data class AdditionalListInt(val propertyKey: String) : DataPlacement<List<Int>, List<Int>> {
+    override val serialName: String = "additional_list_int"
+    override fun dump(lookup: Lookup, t: List<Int>): Lookup =
+      // because of json serializes int to double we cast it by ourselves for predictability
+      lookup.copy(additionalInfo = lookup.additionalInfo + mapOf(propertyKey to t.map { it.toDouble() }))
+
+    override fun restore(props: DataProps): List<List<Int>> = listOfNotNull(props.lookup.additionalInfo[propertyKey] as? List<Double>).map { it.map { v -> v.toInt() } }
+  }
+
   data class AdditionalJsonSerializedStrings(val propertyKey: String) : DataPlacement<List<String>, List<String>> {
     override val serialName: String = "additional_concatenated_snippets"
     override fun dump(lookup: Lookup, t: List<String>): Lookup {
@@ -141,6 +150,7 @@ sealed interface DataPlacement<In, Out> {
         "additional_boolean" -> context?.deserialize(json, AdditionalBoolean::class.java)
         "additional_double" -> context?.deserialize(json, AdditionalDouble::class.java)
         "additional_int" -> context?.deserialize(json, AdditionalInt::class.java)
+        "additional_list_int" -> context?.deserialize(json, AdditionalListInt::class.java)
         "additional_concatenated_lines" -> context?.deserialize(json, AdditionalConcatenatedLines::class.java)
         "additional_concatenated_snippets" -> context?.deserialize(json, AdditionalJsonSerializedStrings::class.java)
         "latency" -> Latency

plugins/evaluation-plugin/core/src/com/intellij/cce/evaluation/data/DataRenderer.kt

@@ -27,6 +27,10 @@ sealed interface DataRenderer<in T> {
     override val serialName: String = "inline_int"
   }
 
+  data object ListInt : DataRenderer<List<Int>> {
+    override val serialName: String = "list_int"
+  }
+
   data object ClickableLink : DataRenderer<String> {
     override val serialName: String = "clickable_link"
   }
@@ -63,6 +67,7 @@ sealed interface DataRenderer<in T> {
         "inline_long" -> InlineLong
         "inline_double" -> InlineDouble
         "inline_int" -> InlineInt
+        "list_int" -> ListInt
         "clickable_link" -> ClickableLink
         "text" -> context?.deserialize(json, Text::class.java)
         "lines" -> Lines

plugins/evaluation-plugin/core/src/com/intellij/cce/evaluation/data/EvalDataConstants.kt

@@ -119,6 +119,10 @@ object Execution {
     name = "Reference",
     description = null,
     DataPlacement.AdditionalText(REFERENCE_PROPERTY),
+    presentation = EvalDataPresentation(
+      PresentationCategory.EXECUTION,
+      DataRenderer.Text(wrapping = true),
+    )
   )
 
   val NAME: TrivialEvalData<String> = EvalDataDescription(
@@ -154,6 +158,46 @@ object Execution {
       ignoreMissingData = true,
     )
   )
+
+  val REFERENCE_START_LINE: TrivialEvalData<Int> = EvalDataDescription(
+    name = "Reference start line",
+    description = "Ground truth start line of the range",
+    DataPlacement.AdditionalInt(REFERENCE_START_LINE_PROPERTY),
+    presentation = EvalDataPresentation(
+      PresentationCategory.EXECUTION,
+      DataRenderer.InlineInt
+    )
+  )
+
+  val REFERENCE_END_LINE: TrivialEvalData<Int> = EvalDataDescription(
+    name = "Reference end line",
+    description = "Ground truth end line of the range",
+    DataPlacement.AdditionalInt(REFERENCE_END_LINE_PROPERTY),
+    presentation = EvalDataPresentation(
+      PresentationCategory.EXECUTION,
+      DataRenderer.InlineInt
+    )
+  )
+
+  val START_LINES: TrivialEvalData<List<Int>> = EvalDataDescription(
+    name = "Predicted start lines",
+    description = "Predicted start lines of the ranges",
+    DataPlacement.AdditionalListInt(START_LINES_PROPERTY),
+    presentation = EvalDataPresentation(
+      PresentationCategory.EXECUTION,
+      DataRenderer.ListInt
+    )
+  )
+
+  val END_LINES: TrivialEvalData<List<Int>> = EvalDataDescription(
+    name = "Predicted end lines",
+    description = "Predicted end lines of the ranges",
+    DataPlacement.AdditionalListInt(END_LINES_PROPERTY),
+    presentation = EvalDataPresentation(
+      PresentationCategory.EXECUTION,
+      DataRenderer.ListInt
+    )
+  )
 }
 
 object Analysis {

plugins/evaluation-plugin/core/src/com/intellij/cce/metric/RangeMetrics.kt

@@ -0,0 +1,97 @@
+package com.intellij.cce.metric
+
+import com.intellij.cce.core.Session
+import com.intellij.cce.evaluable.REFERENCE_START_LINE_PROPERTY
+import com.intellij.cce.evaluable.REFERENCE_END_LINE_PROPERTY
+import com.intellij.cce.evaluable.START_LINES_PROPERTY
+import com.intellij.cce.evaluable.END_LINES_PROPERTY
+import com.intellij.cce.metric.util.Sample
+import com.intellij.cce.metric.util.computeIOU
+import kotlin.collections.forEach
+
+
+abstract class RangeMetricBase : ConfidenceIntervalMetric<Double>() {
+  override val showByDefault: Boolean = true
+  override val valueType = MetricValueType.DOUBLE
+  override val value: Double
+    get() = compute(sample)
+
+  override fun compute(sample: List<Double>): Double = sample.average()
+
+  override fun evaluate(sessions: List<Session>): Double {
+    // this metric is used with an assumption that the session contains only one lookup and all data is stored there
+    val lookups = sessions.flatMap { session -> session.lookups }
+    val fileSample = Sample()
+    lookups.forEach { lookup ->
+      val referenceRange = Pair((lookup.additionalInfo[REFERENCE_START_LINE_PROPERTY] as? Double ?: -1).toInt(),
+                                (lookup.additionalInfo[REFERENCE_END_LINE_PROPERTY] as? Double ?: -1).toInt())
+      val predictedRanges = (lookup.additionalInfo[START_LINES_PROPERTY] as? List<Double> ?: emptyList())
+        .zip(lookup.additionalInfo[END_LINES_PROPERTY] as? List<Double> ?: emptyList()).map {
+          Pair(it.first.toInt(), it.second.toInt())
+        }
+
+      calculateMetric(predictedRanges, referenceRange, fileSample)
+    }
+    return fileSample.mean()
+  }
+
+  abstract fun calculateMetric(predictedRanges: List<Pair<Int, Int>>, referenceRange: Pair<Int, Int>, fileSample: Sample)
+}
+
+interface Scorer {
+  fun computeScore(predictedRange: Pair<Int, Int>, referenceRange: Pair<Int, Int>): Double
+}
+
+interface IOUScorer : Scorer {
+  override fun computeScore(predictedRange: Pair<Int, Int>, referenceRange: Pair<Int, Int>): Double {
+    return computeIOU(predictedRange, referenceRange)
+  }
+}
+
+interface PerfectOverlapScorer : Scorer {
+  override fun computeScore(predictedRange: Pair<Int, Int>, referenceRange: Pair<Int, Int>): Double {
+    val iou = computeIOU(predictedRange, referenceRange)
+    return if (iou == 1.0) 1.0 else 0.0
+  }
+}
+
+abstract class PrecisionRangeMetricBase : RangeMetricBase(), Scorer {
+  override fun calculateMetric(predictedRanges: List<Pair<Int, Int>>, referenceRange: Pair<Int, Int>, fileSample: Sample) {
+    if (predictedRanges.isEmpty()) return
+    val bestOverlap = predictedRanges.maxOfOrNull { computeScore(it, referenceRange) } ?: 0.0
+    (listOf(bestOverlap) + List(predictedRanges.size - 1) { 0.0 }).forEach {
+      fileSample.add(it)
+      coreSample.add(it)
+    }
+  }
+}
+
+abstract class RecallRangeMetricBase : RangeMetricBase(), Scorer {
+  override fun calculateMetric(predictedRanges: List<Pair<Int, Int>>, referenceRange: Pair<Int, Int>, fileSample: Sample) {
+    if (referenceRange.first == -1 || referenceRange.second == -1) return
+    val bestOverlap = predictedRanges.maxOfOrNull { computeScore(it, referenceRange) } ?: 0.0
+    fileSample.add(bestOverlap)
+    coreSample.add(bestOverlap)
+  }
+}
+
+
+class PerfectOverlapPrecisionMetric : PrecisionRangeMetricBase(), PerfectOverlapScorer {
+  override val name = "Perfect Overlap Precision"
+  override val description: String = "Ratio of predicted ranges that overlap with reference ranges"
+}
+
+class PerfectOverlapRecallMetric : RecallRangeMetricBase(), PerfectOverlapScorer {
+  override val name = "Perfect Overlap Recall"
+  override val description: String = "Ratio of reference ranges that overlap with predicted ranges"
+}
+
+class IOUPrecisionMetric : PrecisionRangeMetricBase(), IOUScorer {
+  override val name = "IoU Precision"
+  override val description: String = "Sum of IoU between predicted & reference range divided by total number of predicted ranges"
+}
+
+class IOURecallMetric : RecallRangeMetricBase(), IOUScorer {
+  override val name = "IoU Recall"
+  override val description: String = "Sum of IoU between predicted & reference range divided by total number of reference ranges"
+}

plugins/evaluation-plugin/core/src/com/intellij/cce/metric/SimilarityMetrics.kt

@@ -207,6 +207,32 @@ class LLMJudgeScore(showByDefault: Boolean = true, private val llmJudge: LLMJudg
 }
 
 class SemanticSimilarityScore(showByDefault: Boolean = true, val cloudSemanticSimilarityCalculator: CloudSemanticSimilarityCalculator) : SimilarityMetric(showByDefault) {
+  override val name = "Semantic Similarity Score"
+  override val description: String = "Calculates the Semantic Similarity score between the reference and AIA response via cosine similarity of text embeddings."
+
+  private val project: Project
+    get() = ProjectManager.getInstance().defaultProject
+
+  override fun computeSimilarity(lookup: Lookup, expectedText: String): Double? {
+    val aiaResponse = lookup.additionalInfo[AIA_RESPONSE] as? String ?: return null
+    val reference = lookup.additionalInfo[REFERENCE_PROPERTY] as? String ?: return null
+    val score = runBlockingCancellable {
+      async {
+        cloudSemanticSimilarityCalculator.calculateCosineSimilarity(
+          project,
+          aiaResponse,
+          reference
+        )
+      }.await()
+    }
+    return score
+  }
+
+  override fun computeExpected(lookup: Lookup, expectedText: String): Double = 1.0
+}
+
+
+class ProposalSemanticSimilarityScore(showByDefault: Boolean = true, val cloudSemanticSimilarityCalculator: CloudSemanticSimilarityCalculator) : SimilarityMetric(showByDefault) {
   override val name: String
     get() = NAME
   private val project: Project

plugins/evaluation-plugin/core/src/com/intellij/cce/metric/util/IOU.kt

@@ -0,0 +1,35 @@
+package com.intellij.cce.metric.util
+
+/**
+ * Computes the Intersection over Union (IOU) between two ranges.
+ *
+ * @param range1 Pair of (start, end) positions for the first range
+ * @param range2 Pair of (start, end) positions for the second range
+ * @return IOU value between 0.0 (no overlap) and 1.0 (perfect overlap)
+ */
+fun computeIOU(range1: Pair<Int, Int>, range2: Pair<Int, Int>): Double {
+  val (start1, end1) = range1
+  val (start2, end2) = range2
+
+  require(start1 <= end1) { "First range start must be less than or equal to end" }
+  require(start2 <= end2) { "Second range start must be less than or equal to end" }
+
+  // Calculate intersection
+  val intersectionStart = maxOf(start1, start2)
+  val intersectionEnd = minOf(end1, end2)
+
+  // If there's no intersection, return 0
+  if (intersectionEnd < intersectionStart) {
+    return 0.0
+  }
+
+  val intersectionArea = intersectionEnd - intersectionStart
+
+  // Calculate union
+  val range1Size = end1 - start1
+  val range2Size = end2 - start2
+  val unionArea = range1Size + range2Size - intersectionArea
+
+  // Return IOU
+  return if (unionArea > 0) intersectionArea.toDouble() / unionArea else 0.0
+}

plugins/evaluation-plugin/core/src/com/intellij/cce/report/CardReportGenerator.kt

@@ -353,6 +353,7 @@ private data class PropertyValue(
         is DataRenderer.InlineLong -> PropertyValue(null, "${property.value}")
         is DataRenderer.InlineDouble -> PropertyValue(null, "${property.value}")
         is DataRenderer.InlineInt -> PropertyValue(null, "${property.value}")
+        is DataRenderer.ListInt -> PropertyValue(null, "${property.value}")
         is DataRenderer.ClickableLink -> PropertyValue(null, null, "${property.value}")
         is DataRenderer.Text -> PropertyValue("""openText($element, ${stringValues[0]}, ${description}, ${property.renderer.wrapping});""", null)
         is DataRenderer.Lines -> PropertyValue("""openText($element, ${stringValues[0]}, ${description});""", null)
@@ -378,6 +379,9 @@ private data class PropertyValue(
         is DataPlacement.AdditionalInt -> listOf(
           """sessions["${sessionId}"]["_lookups"][${lookupIndex}]["additionalInfo"]["${placement.propertyKey}"].toString()"""
         )
+        is DataPlacement.AdditionalListInt -> listOf(
+          """sessions["${sessionId}"]["_lookups"][${lookupIndex}]["additionalInfo"]["${placement.propertyKey}"].toString()"""
+        )
         is DataPlacement.Latency -> listOf(
           """sessions["${sessionId}"]["_lookups"][${lookupIndex}]["latency"].toString()"""
         )
@@ -407,6 +411,7 @@ private data class PropertyValue(
         DataRenderer.InlineLong -> listOf("\"${value}\"")
         DataRenderer.InlineDouble -> listOf("\"${value}\"")
         DataRenderer.InlineInt -> listOf("\"${value}\"")
+        DataRenderer.ListInt -> listOf("\"${value}\"")
         DataRenderer.ClickableLink -> listOf()
         is DataRenderer.Text -> listOf(embedString(value as String))
         DataRenderer.Lines -> listOf(embedString((value as List<*>).joinToString("\n") { "• $it" }))

plugins/evaluation-plugin/test/com/intellij/cce/metric/ProposalSemanticSimilarityMetricTest.kt

@@ -11,7 +11,7 @@ import org.mockito.kotlin.eq
 import org.mockito.kotlin.spy
 import org.mockito.kotlin.whenever
 
-class SemanticSimilarityMetricTest : BasePlatformTestCase() {
+class ProposalSemanticSimilarityMetricTest : BasePlatformTestCase() {
 
   fun `test lookup empty`() {
     val additionalInfo = emptyMap<String, List<*>>()
@@ -107,9 +107,9 @@ class SemanticSimilarityMetricTest : BasePlatformTestCase() {
       )).thenReturn(score)
     }
 
-    val semanticSimilarityScoreCalculator = spy(SemanticSimilarityScore(true, mockedEmbeddingsProvider))
+    val proposalSemanticSimilarityScoreCalculator = spy(ProposalSemanticSimilarityScore(true, mockedEmbeddingsProvider))
 
-    semanticSimilarityScoreCalculator.computeSimilarity(createLookup(additionalInfo), "")
+    proposalSemanticSimilarityScoreCalculator.computeSimilarity(createLookup(additionalInfo), "")
   }
 
   private fun createLookup(additionalInfo: Map<String, Any>): Lookup =