diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluable/Constants.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluable/Constants.kt index d1409fb7411b..712988ccb086 100644 --- a/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluable/Constants.kt +++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluable/Constants.kt @@ -16,8 +16,10 @@ const val METHOD_BODY_PROPERTY = "method_body" const val REFERENCE_PROPERTY = "reference" const val CATEGORY_PROPERTY = "category" const val PATH_PROPERTY = "path" -const val START_LINE_PROPERTY = "start_line" -const val END_LINE_PROPERTY = "end_line" +const val START_LINES_PROPERTY = "start_lines" +const val END_LINES_PROPERTY = "end_lines" +const val REFERENCE_START_LINE_PROPERTY = "reference_start_line" +const val REFERENCE_END_LINE_PROPERTY = "reference_end_line" const val FILE_PATTERN_PROPERTY_PREFIX: String = "file_property_regex" const val FILE_UNCHANGED_LINES_PREFIX: String = "file_unchanged_lines" diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluation/data/DataPlacement.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluation/data/DataPlacement.kt index 3809632fed0e..8235450e1305 100644 --- a/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluation/data/DataPlacement.kt +++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluation/data/DataPlacement.kt @@ -61,6 +61,15 @@ sealed interface DataPlacement { override fun restore(props: DataProps): List = listOfNotNull(props.lookup.additionalInfo[propertyKey] as? Double).map { it.toInt() } } + data class AdditionalListInt(val propertyKey: String) : DataPlacement, List> { + override val serialName: String = "additional_list_int" + override fun dump(lookup: Lookup, t: List): Lookup = + // because of json serializes int to double we cast it by ourselves for predictability + lookup.copy(additionalInfo = lookup.additionalInfo + mapOf(propertyKey to t.map { it.toDouble() })) + + override fun restore(props: DataProps): List> = listOfNotNull(props.lookup.additionalInfo[propertyKey] as? List).map { it.map { v -> v.toInt() } } + } + data class AdditionalJsonSerializedStrings(val propertyKey: String) : DataPlacement, List> { override val serialName: String = "additional_concatenated_snippets" override fun dump(lookup: Lookup, t: List): Lookup { @@ -141,6 +150,7 @@ sealed interface DataPlacement { "additional_boolean" -> context?.deserialize(json, AdditionalBoolean::class.java) "additional_double" -> context?.deserialize(json, AdditionalDouble::class.java) "additional_int" -> context?.deserialize(json, AdditionalInt::class.java) + "additional_list_int" -> context?.deserialize(json, AdditionalListInt::class.java) "additional_concatenated_lines" -> context?.deserialize(json, AdditionalConcatenatedLines::class.java) "additional_concatenated_snippets" -> context?.deserialize(json, AdditionalJsonSerializedStrings::class.java) "latency" -> Latency diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluation/data/DataRenderer.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluation/data/DataRenderer.kt index cada7171252f..55c6565d906d 100644 --- a/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluation/data/DataRenderer.kt +++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluation/data/DataRenderer.kt @@ -27,6 +27,10 @@ sealed interface DataRenderer { override val serialName: String = "inline_int" } + data object ListInt : DataRenderer> { + override val serialName: String = "list_int" + } + data object ClickableLink : DataRenderer { override val serialName: String = "clickable_link" } @@ -63,6 +67,7 @@ sealed interface DataRenderer { "inline_long" -> InlineLong "inline_double" -> InlineDouble "inline_int" -> InlineInt + "list_int" -> ListInt "clickable_link" -> ClickableLink "text" -> context?.deserialize(json, Text::class.java) "lines" -> Lines diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluation/data/EvalDataConstants.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluation/data/EvalDataConstants.kt index 15bbfa317083..a9bce9cc563b 100644 --- a/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluation/data/EvalDataConstants.kt +++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluation/data/EvalDataConstants.kt @@ -119,6 +119,10 @@ object Execution { name = "Reference", description = null, DataPlacement.AdditionalText(REFERENCE_PROPERTY), + presentation = EvalDataPresentation( + PresentationCategory.EXECUTION, + DataRenderer.Text(wrapping = true), + ) ) val NAME: TrivialEvalData = EvalDataDescription( @@ -154,6 +158,46 @@ object Execution { ignoreMissingData = true, ) ) + + val REFERENCE_START_LINE: TrivialEvalData = EvalDataDescription( + name = "Reference start line", + description = "Ground truth start line of the range", + DataPlacement.AdditionalInt(REFERENCE_START_LINE_PROPERTY), + presentation = EvalDataPresentation( + PresentationCategory.EXECUTION, + DataRenderer.InlineInt + ) + ) + + val REFERENCE_END_LINE: TrivialEvalData = EvalDataDescription( + name = "Reference end line", + description = "Ground truth end line of the range", + DataPlacement.AdditionalInt(REFERENCE_END_LINE_PROPERTY), + presentation = EvalDataPresentation( + PresentationCategory.EXECUTION, + DataRenderer.InlineInt + ) + ) + + val START_LINES: TrivialEvalData> = EvalDataDescription( + name = "Predicted start lines", + description = "Predicted start lines of the ranges", + DataPlacement.AdditionalListInt(START_LINES_PROPERTY), + presentation = EvalDataPresentation( + PresentationCategory.EXECUTION, + DataRenderer.ListInt + ) + ) + + val END_LINES: TrivialEvalData> = EvalDataDescription( + name = "Predicted end lines", + description = "Predicted end lines of the ranges", + DataPlacement.AdditionalListInt(END_LINES_PROPERTY), + presentation = EvalDataPresentation( + PresentationCategory.EXECUTION, + DataRenderer.ListInt + ) + ) } object Analysis { diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/RangeMetrics.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/RangeMetrics.kt new file mode 100644 index 000000000000..6cb6d9a8330c --- /dev/null +++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/RangeMetrics.kt @@ -0,0 +1,97 @@ +package com.intellij.cce.metric + +import com.intellij.cce.core.Session +import com.intellij.cce.evaluable.REFERENCE_START_LINE_PROPERTY +import com.intellij.cce.evaluable.REFERENCE_END_LINE_PROPERTY +import com.intellij.cce.evaluable.START_LINES_PROPERTY +import com.intellij.cce.evaluable.END_LINES_PROPERTY +import com.intellij.cce.metric.util.Sample +import com.intellij.cce.metric.util.computeIOU +import kotlin.collections.forEach + + +abstract class RangeMetricBase : ConfidenceIntervalMetric() { + override val showByDefault: Boolean = true + override val valueType = MetricValueType.DOUBLE + override val value: Double + get() = compute(sample) + + override fun compute(sample: List): Double = sample.average() + + override fun evaluate(sessions: List): Double { + // this metric is used with an assumption that the session contains only one lookup and all data is stored there + val lookups = sessions.flatMap { session -> session.lookups } + val fileSample = Sample() + lookups.forEach { lookup -> + val referenceRange = Pair((lookup.additionalInfo[REFERENCE_START_LINE_PROPERTY] as? Double ?: -1).toInt(), + (lookup.additionalInfo[REFERENCE_END_LINE_PROPERTY] as? Double ?: -1).toInt()) + val predictedRanges = (lookup.additionalInfo[START_LINES_PROPERTY] as? List ?: emptyList()) + .zip(lookup.additionalInfo[END_LINES_PROPERTY] as? List ?: emptyList()).map { + Pair(it.first.toInt(), it.second.toInt()) + } + + calculateMetric(predictedRanges, referenceRange, fileSample) + } + return fileSample.mean() + } + + abstract fun calculateMetric(predictedRanges: List>, referenceRange: Pair, fileSample: Sample) +} + +interface Scorer { + fun computeScore(predictedRange: Pair, referenceRange: Pair): Double +} + +interface IOUScorer : Scorer { + override fun computeScore(predictedRange: Pair, referenceRange: Pair): Double { + return computeIOU(predictedRange, referenceRange) + } +} + +interface PerfectOverlapScorer : Scorer { + override fun computeScore(predictedRange: Pair, referenceRange: Pair): Double { + val iou = computeIOU(predictedRange, referenceRange) + return if (iou == 1.0) 1.0 else 0.0 + } +} + +abstract class PrecisionRangeMetricBase : RangeMetricBase(), Scorer { + override fun calculateMetric(predictedRanges: List>, referenceRange: Pair, fileSample: Sample) { + if (predictedRanges.isEmpty()) return + val bestOverlap = predictedRanges.maxOfOrNull { computeScore(it, referenceRange) } ?: 0.0 + (listOf(bestOverlap) + List(predictedRanges.size - 1) { 0.0 }).forEach { + fileSample.add(it) + coreSample.add(it) + } + } +} + +abstract class RecallRangeMetricBase : RangeMetricBase(), Scorer { + override fun calculateMetric(predictedRanges: List>, referenceRange: Pair, fileSample: Sample) { + if (referenceRange.first == -1 || referenceRange.second == -1) return + val bestOverlap = predictedRanges.maxOfOrNull { computeScore(it, referenceRange) } ?: 0.0 + fileSample.add(bestOverlap) + coreSample.add(bestOverlap) + } +} + + +class PerfectOverlapPrecisionMetric : PrecisionRangeMetricBase(), PerfectOverlapScorer { + override val name = "Perfect Overlap Precision" + override val description: String = "Ratio of predicted ranges that overlap with reference ranges" +} + +class PerfectOverlapRecallMetric : RecallRangeMetricBase(), PerfectOverlapScorer { + override val name = "Perfect Overlap Recall" + override val description: String = "Ratio of reference ranges that overlap with predicted ranges" +} + +class IOUPrecisionMetric : PrecisionRangeMetricBase(), IOUScorer { + override val name = "IoU Precision" + override val description: String = "Sum of IoU between predicted & reference range divided by total number of predicted ranges" +} + +class IOURecallMetric : RecallRangeMetricBase(), IOUScorer { + override val name = "IoU Recall" + override val description: String = "Sum of IoU between predicted & reference range divided by total number of reference ranges" +} \ No newline at end of file diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/SimilarityMetrics.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/SimilarityMetrics.kt index 011f81a1c603..ac017423a626 100644 --- a/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/SimilarityMetrics.kt +++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/SimilarityMetrics.kt @@ -207,6 +207,32 @@ class LLMJudgeScore(showByDefault: Boolean = true, private val llmJudge: LLMJudg } class SemanticSimilarityScore(showByDefault: Boolean = true, val cloudSemanticSimilarityCalculator: CloudSemanticSimilarityCalculator) : SimilarityMetric(showByDefault) { + override val name = "Semantic Similarity Score" + override val description: String = "Calculates the Semantic Similarity score between the reference and AIA response via cosine similarity of text embeddings." + + private val project: Project + get() = ProjectManager.getInstance().defaultProject + + override fun computeSimilarity(lookup: Lookup, expectedText: String): Double? { + val aiaResponse = lookup.additionalInfo[AIA_RESPONSE] as? String ?: return null + val reference = lookup.additionalInfo[REFERENCE_PROPERTY] as? String ?: return null + val score = runBlockingCancellable { + async { + cloudSemanticSimilarityCalculator.calculateCosineSimilarity( + project, + aiaResponse, + reference + ) + }.await() + } + return score + } + + override fun computeExpected(lookup: Lookup, expectedText: String): Double = 1.0 +} + + +class ProposalSemanticSimilarityScore(showByDefault: Boolean = true, val cloudSemanticSimilarityCalculator: CloudSemanticSimilarityCalculator) : SimilarityMetric(showByDefault) { override val name: String get() = NAME private val project: Project diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/util/IOU.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/util/IOU.kt new file mode 100644 index 000000000000..fe52bec1884a --- /dev/null +++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/util/IOU.kt @@ -0,0 +1,35 @@ +package com.intellij.cce.metric.util + +/** + * Computes the Intersection over Union (IOU) between two ranges. + * + * @param range1 Pair of (start, end) positions for the first range + * @param range2 Pair of (start, end) positions for the second range + * @return IOU value between 0.0 (no overlap) and 1.0 (perfect overlap) + */ +fun computeIOU(range1: Pair, range2: Pair): Double { + val (start1, end1) = range1 + val (start2, end2) = range2 + + require(start1 <= end1) { "First range start must be less than or equal to end" } + require(start2 <= end2) { "Second range start must be less than or equal to end" } + + // Calculate intersection + val intersectionStart = maxOf(start1, start2) + val intersectionEnd = minOf(end1, end2) + + // If there's no intersection, return 0 + if (intersectionEnd < intersectionStart) { + return 0.0 + } + + val intersectionArea = intersectionEnd - intersectionStart + + // Calculate union + val range1Size = end1 - start1 + val range2Size = end2 - start2 + val unionArea = range1Size + range2Size - intersectionArea + + // Return IOU + return if (unionArea > 0) intersectionArea.toDouble() / unionArea else 0.0 +} \ No newline at end of file diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/report/CardReportGenerator.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/report/CardReportGenerator.kt index 09ee19ae588c..20cfd6b889e7 100644 --- a/plugins/evaluation-plugin/core/src/com/intellij/cce/report/CardReportGenerator.kt +++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/report/CardReportGenerator.kt @@ -353,6 +353,7 @@ private data class PropertyValue( is DataRenderer.InlineLong -> PropertyValue(null, "${property.value}") is DataRenderer.InlineDouble -> PropertyValue(null, "${property.value}") is DataRenderer.InlineInt -> PropertyValue(null, "${property.value}") + is DataRenderer.ListInt -> PropertyValue(null, "${property.value}") is DataRenderer.ClickableLink -> PropertyValue(null, null, "${property.value}") is DataRenderer.Text -> PropertyValue("""openText($element, ${stringValues[0]}, ${description}, ${property.renderer.wrapping});""", null) is DataRenderer.Lines -> PropertyValue("""openText($element, ${stringValues[0]}, ${description});""", null) @@ -378,6 +379,9 @@ private data class PropertyValue( is DataPlacement.AdditionalInt -> listOf( """sessions["${sessionId}"]["_lookups"][${lookupIndex}]["additionalInfo"]["${placement.propertyKey}"].toString()""" ) + is DataPlacement.AdditionalListInt -> listOf( + """sessions["${sessionId}"]["_lookups"][${lookupIndex}]["additionalInfo"]["${placement.propertyKey}"].toString()""" + ) is DataPlacement.Latency -> listOf( """sessions["${sessionId}"]["_lookups"][${lookupIndex}]["latency"].toString()""" ) @@ -407,6 +411,7 @@ private data class PropertyValue( DataRenderer.InlineLong -> listOf("\"${value}\"") DataRenderer.InlineDouble -> listOf("\"${value}\"") DataRenderer.InlineInt -> listOf("\"${value}\"") + DataRenderer.ListInt -> listOf("\"${value}\"") DataRenderer.ClickableLink -> listOf() is DataRenderer.Text -> listOf(embedString(value as String)) DataRenderer.Lines -> listOf(embedString((value as List<*>).joinToString("\n") { "• $it" })) diff --git a/plugins/evaluation-plugin/test/com/intellij/cce/metric/SemanticSimilarityMetricTest.kt b/plugins/evaluation-plugin/test/com/intellij/cce/metric/ProposalSemanticSimilarityMetricTest.kt similarity index 93% rename from plugins/evaluation-plugin/test/com/intellij/cce/metric/SemanticSimilarityMetricTest.kt rename to plugins/evaluation-plugin/test/com/intellij/cce/metric/ProposalSemanticSimilarityMetricTest.kt index 3d096e349dba..3f186ecc097d 100644 --- a/plugins/evaluation-plugin/test/com/intellij/cce/metric/SemanticSimilarityMetricTest.kt +++ b/plugins/evaluation-plugin/test/com/intellij/cce/metric/ProposalSemanticSimilarityMetricTest.kt @@ -11,7 +11,7 @@ import org.mockito.kotlin.eq import org.mockito.kotlin.spy import org.mockito.kotlin.whenever -class SemanticSimilarityMetricTest : BasePlatformTestCase() { +class ProposalSemanticSimilarityMetricTest : BasePlatformTestCase() { fun `test lookup empty`() { val additionalInfo = emptyMap>() @@ -107,9 +107,9 @@ class SemanticSimilarityMetricTest : BasePlatformTestCase() { )).thenReturn(score) } - val semanticSimilarityScoreCalculator = spy(SemanticSimilarityScore(true, mockedEmbeddingsProvider)) + val proposalSemanticSimilarityScoreCalculator = spy(ProposalSemanticSimilarityScore(true, mockedEmbeddingsProvider)) - semanticSimilarityScoreCalculator.computeSimilarity(createLookup(additionalInfo), "") + proposalSemanticSimilarityScoreCalculator.computeSimilarity(createLookup(additionalInfo), "") } private fun createLookup(additionalInfo: Map): Lookup =