[evaluation-plugin] ML-632 Add interval estimates for metrics in CG

GitOrigin-RevId: 983b0dcca44663890631d9242cf5b2caf9da69d2

plugins/evaluation-plugin/core/src/com/intellij/cce/metric/CompletionGolfMetrics.kt

@@ -3,6 +3,7 @@ package com.intellij.cce.metric
 import com.intellij.cce.actions.selectedWithoutPrefix
 import com.intellij.cce.core.Lookup
 import com.intellij.cce.core.Session
+import com.intellij.cce.metric.util.Bootstrap
 import com.intellij.cce.metric.util.Sample
 import org.apache.commons.lang.StringUtils
 
@@ -144,12 +145,17 @@ internal class MovesCountNormalised : Metric {
 internal abstract class SimilarityMetric : Metric {
   private var totalMatched: Double = 0.0
   private var totalExpected: Double = 0.0
+  private var sample: MutableList<Pair<Double, Double>> = mutableListOf()
 
   override val valueType = MetricValueType.DOUBLE
   override val showByDefault: Boolean = false
   override val value: Double
     get() = totalMatched / totalExpected
 
+  override fun confidenceInterval(): Pair<Double, Double> = Bootstrap.computeInterval(sample) { values ->
+    values.sumOf { it.first } / values.sumOf { it.second }
+  }
+
   override fun evaluate(sessions: List<Session>, comparator: SuggestionsComparator): Double {
     var matched = 0.0
     var expected = 0.0
@@ -157,7 +163,9 @@ internal abstract class SimilarityMetric : Metric {
       for (lookup in session.lookups) {
         val expectedText = session.expectedText.substring(lookup.offset)
         expected += expectedText.length
-        matched += computeSimilarity(lookup, expectedText) ?: 0.0
+        val similarity = computeSimilarity(lookup, expectedText) ?: 0.0
+        matched += similarity
+        sample.add(Pair(similarity, expectedText.length.toDouble()))
       }
     }
     totalMatched += matched
@@ -209,11 +217,13 @@ internal class PerfectLine : CompletionGolfMetric<Int>() {
 }
 
 internal class Precision : Metric {
-  private val sample = Sample()
+  private val sample = mutableListOf<Double>()
   override val name = "Precision"
   override val valueType = MetricValueType.DOUBLE
   override val value: Double
-    get() = sample.mean()
+    get() = sample.average()
+
+  override fun confidenceInterval(): Pair<Double, Double> = Bootstrap.computeInterval(sample) { it.average() }
 
   override fun evaluate(sessions: List<Session>, comparator: SuggestionsComparator): Double {
     val fileSample = Sample()
@@ -234,11 +244,13 @@ internal class Precision : Metric {
 }
 
 internal open class RecallAt(private val n: Int) : Metric {
-  private val sample = Sample()
+  private val sample = mutableListOf<Double>()
   override val name = "RecallAt$n"
   override val valueType = MetricValueType.DOUBLE
   override val value: Double
-    get() = sample.mean()
+    get() = sample.average()
+
+  override fun confidenceInterval(): Pair<Double, Double> = Bootstrap.computeInterval(sample) { it.average() }
 
   override fun evaluate(sessions: List<Session>, comparator: SuggestionsComparator): Double {
     val fileSample = Sample()

plugins/evaluation-plugin/core/src/com/intellij/cce/metric/LatencyMetrics.kt

@@ -2,17 +2,19 @@ package com.intellij.cce.metric
 
 import com.intellij.cce.core.Lookup
 import com.intellij.cce.core.Session
-import com.intellij.cce.metric.util.Sample
+import com.intellij.cce.metric.util.Bootstrap
 
 abstract class LatencyMetric(override val name: String) : Metric {
-  private val sample = Sample()
+  private val sample = mutableListOf<Double>()
   override val value: Double
     get() = compute(sample)
 
+  override fun confidenceInterval(): Pair<Double, Double>? = Bootstrap.computeInterval(sample) { compute(it) }
+
   override fun evaluate(sessions: List<Session>, comparator: SuggestionsComparator): Double {
-    val fileSample = Sample()
-    sessions.stream()
-      .flatMap { session -> session.lookups.stream() }
+    val fileSample = mutableListOf<Double>()
+    sessions
+      .flatMap { session -> session.lookups }
       .filter(::shouldInclude)
       .forEach {
         this.sample.add(it.latency.toDouble())
@@ -21,7 +23,7 @@ abstract class LatencyMetric(override val name: String) : Metric {
     return compute(fileSample)
   }
 
-  abstract fun compute(sample: Sample): Double
+  abstract fun compute(sample: List<Double>): Double
 
   open fun shouldInclude(lookup: Lookup) = true
 }
@@ -29,7 +31,7 @@ abstract class LatencyMetric(override val name: String) : Metric {
 class MaxLatencyMetric : LatencyMetric(NAME) {
   override val valueType = MetricValueType.INT
 
-  override fun compute(sample: Sample): Double = sample.max()
+  override fun compute(sample: List<Double>): Double = sample.max()
 
   companion object {
     const val NAME = "Max Latency"
@@ -40,7 +42,7 @@ class TotalLatencyMetric : LatencyMetric(NAME) {
   override val valueType = MetricValueType.DOUBLE
   override val showByDefault = false
 
-  override fun compute(sample: Sample): Double = sample.sum()
+  override fun compute(sample: List<Double>): Double = sample.sum()
 
   companion object {
     const val NAME = "Total Latency"
@@ -50,7 +52,7 @@ class TotalLatencyMetric : LatencyMetric(NAME) {
 class MeanLatencyMetric(private val filterZeroes: Boolean = false) : LatencyMetric(NAME) {
   override val valueType = MetricValueType.DOUBLE
 
-  override fun compute(sample: Sample): Double = sample.mean()
+  override fun compute(sample: List<Double>): Double = sample.average()
 
   override fun shouldInclude(lookup: Lookup) = if (filterZeroes) lookup.latency > 0 else true
 

plugins/evaluation-plugin/core/src/com/intellij/cce/metric/Metric.kt

@@ -4,9 +4,15 @@ import com.intellij.cce.core.Session
 
 interface Metric {
   fun evaluate(sessions: List<Session>, comparator: SuggestionsComparator = SuggestionsComparator.DEFAULT): Number
+
+  fun confidenceInterval(): Pair<Double, Double>? = null
+
   val value: Double
+
   val name: String
+
   val valueType: MetricValueType
+
   val showByDefault: Boolean
     get() = true
 }

plugins/evaluation-plugin/core/src/com/intellij/cce/metric/MetricInfo.kt

@@ -1,6 +1,13 @@
 package com.intellij.cce.metric
 
-class MetricInfo(name: String, val value: Double, evaluationType: String, val valueType: MetricValueType, val showByDefault: Boolean) {
+class MetricInfo(
+  name: String,
+  val value: Double,
+  val confidenceInterval: Pair<Double, Double>?,
+  evaluationType: String,
+  val valueType: MetricValueType,
+  val showByDefault: Boolean) {
+
   val name = name.filter { it.isLetterOrDigit() }
   val evaluationType = evaluationType.filter { it.isLetterOrDigit() }
-}
+}

plugins/evaluation-plugin/core/src/com/intellij/cce/metric/MetricsEvaluator.kt

@@ -44,10 +44,20 @@ class MetricsEvaluator private constructor(private val evaluationType: String) {
   private fun registerMetrics(metrics: Collection<Metric>) = this.metrics.addAll(metrics)
 
   fun evaluate(sessions: List<Session>, comparator: SuggestionsComparator = SuggestionsComparator.DEFAULT): List<MetricInfo> {
-    return metrics.map { MetricInfo(it.name, it.evaluate(sessions, comparator).toDouble(), evaluationType, it.valueType, it.showByDefault) }
+    val result =  metrics.map {
+      MetricInfo(
+        name = it.name,
+        value = it.evaluate(sessions, comparator).toDouble(),
+        confidenceInterval = null,
+        evaluationType = evaluationType,
+        valueType = it.valueType,
+        showByDefault = it.showByDefault
+      )
+    }
+    return result
   }
 
   fun result(): List<MetricInfo> {
-    return metrics.map { MetricInfo(it.name, it.value, evaluationType, it.valueType, it.showByDefault) }
+    return metrics.map { MetricInfo(it.name, it.value, it.confidenceInterval(), evaluationType, it.valueType, it.showByDefault) }
   }
 }

plugins/evaluation-plugin/core/src/com/intellij/cce/metric/util/Bootstrap.kt

@@ -0,0 +1,20 @@
+package com.intellij.cce.metric.util
+
+import kotlin.random.Random
+
+internal object Bootstrap {
+  fun <T> computeInterval(sample: List<T>, callback: (List<T>) -> Double): Pair<Double, Double> {
+    val bootstrapSample = mutableListOf<Double>()
+    for (i in 0 until SAMPLE_SIZE) {
+      val newSample = mutableListOf<T>()
+      for (j in sample.indices) {
+        newSample.add(sample[Random.nextInt(sample.size)])
+      }
+      bootstrapSample.add(callback(newSample))
+    }
+    bootstrapSample.sort()
+    return bootstrapSample[(SAMPLE_SIZE * 0.025).toInt()] to bootstrapSample[(SAMPLE_SIZE * 0.975).toInt()]
+  }
+
+  private const val SAMPLE_SIZE = 100000
+}

plugins/evaluation-plugin/core/src/com/intellij/cce/report/HtmlReportGenerator.kt

@@ -149,13 +149,14 @@ class HtmlReportGenerator(
     if (withDiff) evaluationTypes.add(diffColumnTitle)
     var rowId = 1
 
-    val errorMetrics = globalMetrics.map { MetricInfo(it.name, Double.NaN, it.evaluationType, it.valueType, it.showByDefault) }
+    val errorMetrics = globalMetrics.map { MetricInfo(it.name, Double.NaN, null, it.evaluationType, it.valueType, it.showByDefault) }
 
     fun getReportMetrics(repRef: ReferenceInfo) = globalMetrics.map { metric ->
+      val refMetric = repRef.metrics.find { it.name == metric.name && it.evaluationType == metric.evaluationType }
       MetricInfo(
         metric.name,
-        repRef.metrics.find { it.name == metric.name && it.evaluationType == metric.evaluationType }?.value
-        ?: Double.NaN,
+        refMetric?.value ?: Double.NaN,
+        refMetric?.confidenceInterval,
         metric.evaluationType,
         metric.valueType,
         metric.showByDefault
@@ -166,11 +167,11 @@ class HtmlReportGenerator(
       if (withDiff) listOf(metrics, metrics
         .groupBy({ it.name }, { Triple(it.value, it.valueType, it.showByDefault) })
         .mapValues { with(it.value) { Triple(first().first - last().first, first().second, first().third) } }
-        .map { MetricInfo(it.key, it.value.first, diffColumnTitle, it.value.second, it.value.third) }).flatten()
+        .map { MetricInfo(it.key, it.value.first, null, diffColumnTitle, it.value.second, it.value.third) }).flatten()
       else metrics
                                                            ).joinToString(",") {
         "${it.name}${it.evaluationType}:'${
-          formatMetricValue(it.value, it.valueType)
+          formatMetricValue(it.value, it.confidenceInterval, it.valueType)
         }'"
       }
 
@@ -202,13 +203,19 @@ class HtmlReportGenerator(
 
   private fun MetricInfo.visible(): Boolean = if (defaultMetrics != null) name in defaultMetrics else showByDefault
 
-  private fun formatMetricValue(value: Double, type: MetricValueType): String = when {
+  private fun formatMetricValue(value: Double, confidenceInterval: Pair<Double, Double>?, type: MetricValueType): String = when {
     value.isNaN() -> "—"
-    type == MetricValueType.INT -> "${value.toInt()}"
-    type == MetricValueType.DOUBLE -> "%.3f".format(Locale.US, value)
+    type == MetricValueType.INT -> "${value.toInt()}" + (confidenceInterval?.let {
+      " (${confidenceInterval.first.toInt()}; ${confidenceInterval.second.toInt()})"
+    } ?: "")
+    type == MetricValueType.DOUBLE -> value.format() + (confidenceInterval?.let {
+      " (${confidenceInterval.first.format()}; ${confidenceInterval.second.format()})"
+    } ?: "")
     else -> throw IllegalArgumentException("Unknown metric value type")
   }
 
+  private fun Double.format() = "%.3f".format(Locale.US, this)
+
   private fun getErrorLink(errRef: Map.Entry<String, Path>): String =
     "\"<a href='${getHtmlRelativePath(dirs.filterDir, errRef.value)}' class='errRef' target='_blank'>${
       Paths.get(errRef.key).fileName

plugins/evaluation-plugin/src/com/intellij/cce/evaluation/step/ReportGenerationStep.kt

@@ -160,9 +160,4 @@ class ReportGenerationStep(
       ReportInfo(it.type, it.generateGlobalReport(metrics))
     }
   }
-
-  private fun List<Boolean>.allEquals(): Boolean {
-    return distinct().singleOrNull()
-           ?: throw IllegalArgumentException("Can't create workspace for code golf and not code golf evaluating together")
-  }
 }