diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/CompletionGolfMetrics.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/CompletionGolfMetrics.kt index 1a28401f1652..5feb2032d835 100644 --- a/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/CompletionGolfMetrics.kt +++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/CompletionGolfMetrics.kt @@ -3,6 +3,7 @@ package com.intellij.cce.metric import com.intellij.cce.actions.selectedWithoutPrefix import com.intellij.cce.core.Lookup import com.intellij.cce.core.Session +import com.intellij.cce.metric.util.Bootstrap import com.intellij.cce.metric.util.Sample import org.apache.commons.lang.StringUtils @@ -144,12 +145,17 @@ internal class MovesCountNormalised : Metric { internal abstract class SimilarityMetric : Metric { private var totalMatched: Double = 0.0 private var totalExpected: Double = 0.0 + private var sample: MutableList> = mutableListOf() override val valueType = MetricValueType.DOUBLE override val showByDefault: Boolean = false override val value: Double get() = totalMatched / totalExpected + override fun confidenceInterval(): Pair = Bootstrap.computeInterval(sample) { values -> + values.sumOf { it.first } / values.sumOf { it.second } + } + override fun evaluate(sessions: List, comparator: SuggestionsComparator): Double { var matched = 0.0 var expected = 0.0 @@ -157,7 +163,9 @@ internal abstract class SimilarityMetric : Metric { for (lookup in session.lookups) { val expectedText = session.expectedText.substring(lookup.offset) expected += expectedText.length - matched += computeSimilarity(lookup, expectedText) ?: 0.0 + val similarity = computeSimilarity(lookup, expectedText) ?: 0.0 + matched += similarity + sample.add(Pair(similarity, expectedText.length.toDouble())) } } totalMatched += matched @@ -209,11 +217,13 @@ internal class PerfectLine : CompletionGolfMetric() { } internal class Precision : Metric { - private val sample = Sample() + private val sample = mutableListOf() override val name = "Precision" override val valueType = MetricValueType.DOUBLE override val value: Double - get() = sample.mean() + get() = sample.average() + + override fun confidenceInterval(): Pair = Bootstrap.computeInterval(sample) { it.average() } override fun evaluate(sessions: List, comparator: SuggestionsComparator): Double { val fileSample = Sample() @@ -234,11 +244,13 @@ internal class Precision : Metric { } internal open class RecallAt(private val n: Int) : Metric { - private val sample = Sample() + private val sample = mutableListOf() override val name = "RecallAt$n" override val valueType = MetricValueType.DOUBLE override val value: Double - get() = sample.mean() + get() = sample.average() + + override fun confidenceInterval(): Pair = Bootstrap.computeInterval(sample) { it.average() } override fun evaluate(sessions: List, comparator: SuggestionsComparator): Double { val fileSample = Sample() diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/LatencyMetrics.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/LatencyMetrics.kt index 627309796183..a22c987d682c 100644 --- a/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/LatencyMetrics.kt +++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/LatencyMetrics.kt @@ -2,17 +2,19 @@ package com.intellij.cce.metric import com.intellij.cce.core.Lookup import com.intellij.cce.core.Session -import com.intellij.cce.metric.util.Sample +import com.intellij.cce.metric.util.Bootstrap abstract class LatencyMetric(override val name: String) : Metric { - private val sample = Sample() + private val sample = mutableListOf() override val value: Double get() = compute(sample) + override fun confidenceInterval(): Pair? = Bootstrap.computeInterval(sample) { compute(it) } + override fun evaluate(sessions: List, comparator: SuggestionsComparator): Double { - val fileSample = Sample() - sessions.stream() - .flatMap { session -> session.lookups.stream() } + val fileSample = mutableListOf() + sessions + .flatMap { session -> session.lookups } .filter(::shouldInclude) .forEach { this.sample.add(it.latency.toDouble()) @@ -21,7 +23,7 @@ abstract class LatencyMetric(override val name: String) : Metric { return compute(fileSample) } - abstract fun compute(sample: Sample): Double + abstract fun compute(sample: List): Double open fun shouldInclude(lookup: Lookup) = true } @@ -29,7 +31,7 @@ abstract class LatencyMetric(override val name: String) : Metric { class MaxLatencyMetric : LatencyMetric(NAME) { override val valueType = MetricValueType.INT - override fun compute(sample: Sample): Double = sample.max() + override fun compute(sample: List): Double = sample.max() companion object { const val NAME = "Max Latency" @@ -40,7 +42,7 @@ class TotalLatencyMetric : LatencyMetric(NAME) { override val valueType = MetricValueType.DOUBLE override val showByDefault = false - override fun compute(sample: Sample): Double = sample.sum() + override fun compute(sample: List): Double = sample.sum() companion object { const val NAME = "Total Latency" @@ -50,7 +52,7 @@ class TotalLatencyMetric : LatencyMetric(NAME) { class MeanLatencyMetric(private val filterZeroes: Boolean = false) : LatencyMetric(NAME) { override val valueType = MetricValueType.DOUBLE - override fun compute(sample: Sample): Double = sample.mean() + override fun compute(sample: List): Double = sample.average() override fun shouldInclude(lookup: Lookup) = if (filterZeroes) lookup.latency > 0 else true diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/Metric.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/Metric.kt index 01a966253476..82b134f28cc8 100644 --- a/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/Metric.kt +++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/Metric.kt @@ -4,9 +4,15 @@ import com.intellij.cce.core.Session interface Metric { fun evaluate(sessions: List, comparator: SuggestionsComparator = SuggestionsComparator.DEFAULT): Number + + fun confidenceInterval(): Pair? = null + val value: Double + val name: String + val valueType: MetricValueType + val showByDefault: Boolean get() = true } diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/MetricInfo.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/MetricInfo.kt index 9f9e6b958b5e..062cd3ed03ed 100644 --- a/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/MetricInfo.kt +++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/MetricInfo.kt @@ -1,6 +1,13 @@ package com.intellij.cce.metric -class MetricInfo(name: String, val value: Double, evaluationType: String, val valueType: MetricValueType, val showByDefault: Boolean) { +class MetricInfo( + name: String, + val value: Double, + val confidenceInterval: Pair?, + evaluationType: String, + val valueType: MetricValueType, + val showByDefault: Boolean) { + val name = name.filter { it.isLetterOrDigit() } val evaluationType = evaluationType.filter { it.isLetterOrDigit() } -} \ No newline at end of file +} diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/MetricsEvaluator.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/MetricsEvaluator.kt index 018238e1e8ab..dd27381e8b65 100644 --- a/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/MetricsEvaluator.kt +++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/MetricsEvaluator.kt @@ -44,10 +44,20 @@ class MetricsEvaluator private constructor(private val evaluationType: String) { private fun registerMetrics(metrics: Collection) = this.metrics.addAll(metrics) fun evaluate(sessions: List, comparator: SuggestionsComparator = SuggestionsComparator.DEFAULT): List { - return metrics.map { MetricInfo(it.name, it.evaluate(sessions, comparator).toDouble(), evaluationType, it.valueType, it.showByDefault) } + val result = metrics.map { + MetricInfo( + name = it.name, + value = it.evaluate(sessions, comparator).toDouble(), + confidenceInterval = null, + evaluationType = evaluationType, + valueType = it.valueType, + showByDefault = it.showByDefault + ) + } + return result } fun result(): List { - return metrics.map { MetricInfo(it.name, it.value, evaluationType, it.valueType, it.showByDefault) } + return metrics.map { MetricInfo(it.name, it.value, it.confidenceInterval(), evaluationType, it.valueType, it.showByDefault) } } } diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/util/Bootstrap.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/util/Bootstrap.kt new file mode 100644 index 000000000000..c05aad40c45f --- /dev/null +++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/util/Bootstrap.kt @@ -0,0 +1,20 @@ +package com.intellij.cce.metric.util + +import kotlin.random.Random + +internal object Bootstrap { + fun computeInterval(sample: List, callback: (List) -> Double): Pair { + val bootstrapSample = mutableListOf() + for (i in 0 until SAMPLE_SIZE) { + val newSample = mutableListOf() + for (j in sample.indices) { + newSample.add(sample[Random.nextInt(sample.size)]) + } + bootstrapSample.add(callback(newSample)) + } + bootstrapSample.sort() + return bootstrapSample[(SAMPLE_SIZE * 0.025).toInt()] to bootstrapSample[(SAMPLE_SIZE * 0.975).toInt()] + } + + private const val SAMPLE_SIZE = 100000 +} \ No newline at end of file diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/report/HtmlReportGenerator.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/report/HtmlReportGenerator.kt index d0bef83983d2..b0fce1ac9693 100644 --- a/plugins/evaluation-plugin/core/src/com/intellij/cce/report/HtmlReportGenerator.kt +++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/report/HtmlReportGenerator.kt @@ -149,13 +149,14 @@ class HtmlReportGenerator( if (withDiff) evaluationTypes.add(diffColumnTitle) var rowId = 1 - val errorMetrics = globalMetrics.map { MetricInfo(it.name, Double.NaN, it.evaluationType, it.valueType, it.showByDefault) } + val errorMetrics = globalMetrics.map { MetricInfo(it.name, Double.NaN, null, it.evaluationType, it.valueType, it.showByDefault) } fun getReportMetrics(repRef: ReferenceInfo) = globalMetrics.map { metric -> + val refMetric = repRef.metrics.find { it.name == metric.name && it.evaluationType == metric.evaluationType } MetricInfo( metric.name, - repRef.metrics.find { it.name == metric.name && it.evaluationType == metric.evaluationType }?.value - ?: Double.NaN, + refMetric?.value ?: Double.NaN, + refMetric?.confidenceInterval, metric.evaluationType, metric.valueType, metric.showByDefault @@ -166,11 +167,11 @@ class HtmlReportGenerator( if (withDiff) listOf(metrics, metrics .groupBy({ it.name }, { Triple(it.value, it.valueType, it.showByDefault) }) .mapValues { with(it.value) { Triple(first().first - last().first, first().second, first().third) } } - .map { MetricInfo(it.key, it.value.first, diffColumnTitle, it.value.second, it.value.third) }).flatten() + .map { MetricInfo(it.key, it.value.first, null, diffColumnTitle, it.value.second, it.value.third) }).flatten() else metrics ).joinToString(",") { "${it.name}${it.evaluationType}:'${ - formatMetricValue(it.value, it.valueType) + formatMetricValue(it.value, it.confidenceInterval, it.valueType) }'" } @@ -202,13 +203,19 @@ class HtmlReportGenerator( private fun MetricInfo.visible(): Boolean = if (defaultMetrics != null) name in defaultMetrics else showByDefault - private fun formatMetricValue(value: Double, type: MetricValueType): String = when { + private fun formatMetricValue(value: Double, confidenceInterval: Pair?, type: MetricValueType): String = when { value.isNaN() -> "—" - type == MetricValueType.INT -> "${value.toInt()}" - type == MetricValueType.DOUBLE -> "%.3f".format(Locale.US, value) + type == MetricValueType.INT -> "${value.toInt()}" + (confidenceInterval?.let { + " (${confidenceInterval.first.toInt()}; ${confidenceInterval.second.toInt()})" + } ?: "") + type == MetricValueType.DOUBLE -> value.format() + (confidenceInterval?.let { + " (${confidenceInterval.first.format()}; ${confidenceInterval.second.format()})" + } ?: "") else -> throw IllegalArgumentException("Unknown metric value type") } + private fun Double.format() = "%.3f".format(Locale.US, this) + private fun getErrorLink(errRef: Map.Entry): String = "\"${ Paths.get(errRef.key).fileName diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/step/ReportGenerationStep.kt b/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/step/ReportGenerationStep.kt index 881925b8865f..bc9af9f18d83 100644 --- a/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/step/ReportGenerationStep.kt +++ b/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/step/ReportGenerationStep.kt @@ -160,9 +160,4 @@ class ReportGenerationStep( ReportInfo(it.type, it.generateGlobalReport(metrics)) } } - - private fun List.allEquals(): Boolean { - return distinct().singleOrNull() - ?: throw IllegalArgumentException("Can't create workspace for code golf and not code golf evaluating together") - } }