From 72e1649597fbb0f9f2a2de4d33297604c87f67d3 Mon Sep 17 00:00:00 2001 From: "Sebastian.Chiriac" Date: Tue, 4 Feb 2025 17:56:45 +0100 Subject: [PATCH] LLM-14672: added maximum Sessions threshold for some metrics that were causing timeout in confidenceInterval builds GitOrigin-RevId: 059e6651fdf723c31f56b81b2a2cb49703082c60 --- .../com/intellij/cce/metric/LatencyMetrics.kt | 17 +++++++++++++++ .../src/com/intellij/cce/metric/Metric.kt | 14 +++++++++---- .../intellij/cce/metric/MetricsEvaluator.kt | 8 +++---- .../evaluation/step/ReportGenerationStep.kt | 21 ++++++++++++++++--- .../cce/metric/MetricsEvaluatorTest.kt | 2 +- 5 files changed, 50 insertions(+), 12 deletions(-) diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/LatencyMetrics.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/LatencyMetrics.kt index 39020ece933e..373f1230c7f9 100644 --- a/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/LatencyMetrics.kt +++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/LatencyMetrics.kt @@ -3,6 +3,7 @@ package com.intellij.cce.metric import com.intellij.cce.core.Lookup import com.intellij.cce.core.Session +import com.intellij.cce.metric.Metric.Companion.LOG import com.intellij.cce.metric.util.Bootstrap abstract class LatencyMetric(override val name: String) : Metric { @@ -70,6 +71,14 @@ class PercentileLatencyMetric(private val percentile: Int) : LatencyMetric("Late override val description: String = "Latency $percentile percentile by all invocations" override val showByDefault = false + override fun shouldComputeIntervals(numberOfSessions: Int): Boolean { + val maximumSessions = 10000 + + if (numberOfSessions > maximumSessions) LOG.warn("Confidence Interval not calculated for metric $name because number of sessions $numberOfSessions exceeds maximum threshold $maximumSessions") + + return super.shouldComputeIntervals(numberOfSessions) && numberOfSessions <= 10000 + } + override fun compute(sample: List): Double = computePercentile(sample, percentile) } @@ -78,6 +87,14 @@ class SuccessPercentileLatencyMetric(private val percentile: Int) : LatencyMetri override val description: String = "Latency $percentile percentile by invocations with selected proposal" override val showByDefault = false + override fun shouldComputeIntervals(numberOfSessions: Int): Boolean { + val maximumSessions = 10000 + + if (numberOfSessions > maximumSessions) LOG.warn("Confidence Interval not calculated for metric $name because number of sessions $numberOfSessions exceeds maximum threshold $maximumSessions") + + return super.shouldComputeIntervals(numberOfSessions) && numberOfSessions <= 10000 + } + override fun compute(sample: List): Double = computePercentile(sample, percentile) override fun shouldInclude(lookup: Lookup): Boolean = lookup.selectedPosition >= 0 diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/Metric.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/Metric.kt index 379372b5508c..392781ecd0ff 100644 --- a/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/Metric.kt +++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/Metric.kt @@ -4,8 +4,6 @@ package com.intellij.cce.metric import com.intellij.cce.core.Session import com.intellij.openapi.diagnostic.Logger -private val LOG: Logger = Logger.getInstance(Metric::class.java) - interface Metric { val supportsIndividualScores: Boolean get() = false @@ -34,6 +32,14 @@ interface Metric { val showByDefault: Boolean - val shouldComputeIntervals: Boolean - get() = System.getenv("cce_compute_confidence_intervals")?.toBooleanStrictOrNull() == true + fun shouldComputeIntervals(numberOfSessions: Int): Boolean { + val maximumSessions = 200000 + + if (numberOfSessions > maximumSessions) LOG.warn("Confidence Interval not calculated for metric $name because number of sessions $numberOfSessions exceeds maximum threshold $maximumSessions") + return numberOfSessions <= maximumSessions && System.getenv("cce_compute_confidence_intervals")?.toBooleanStrictOrNull() == true + } + + companion object { + val LOG: Logger = Logger.getInstance(Metric::class.java) + } } diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/MetricsEvaluator.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/MetricsEvaluator.kt index 23c915a0e8fe..2c6686f2675e 100644 --- a/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/MetricsEvaluator.kt +++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/MetricsEvaluator.kt @@ -34,7 +34,7 @@ class MetricsEvaluator private constructor(private val evaluationType: String) { private fun registerMetrics(metrics: Collection) = this.metrics.addAll(metrics) - fun evaluate(sessions: List): List { + fun evaluate(sessions: List, numberOfSessions: Int): List { return metrics.map { metric -> val (overallScore, individualScores) = if (metric.supportsIndividualScores) { val evaluationResult = metric.evaluateWithIndividualScores(sessions) @@ -47,7 +47,7 @@ class MetricsEvaluator private constructor(private val evaluationType: String) { name = metric.name, description = metric.description, value = overallScore, - confidenceInterval = metric.confidenceInterval(), + confidenceInterval = if (metric.shouldComputeIntervals(numberOfSessions)) metric.confidenceInterval() else null, evaluationType = evaluationType, valueType = metric.valueType, showByDefault = metric.showByDefault, @@ -56,13 +56,13 @@ class MetricsEvaluator private constructor(private val evaluationType: String) { } } - fun globalMetricInfos(): List { + fun globalMetricInfos(numberOfSessions: Int): List { return metrics.map { MetricInfo( name = it.name, description = it.description, value = it.value, - confidenceInterval = if (it.shouldComputeIntervals) it.confidenceInterval() else null, + confidenceInterval = if (it.shouldComputeIntervals(numberOfSessions)) it.confidenceInterval() else null, evaluationType = evaluationType, valueType = it.valueType, showByDefault = it.showByDefault, diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/step/ReportGenerationStep.kt b/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/step/ReportGenerationStep.kt index b534c9e4fa17..b6845aca2fe3 100644 --- a/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/step/ReportGenerationStep.kt +++ b/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/step/ReportGenerationStep.kt @@ -13,9 +13,9 @@ import com.intellij.cce.workspace.filter.CompareSessionsFilter import com.intellij.cce.workspace.filter.CompareSessionsStorage import com.intellij.cce.workspace.filter.CompareSessionsStorageImpl import com.intellij.cce.workspace.filter.SessionsFilter +import com.intellij.cce.workspace.info.FileEvaluationDataInfo import com.intellij.cce.workspace.info.FileEvaluationInfo import com.intellij.cce.workspace.info.FileSessionsInfo -import com.intellij.cce.workspace.info.FileEvaluationDataInfo import com.intellij.cce.workspace.storages.FileErrorsStorage import com.intellij.cce.workspace.storages.SessionsStorage import com.intellij.openapi.application.ApplicationManager @@ -128,6 +128,21 @@ class ReportGenerationStep( title to MetricsEvaluator.withMetrics(title, feature.getMetrics()) }.toMap() + var numberOfSessions = 0 + + for (sessionFile in sessionFiles.filter { it.value.size == sessionStorages.size }) { + var sessionsInfo: FileSessionsInfo? = null + for (file in sessionFile.value) { + sessionsInfo = sessionStorages[evaluationTitles.indexOf(file.evaluationType)].getSessions(file.path) + comparisonStorage.add(file.evaluationType, sessionsInfo.sessions) + } + if (sessionsInfo == null) throw IllegalStateException("Sessions file doesn't exist") + for (file in sessionFile.value) { + val sessionsEvaluation = sessionsInfo.copy(sessions = comparisonStorage.get(file.evaluationType)) + numberOfSessions += sessionsEvaluation.sessions.sumOf { it.lookups.size } + } + } + for (sessionFile in sessionFiles.filter { it.value.size == sessionStorages.size }) { val fileEvaluations = mutableListOf() var sessionsInfo: FileSessionsInfo? = null @@ -141,7 +156,7 @@ class ReportGenerationStep( sessions = comparisonStorage.get(file.evaluationType) ) val evaluator = title2evaluator.getValue(file.evaluationType) - val metricsEvaluation = evaluator.evaluate(sessionsEvaluation.sessions) + val metricsEvaluation = evaluator.evaluate(sessionsEvaluation.sessions, numberOfSessions) val sessionIndividualEvaluationMap = metricsEvaluation .flatMap { it.individualScores?.entries ?: emptySet() } @@ -171,7 +186,7 @@ class ReportGenerationStep( for (errorsStorage in errorStorages) { reportGenerators.forEach { it.generateErrorReports(errorsStorage.getErrors()) } } - val globalMetricInfos = title2evaluator.values.flatMap(MetricsEvaluator::globalMetricInfos) + val globalMetricInfos = title2evaluator.values.flatMap { it.globalMetricInfos(numberOfSessions) } return reportGenerators.map { ReportInfo(it.type, it.generateGlobalReport(globalMetricInfos)) diff --git a/plugins/evaluation-plugin/test/com/intellij/cce/metric/MetricsEvaluatorTest.kt b/plugins/evaluation-plugin/test/com/intellij/cce/metric/MetricsEvaluatorTest.kt index 0e972aae674a..861baa889bc2 100644 --- a/plugins/evaluation-plugin/test/com/intellij/cce/metric/MetricsEvaluatorTest.kt +++ b/plugins/evaluation-plugin/test/com/intellij/cce/metric/MetricsEvaluatorTest.kt @@ -55,7 +55,7 @@ class MetricsEvaluatorTest { @Test fun `test metrics evaluator`() { val evaluator = MetricsEvaluator.withDefaultMetrics("") - val result = evaluator.evaluate(listOf(sessionTop1, sessionTop3, sessionTop3, sessionNone)) + val result = evaluator.evaluate(listOf(sessionTop1, sessionTop3, sessionTop3, sessionNone), 0) Assertions.assertTrue(result.isNotEmpty()) }