MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

LLM-14672: added maximum Sessions threshold for some metrics that were causing timeout in confidenceInterval builds

Browse Source 
GitOrigin-RevId: 059e6651fdf723c31f56b81b2a2cb49703082c60
This commit is contained in:
Sebastian.Chiriac
2025-02-05 22:50:31 +00:00
committed by intellij-monorepo-bot
parent 232a78247c
commit 72e1649597
5 changed files with 50 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/evaluation-plugin/core/src/com/intellij/cce/metric/LatencyMetrics.kt
+17
View File
@@ -3,6 +3,7 @@ package com.intellij.cce.metric
import com.intellij.cce.core.Lookup
import com.intellij.cce.core.Session
import com.intellij.cce.metric.Metric.Companion.LOG
import com.intellij.cce.metric.util.Bootstrap
abstract class LatencyMetric(override val name: String) : Metric {
@@ -70,6 +71,14 @@ class PercentileLatencyMetric(private val percentile: Int) : LatencyMetric("Late
override val description: String = "Latency $percentile percentile by all invocations"
override val showByDefault = false
override fun shouldComputeIntervals(numberOfSessions: Int): Boolean {
val maximumSessions = 10000
if (numberOfSessions > maximumSessions) LOG.warn("Confidence Interval not calculated for metric $name because number of sessions $numberOfSessions exceeds maximum threshold $maximumSessions")
return super.shouldComputeIntervals(numberOfSessions) && numberOfSessions <= 10000
}
override fun compute(sample: List<Double>): Double = computePercentile(sample, percentile)
}
@@ -78,6 +87,14 @@ class SuccessPercentileLatencyMetric(private val percentile: Int) : LatencyMetri
override val description: String = "Latency $percentile percentile by invocations with selected proposal"
override val showByDefault = false
override fun shouldComputeIntervals(numberOfSessions: Int): Boolean {
val maximumSessions = 10000
if (numberOfSessions > maximumSessions) LOG.warn("Confidence Interval not calculated for metric $name because number of sessions $numberOfSessions exceeds maximum threshold $maximumSessions")
return super.shouldComputeIntervals(numberOfSessions) && numberOfSessions <= 10000
}
override fun compute(sample: List<Double>): Double = computePercentile(sample, percentile)
override fun shouldInclude(lookup: Lookup): Boolean = lookup.selectedPosition >= 0
plugins/evaluation-plugin/core/src/com/intellij/cce/metric/Metric.kt
+10 -4
View File
@@ -4,8 +4,6 @@ package com.intellij.cce.metric
import com.intellij.cce.core.Session
import com.intellij.openapi.diagnostic.Logger
private val LOG: Logger = Logger.getInstance(Metric::class.java)
interface Metric {
val supportsIndividualScores: Boolean
get() = false
@@ -34,6 +32,14 @@ interface Metric {
val showByDefault: Boolean
val shouldComputeIntervals: Boolean
get() = System.getenv("cce_compute_confidence_intervals")?.toBooleanStrictOrNull() == true
fun shouldComputeIntervals(numberOfSessions: Int): Boolean {
val maximumSessions = 200000
if (numberOfSessions > maximumSessions) LOG.warn("Confidence Interval not calculated for metric $name because number of sessions $numberOfSessions exceeds maximum threshold $maximumSessions")
return numberOfSessions <= maximumSessions && System.getenv("cce_compute_confidence_intervals")?.toBooleanStrictOrNull() == true
}
companion object {
val LOG: Logger = Logger.getInstance(Metric::class.java)
}
}
plugins/evaluation-plugin/core/src/com/intellij/cce/metric/MetricsEvaluator.kt
+4 -4
View File
@@ -34,7 +34,7 @@ class MetricsEvaluator private constructor(private val evaluationType: String) {
private fun registerMetrics(metrics: Collection<Metric>) = this.metrics.addAll(metrics)
fun evaluate(sessions: List<Session>): List<MetricInfo> {
fun evaluate(sessions: List<Session>, numberOfSessions: Int): List<MetricInfo> {
return metrics.map { metric ->
val (overallScore, individualScores) = if (metric.supportsIndividualScores) {
val evaluationResult = metric.evaluateWithIndividualScores(sessions)
@@ -47,7 +47,7 @@ class MetricsEvaluator private constructor(private val evaluationType: String) {
name = metric.name,
description = metric.description,
value = overallScore,
confidenceInterval = metric.confidenceInterval(),
confidenceInterval = if (metric.shouldComputeIntervals(numberOfSessions)) metric.confidenceInterval() else null,
evaluationType = evaluationType,
valueType = metric.valueType,
showByDefault = metric.showByDefault,
@@ -56,13 +56,13 @@ class MetricsEvaluator private constructor(private val evaluationType: String) {
}
}
fun globalMetricInfos(): List<MetricInfo> {
fun globalMetricInfos(numberOfSessions: Int): List<MetricInfo> {
return metrics.map {
MetricInfo(
name = it.name,
description = it.description,
value = it.value,
confidenceInterval = if (it.shouldComputeIntervals) it.confidenceInterval() else null,
confidenceInterval = if (it.shouldComputeIntervals(numberOfSessions)) it.confidenceInterval() else null,
evaluationType = evaluationType,
valueType = it.valueType,
showByDefault = it.showByDefault,
plugins/evaluation-plugin/src/com/intellij/cce/evaluation/step/ReportGenerationStep.kt
+18 -3
View File
@@ -13,9 +13,9 @@ import com.intellij.cce.workspace.filter.CompareSessionsFilter
import com.intellij.cce.workspace.filter.CompareSessionsStorage
import com.intellij.cce.workspace.filter.CompareSessionsStorageImpl
import com.intellij.cce.workspace.filter.SessionsFilter
import com.intellij.cce.workspace.info.FileEvaluationDataInfo
import com.intellij.cce.workspace.info.FileEvaluationInfo
import com.intellij.cce.workspace.info.FileSessionsInfo
import com.intellij.cce.workspace.info.FileEvaluationDataInfo
import com.intellij.cce.workspace.storages.FileErrorsStorage
import com.intellij.cce.workspace.storages.SessionsStorage
import com.intellij.openapi.application.ApplicationManager
@@ -128,6 +128,21 @@ class ReportGenerationStep<T : EvaluationStrategy>(
title to MetricsEvaluator.withMetrics(title, feature.getMetrics())
}.toMap()
var numberOfSessions = 0
for (sessionFile in sessionFiles.filter { it.value.size == sessionStorages.size }) {
var sessionsInfo: FileSessionsInfo? = null
for (file in sessionFile.value) {
sessionsInfo = sessionStorages[evaluationTitles.indexOf(file.evaluationType)].getSessions(file.path)
comparisonStorage.add(file.evaluationType, sessionsInfo.sessions)
}
if (sessionsInfo == null) throw IllegalStateException("Sessions file doesn't exist")
for (file in sessionFile.value) {
val sessionsEvaluation = sessionsInfo.copy(sessions = comparisonStorage.get(file.evaluationType))
numberOfSessions += sessionsEvaluation.sessions.sumOf { it.lookups.size }
}
}
for (sessionFile in sessionFiles.filter { it.value.size == sessionStorages.size }) {
val fileEvaluations = mutableListOf<FileEvaluationInfo>()
var sessionsInfo: FileSessionsInfo? = null
@@ -141,7 +156,7 @@ class ReportGenerationStep<T : EvaluationStrategy>(
sessions = comparisonStorage.get(file.evaluationType)
)
val evaluator = title2evaluator.getValue(file.evaluationType)
val metricsEvaluation = evaluator.evaluate(sessionsEvaluation.sessions)
val metricsEvaluation = evaluator.evaluate(sessionsEvaluation.sessions, numberOfSessions)
val sessionIndividualEvaluationMap = metricsEvaluation
.flatMap { it.individualScores?.entries ?: emptySet() }
@@ -171,7 +186,7 @@ class ReportGenerationStep<T : EvaluationStrategy>(
for (errorsStorage in errorStorages) {
reportGenerators.forEach { it.generateErrorReports(errorsStorage.getErrors()) }
}
val globalMetricInfos = title2evaluator.values.flatMap(MetricsEvaluator::globalMetricInfos)
val globalMetricInfos = title2evaluator.values.flatMap { it.globalMetricInfos(numberOfSessions) }
return reportGenerators.map {
ReportInfo(it.type, it.generateGlobalReport(globalMetricInfos))
plugins/evaluation-plugin/test/com/intellij/cce/metric/MetricsEvaluatorTest.kt
+1 -1
View File
@@ -55,7 +55,7 @@ class MetricsEvaluatorTest {
@Test
fun `test metrics evaluator`() {
val evaluator = MetricsEvaluator.withDefaultMetrics("")
val result = evaluator.evaluate(listOf(sessionTop1, sessionTop3, sessionTop3, sessionNone))
val result = evaluator.evaluate(listOf(sessionTop1, sessionTop3, sessionTop3, sessionNone), 0)
Assertions.assertTrue(result.isNotEmpty())
}