From 18cc7383376c48aec7df1e2a017b75276b7c69fc Mon Sep 17 00:00:00 2001
From: Alexey Kalina <alexey.kalina@jetbrains.com>
Date: Thu, 18 Jul 2024 11:21:40 +0200
Subject: [PATCH] [evaluation-plugin] take into account empty lookups in
 NotEmptyResultsRatio metric (LME-10)

GitOrigin-RevId: 839f9efad7adc00ea9c0990a42d9535ddbefdaa2
---
 .../cce/metric/NotEmptyResultsRatio.kt        |  2 +-
 .../cce/metric/CodeGenerationMetricsTest.kt   | 38 ++++++++++++-------
 2 files changed, 26 insertions(+), 14 deletions(-)

diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/NotEmptyResultsRatio.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/NotEmptyResultsRatio.kt
index e79d7d405da5..6e9b71a1b5c4 100644
--- a/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/NotEmptyResultsRatio.kt
+++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/metric/NotEmptyResultsRatio.kt
@@ -19,7 +19,7 @@ class NotEmptyResultsRatio : Metric {
     sessions
       .flatMap { session -> session.lookups }
       .forEach {
-        if (it.suggestions.all { suggestion -> suggestion.text.isBlank() }) {
+        if (it.suggestions.isEmpty() || it.suggestions.all { suggestion -> suggestion.text.isBlank() }) {
           sample.add(0.0)
           fileSample.add(0.0)
         }
diff --git a/plugins/evaluation-plugin/test/com/intellij/cce/metric/CodeGenerationMetricsTest.kt b/plugins/evaluation-plugin/test/com/intellij/cce/metric/CodeGenerationMetricsTest.kt
index 8698b37b28a7..f03e469f110a 100644
--- a/plugins/evaluation-plugin/test/com/intellij/cce/metric/CodeGenerationMetricsTest.kt
+++ b/plugins/evaluation-plugin/test/com/intellij/cce/metric/CodeGenerationMetricsTest.kt
@@ -9,9 +9,9 @@ class CodeGenerationMetricsTest {
   @Test
   fun `test correct results`() {
     doTest(sessions = listOf(
-      session(result = "not-empty", withSyntaxErrors = false),
-      session(result = "not-empty-2", withSyntaxErrors = false),
-      session(result = "not-empty-3", withSyntaxErrors = false),
+      session(singleSuggestion(result = "not-empty"), withSyntaxErrors = false),
+      session(singleSuggestion(result = "not-empty-2"), withSyntaxErrors = false),
+      session(singleSuggestion(result = "not-empty-3"), withSyntaxErrors = false),
     ), listOf(
       WithoutSyntaxErrorsSessionRatio() to 1.0,
       NotEmptyResultsRatio() to 1.0
@@ -21,10 +21,10 @@ class CodeGenerationMetricsTest {
   @Test
   fun `test incorrect results`() {
     doTest(sessions = listOf(
-      session(result = "", withSyntaxErrors = true),
-      session(result = "", withSyntaxErrors = true),
-      session(result = "with-errors", withSyntaxErrors = true),
-      session(result = "with-errors-2", withSyntaxErrors = true),
+      session(singleSuggestion(result = ""), withSyntaxErrors = true),
+      session(singleSuggestion(result = ""), withSyntaxErrors = true),
+      session(singleSuggestion(result = "with-errors"), withSyntaxErrors = true),
+      session(singleSuggestion(result = "with-errors-2"), withSyntaxErrors = true),
     ), listOf(
       WithoutSyntaxErrorsSessionRatio() to 0.0,
       NotEmptyResultsRatio() to 0.5
@@ -34,28 +34,40 @@ class CodeGenerationMetricsTest {
   @Test
   fun `test mixed results`() {
     doTest(sessions = listOf(
-      session(result = "", withSyntaxErrors = true),
-      session(result = "not-empty", withSyntaxErrors = false),
-      session(result = "not-empty-2", withSyntaxErrors = false),
-      session(result = "with errors", withSyntaxErrors = true),
+      session(singleSuggestion(result = ""), withSyntaxErrors = true),
+      session(singleSuggestion(result = "not-empty"), withSyntaxErrors = false),
+      session(singleSuggestion(result = "not-empty-2"), withSyntaxErrors = false),
+      session(singleSuggestion(result = "with errors"), withSyntaxErrors = true),
     ), listOf(
       WithoutSyntaxErrorsSessionRatio() to 0.5,
       NotEmptyResultsRatio() to 0.75
     ))
   }
 
+  @Test
+  fun `test session without suggestions`() {
+    doTest(sessions = listOf(
+      session(emptyList(), withSyntaxErrors = true),
+    ), listOf(
+      WithoutSyntaxErrorsSessionRatio() to 0.0,
+      NotEmptyResultsRatio() to 0.0
+    ))
+  }
+
   private fun doTest(sessions: List<Session>, metric2expected: List<Pair<Metric, Double>>) {
     for ((metric, expected) in metric2expected) {
       Assertions.assertEquals(expected, metric.evaluate(sessions))
     }
   }
 
-  private fun session(result: String, withSyntaxErrors: Boolean): Session {
+  private fun singleSuggestion(result: String): List<Suggestion> = listOf(Suggestion(result, result, SuggestionSource.INTELLIJ))
+
+  private fun session(suggestions: List<Suggestion>, withSyntaxErrors: Boolean): Session {
     val session = Session(0, "", 0, TokenProperties.UNKNOWN)
     session.addLookup(Lookup(
       prefix = "",
       offset = 0,
-      suggestions = listOf(Suggestion(result, result, SuggestionSource.INTELLIJ)),
+      suggestions = suggestions,
       latency = 0,
       features = null,
       selectedPosition = 0,