From 00e64bd481aaafcac25ac3a89f166f7786fe8025 Mon Sep 17 00:00:00 2001
From: Anton Spilnyy <Anton.Spilnyy@jetbrains.com>
Date: Fri, 21 Nov 2025 19:32:49 +0000
Subject: [PATCH] [aia-eval] LME-681 Configure eval for codex+gemini in AIA

GitOrigin-RevId: 2e5beeb7968b2fab572c5a259d6623da87d7dbc4
---
 .../codegeneration/java_swe_codex.json        | 38 +++++++++++++++++++
 .../codegeneration/java_swe_gemini.json       | 38 +++++++++++++++++++
 2 files changed, 76 insertions(+)
 create mode 100644 plugins/evaluation-plugin/languages/evaluationconfig/codegeneration/java_swe_codex.json
 create mode 100644 plugins/evaluation-plugin/languages/evaluationconfig/codegeneration/java_swe_gemini.json

diff --git a/plugins/evaluation-plugin/languages/evaluationconfig/codegeneration/java_swe_codex.json b/plugins/evaluation-plugin/languages/evaluationconfig/codegeneration/java_swe_codex.json
new file mode 100644
index 000000000000..6abc0c979ec9
--- /dev/null
+++ b/plugins/evaluation-plugin/languages/evaluationconfig/codegeneration/java_swe_codex.json
@@ -0,0 +1,38 @@
+{
+  "projectPath": "../ml-eval/jackson-core",
+  "repository": "https://github.com/fasterxml/jackson-core",
+  "revision": "f1dc3c512d211ae3e14fb59af231caebf037d510",
+  "outputDir": "bin/ml-eval-code-generation-swe-java-output",
+  "strategy": {
+    "instanceId": "fasterxml__jackson-core-1142",
+    "runner": "CODEX",
+    "invocationParams": {
+      "runFeatureTimeoutMin": 20
+    }
+  },
+  "fileDataset": {
+    "url": "remote:https://huggingface.co/datasets/JetBrains/eval_plugin/resolve/main/code-generation-swe/java/java_swe_bench.json"
+  },
+  "interpret": {
+    "saveLogs": true,
+    "sessionProbability": 1.0,
+    "sessionSeed": null,
+    "sessionsLimit": 400,
+    "filesLimit": 100,
+    "order": "LINEAR",
+    "trainTestSplit": 70,
+    "registry": "",
+    "iterationCount": 1
+  },
+  "reports": {
+    "evaluationTitle": "Default",
+    "sessionsFilters": [],
+    "comparisonFilters": []
+  },
+  "language": "Java",
+  "actions": {
+    "evaluationRoots": [
+      "."
+    ]
+  }
+}
diff --git a/plugins/evaluation-plugin/languages/evaluationconfig/codegeneration/java_swe_gemini.json b/plugins/evaluation-plugin/languages/evaluationconfig/codegeneration/java_swe_gemini.json
new file mode 100644
index 000000000000..f2a17f2eba1d
--- /dev/null
+++ b/plugins/evaluation-plugin/languages/evaluationconfig/codegeneration/java_swe_gemini.json
@@ -0,0 +1,38 @@
+{
+  "projectPath": "../ml-eval/jackson-core",
+  "repository": "https://github.com/fasterxml/jackson-core",
+  "revision": "f1dc3c512d211ae3e14fb59af231caebf037d510",
+  "outputDir": "bin/ml-eval-code-generation-swe-java-output",
+  "strategy": {
+    "instanceId": "fasterxml__jackson-core-1142",
+    "runner": "GEMINI",
+    "invocationParams": {
+      "runFeatureTimeoutMin": 20
+    }
+  },
+  "fileDataset": {
+    "url": "remote:https://huggingface.co/datasets/JetBrains/eval_plugin/resolve/main/code-generation-swe/java/java_swe_bench.json"
+  },
+  "interpret": {
+    "saveLogs": true,
+    "sessionProbability": 1.0,
+    "sessionSeed": null,
+    "sessionsLimit": 400,
+    "filesLimit": 100,
+    "order": "LINEAR",
+    "trainTestSplit": 70,
+    "registry": "",
+    "iterationCount": 1
+  },
+  "reports": {
+    "evaluationTitle": "Default",
+    "sessionsFilters": [],
+    "comparisonFilters": []
+  },
+  "language": "Java",
+  "actions": {
+    "evaluationRoots": [
+      "."
+    ]
+  }
+}