diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluable/AiaConstants.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluable/AiaConstants.kt index 728fd281fe4f..dabd73e817ac 100644 --- a/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluable/AiaConstants.kt +++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluable/AiaConstants.kt @@ -31,4 +31,6 @@ const val AIA_EXACT_MATCH: String = "exact_match" const val AIA_AST_MATCH: String = "ast_match" const val AIA_PROBLEMS: String = "aia_problems" const val AIA_LLMC_LOG: String = "aia_log_llmc" -const val AIA_HTTP_LOG: String = "aia_log_http" \ No newline at end of file +const val AIA_HTTP_LOG: String = "aia_log_http" +const val AIA_CHAT_AND_APPLY_SCORE_KEY: String = "aia_chat_and_apply_score" +const val AIA_APPLY_SCORE_KEY: String = "aia_apply_score" \ No newline at end of file diff --git a/plugins/evaluation-plugin/languages/evaluationconfig/chatcodegeneration/v2/chat_apply.json b/plugins/evaluation-plugin/languages/evaluationconfig/chatcodegeneration/v2/chat_apply.json new file mode 100644 index 000000000000..30438e244e83 --- /dev/null +++ b/plugins/evaluation-plugin/languages/evaluationconfig/chatcodegeneration/v2/chat_apply.json @@ -0,0 +1,33 @@ +{ + "language": "Another", + "projectPath": "../ml-eval/apply-snippet-dataset", + "repository": "https://git.jetbrains.team/code-generation-evaluation/apply-snippet-eval-experiment.git", + "revision": "6261040e5fc8ff18985816e49a02c7ac81e45eba", + "outputDir": "bin/ml-eval-chat-apply-output", + "strategy": { + "behaviour": "FLASH_SEARCH_REPLACE", + "collectContextOnly": false, + "responseBehaviour": "USE_STATIC_RESPONSE" + }, + "actions": { + "sourceFile": "../ml-eval/apply-snippet-dataset/evaluation/actions.json", + "evaluationRoots": [ + "." + ] + }, + "interpret": { + "saveLogs": true, + "sessionProbability": 1.0, + "sessionSeed": null, + "sessionsLimit": 5, + "filesLimit": 2, + "order": "LINEAR", + "trainTestSplit": 70, + "registry": "" + }, + "reports": { + "evaluationTitle": "Default", + "sessionsFilters": [], + "comparisonFilters": [] + } +}