MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2025-12-16 14:23:28 +07:00
Code Issues Packages Projects Releases Wiki Activity

LLM-17146 [apply eval] introduce chat-apply eval feature

Browse Source 
(cherry picked from commit 5bbbc88256db36826c97e514031bdb6a9901ce31)

GitOrigin-RevId: c158e106e121ef78c6100d241aed56ae34e6ce00
This commit is contained in:
Dmitry Kozhevnikov
2025-07-14 13:52:03 +03:00
committed by intellij-monorepo-bot
parent 1884609ed6
commit cc233de7ed
2 changed files with 36 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
plugins/evaluation-plugin/core/src/com/intellij/cce/evaluable/AiaConstants.kt
View File

@@ -31,4 +31,6 @@ const val AIA_EXACT_MATCH: String = "exact_match"
const val AIA_AST_MATCH: String = "ast_match" const val AIA_AST_MATCH: String = "ast_match"
const val AIA_PROBLEMS: String = "aia_problems" const val AIA_PROBLEMS: String = "aia_problems"
const val AIA_LLMC_LOG: String = "aia_log_llmc" const val AIA_LLMC_LOG: String = "aia_log_llmc"
const val AIA_HTTP_LOG: String = "aia_log_http" const val AIA_HTTP_LOG: String = "aia_log_http"
const val AIA_CHAT_AND_APPLY_SCORE_KEY: String = "aia_chat_and_apply_score"
const val AIA_APPLY_SCORE_KEY: String = "aia_apply_score"

33
plugins/evaluation-plugin/languages/evaluationconfig/chatcodegeneration/v2/chat_apply.json Normal file
View File

@@ -0,0 +1,33 @@
{
"language": "Another",
"projectPath": "../ml-eval/apply-snippet-dataset",
"repository": "https://git.jetbrains.team/code-generation-evaluation/apply-snippet-eval-experiment.git",
"revision": "6261040e5fc8ff18985816e49a02c7ac81e45eba",
"outputDir": "bin/ml-eval-chat-apply-output",
"strategy": {
"behaviour": "FLASH_SEARCH_REPLACE",
"collectContextOnly": false,
"responseBehaviour": "USE_STATIC_RESPONSE"
},
"actions": {
"sourceFile": "../ml-eval/apply-snippet-dataset/evaluation/actions.json",
"evaluationRoots": [
"."
]
},
"interpret": {
"saveLogs": true,
"sessionProbability": 1.0,
"sessionSeed": null,
"sessionsLimit": 5,
"filesLimit": 2,
"order": "LINEAR",
"trainTestSplit": 70,
"registry": ""
},
"reports": {
"evaluationTitle": "Default",
"sessionsFilters": [],
"comparisonFilters": []
}
}