From 18fbb9b96b8cb01fa472af517ec646733c5d8057 Mon Sep 17 00:00:00 2001 From: "Nikita.Lyubimov" Date: Fri, 26 Sep 2025 08:13:59 +0300 Subject: [PATCH] [ijnext] Add Self Review Verifier feature and Kotlin evaluation setup GitOrigin-RevId: 6175cebd15577e605ffa66e872d0879b383f44f4 --- .../selfReviewVerifier/kotlin.json | 32 +++++++++++++++++++ .../ultimate_workspace.json_hack | 7 ++++ 2 files changed, 39 insertions(+) create mode 100644 plugins/evaluation-plugin/languages/evaluationconfig/selfReviewVerifier/kotlin.json create mode 100644 plugins/evaluation-plugin/languages/evaluationconfig/selfReviewVerifier/ultimate_workspace.json_hack diff --git a/plugins/evaluation-plugin/languages/evaluationconfig/selfReviewVerifier/kotlin.json b/plugins/evaluation-plugin/languages/evaluationconfig/selfReviewVerifier/kotlin.json new file mode 100644 index 000000000000..139feb3f0443 --- /dev/null +++ b/plugins/evaluation-plugin/languages/evaluationconfig/selfReviewVerifier/kotlin.json @@ -0,0 +1,32 @@ +{ + "language": "Kotlin", + "projectPath": "../ml-eval/ultimate", + "outputDir": "bin/ml-eval-self-review-verifier-kotlin-output", + "strategy": { + "instanceId": "jetbrains__ultimate-1142" + }, + "fileDataset": { + "url": "config:./ultimate_workspace.json_hack" + }, + "actions": { + "evaluationRoots": [ + "." + ] + }, + "interpret": { + "saveLogs": true, + "sessionProbability": 1.0, + "sessionSeed": null, + "sessionsLimit": 1000, + "filesLimit": 1000, + "order": "LINEAR", + "trainTestSplit": 70, + "registry": "" + }, + "reports": { + "evaluationTitle": "Kotlin-Self-Review-Verifier", + "defaultMetrics": null, + "sessionsFilters": [], + "comparisonFilters": [] + } +} diff --git a/plugins/evaluation-plugin/languages/evaluationconfig/selfReviewVerifier/ultimate_workspace.json_hack b/plugins/evaluation-plugin/languages/evaluationconfig/selfReviewVerifier/ultimate_workspace.json_hack new file mode 100644 index 000000000000..716b3305a76a --- /dev/null +++ b/plugins/evaluation-plugin/languages/evaluationconfig/selfReviewVerifier/ultimate_workspace.json_hack @@ -0,0 +1,7 @@ +[ + { + "instance_id": "jetbrains__ultimate-1142", + "patch": "Subject: [PATCH] [workspace model] Create gem manager --- Index: community/platform/platform-resources/src/META-INF/ProjectModelLangExtensions.xml IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 ====================================================================", + "rules": "{\"rules\":[{\"id\":1,\"title\":\"Prefer WorkspaceModel entity queries over legacy managers for data access\",\"text\":\"**Prefer WorkspaceModel entity queries over legacy managers for data access** \\nWorkspaceModel is the single source of truth for project structure. Access entities directly through WorkspaceModel queries rather than legacy managers (ModuleManager, LibraryTable, etc.) when possible. Legacy managers are bridge implementations that eventually delegate to WorkspaceModel anyway. Direct entity access is more efficient, provides better type safety, doesn't require read action for immutable snapshots, and aligns with the modern architecture direction.\\n\\n - **No read action needed**: `currentSnapshot` returns immutable data safe to read from any thread\\n - Prefer: `workspaceModel.currentSnapshot.entities()` over `ModuleManager.getInstance(project).modules`\\n - Prefer: `workspaceModel.currentSnapshot.entities()` over `LibraryTablesRegistrar.getInstance().getLibraryTable()`\\n - Use entity queries for filtering: `entities().filter { it.name.startsWith(\\\"app-\\\") }`\\n - Legacy managers require read action and add overhead - new code should use WorkspaceModel directly\\n\\n **Evidence:**\\n ```kotlin\\n // No read action needed, works from any thread\\n val modules = workspaceModel.currentSnapshot // Immutable, thread-safe\\n .entities()\\n .map { it.name }\\n ```\",\"evidence\":\"kotlin\\n// No read action needed, works from any thread\\nval modules = workspaceModel.currentSnapshot // Immutable, thread-safe\\n .entities()\\n .map { it.name }\"},{\"id\":2,\"title\":\"All workspace mutations MUST use updateProjectModel within write action and descriptive reason\",\"text\":\"**All workspace mutations MUST use updateProjectModel within write action and descriptive reason** \\nWorkspaceModel modifications require both thread safety (write action) and transactional semantics (updateProjectModel). The updateProjectModel method internally asserts write access via ThreadingAssertions, ensuring mutations happen on the correct thread. Direct storage manipulation or using obsolete methods bypasses critical consistency checks, event notifications, and can corrupt the model. Every mutation needs a descriptive reason for debugging and audit trails.\\n\\n - Use `workspaceModel.updateProjectModel(\\\"ComponentName: specific action\\\") { builder -> ... }` for all mutations\\n - Never use updateProjectModelSilent (obsolete) - bypasses notifications and will be removed\\n - Never manipulate storage directly - breaks versioning and event propagation\\n - Descriptive reasons help debugging: \\\"ModuleManager: loading modules from disk\\\"\\n\\n **Evidence:**\\n ```kotlin\\n // Usage - automatically in write action\\n workspaceModel.updateProjectModel(\\\"GradleSync: updating module dependencies\\\") { builder ->\\n val moduleEntity = builder.resolve(ModuleId(moduleName)) ?: return@updateProjectModel\\n builder.modifyEntity(moduleEntity) { \\n this.dependencies = updatedDeps \\n }\\n }\\n ```\",\"evidence\":\"kotlin\\n// Usage - automatically in write action\\nworkspaceModel.updateProjectModel(\\\"GradleSync: updating module dependencies\\\") { builder ->\\n val moduleEntity = builder.resolve(ModuleId(moduleName)) ?: return@updateProjectModel\\n builder.modifyEntity(moduleEntity) { \\n this.dependencies = updatedDeps \\n }\\n}\"},{\"id\":3,\"title\":\"Access entity storage with versioned snapshots, never cache storage references\",\"text\":\"**Access entity storage with versioned snapshots, never cache storage references** \\nThe WorkspaceModel uses immutable versioned snapshots - each `currentSnapshot` returns a frozen view at a \\n-specific version. Caching these references causes stale data bugs because the model updates frequently from multiple \\n-sources (build system sync, user actions, plugins). Always get a fresh snapshot for each read operation to ensure \\n-data consistency.\\n\\n - Read-only access: `workspaceModel.currentSnapshot` - returns current immutable version\\n - Never store snapshot in fields/variables for later use - becomes stale after any update\\n\\n **Evidence:**\\n ```kotlin\\n // WRONG - Caching storage reference\\n class MyService {\\n private val cachedStorage = workspaceModel.currentSnapshot // STALE after any update!\\n \\n fun getModules() = cachedStorage.entities() // Returns outdated data!\\n }\\n \\n // CORRECT - Always get fresh snapshot\\n class MyService {\\n private val workspaceModel = WorkspaceModel.getInstance(project)\\n \\n fun getModules() = workspaceModel.currentSnapshot.entities() // Always current\\n }\\n ``` \",\"evidence\":\"kotlin\\n// WRONG - Caching storage reference\\nclass MyService {\\n private val cachedStorage = workspaceModel.currentSnapshot // STALE after any update!\\n \\n fun getModules() = cachedStorage.entities() // Returns outdated data!\\n}\\n\\n// CORRECT - Always get fresh snapshot\\nclass MyService {\\n private val workspaceModel = WorkspaceModel.getInstance(project)\\n \\n fun getModules() = workspaceModel.currentSnapshot.entities() // Always current\\n}\"},{\"id\":4,\"title\":\"Obtain VirtualFileUrlManager exclusively from workspace model scope\",\"text\":\"**Obtain VirtualFileUrlManager exclusively from workspace model scope** \\nVirtualFileUrlManager maintains an interned cache of VirtualFileUrls for each workspace scope. \\nEach workspace (project or global) has its own URL namespace to ensure proper isolation between project-specific and \\napplication(global)-level resources. Using the wrong manager or creating your own breaks URL equality checks, wastes memory through \\nduplicate URLs, and causes serialization issues. The manager is tied to the workspace lifecycle - caching it separately \\nleads to using outdated instances after workspace updates.\\n\\n - Project scope URLs: `workspaceModel.getVirtualFileUrlManager()` - for project entities\\n - Application(Global) scope URLs: `GlobalWorkspaceModel.getInstance(machine).getVirtualFileUrlManager()` - for SDKs, global libraries\\n - Never instantiate (`new VirtualFileUrlManager()`) or cache managers - breaks URL consistency\\n - URL equality only works within same manager scope - mixing managers breaks comparisons\\n\\n **Evidence:**\\n ```kotlin\\n // WRONG - Creating or caching manager separately\\n val manager = VirtualFileUrlManager() // NEVER create directly!\\n private val cachedManager = workspaceModel.getVirtualFileUrlManager() // Don't cache!\\n \\n // CORRECT - Always get from appropriate scope\\n // For project entities\\n val projectUrl = workspaceModel.getVirtualFileUrlManager().getOrCreateFromUrl(path)\\n \\n // For global/machine entities (SDKs, global libraries)\\n val globalUrl = GlobalWorkspaceModel.getInstance(machine)\\n .getVirtualFileUrlManager()\\n .getOrCreateFromUrl(sdkPath)\\n \\n // URLs from different managers are NOT equal!\\n projectUrl != globalUrl // Even if same path!\\n ```\\n\\n\",\"evidence\":\"kotlin\\n// WRONG - Creating or caching manager separately\\nval manager = VirtualFileUrlManager() // NEVER create directly!\\nprivate val cachedManager = workspaceModel.getVirtualFileUrlManager() // Don't cache!\\n\\n// CORRECT - Always get from appropriate scope\\n// For project entities\\nval projectUrl = workspaceModel.getVirtualFileUrlManager().getOrCreateFromUrl(path)\\n\\n// For global/machine entities (SDKs, global libraries)\\nval globalUrl = GlobalWorkspaceModel.getInstance(machine)\\n .getVirtualFileUrlManager()\\n .getOrCreateFromUrl(sdkPath)\\n\\n// URLs from different managers are NOT equal!\\nprojectUrl != globalUrl // Even if same path!\"},{\"id\":5,\"title\":\"Prefer Flow-based eventLog subscription over WorkspaceModelChangeListener\",\"text\":\"**Prefer Flow-based eventLog subscription over WorkspaceModelChangeListener** \\nThe `eventLog` Flow is the modern, preferred way to observe workspace changes - it provides automatic lifecycle management, runs off EDT, and supports reactive operators. Traditional `WorkspaceModelChangeListener` runs on EDT and can freeze the UI. With eventLog, blocking in collect{} won't freeze UI but will delay processing of subsequent events in your pipeline. Each collect{} is a suspension point - use Flow operators and concurrent patterns to maintain responsiveness.\\n\\n - **Prefer**: `workspaceModel.eventLog.collect { }` over `WorkspaceModelChangeListener`\\n - Flow collectors run OFF EDT automatically - no UI freezing unlike traditional listeners\\n - Spawn separate coroutines for heavy work: `collect { launch { heavyWork() } }` - doesn't block collection\\n\\n **Evidence:**\\n ```kotlin\\n workspaceModel.eventLog.collect { event ->\\n launch(Dispatchers.Default) {\\n processHeavyComputation(event) // Runs async\\n }\\n // Returns immediately, ready for next event\\n }\\n ```\",\"evidence\":\"kotlin\\nworkspaceModel.eventLog.collect { event ->\\n launch(Dispatchers.Default) {\\n processHeavyComputation(event) // Runs async\\n }\\n // Returns immediately, ready for next event\\n}\"},{\"id\":6,\"title\":\"Never store entity references long-term - use EntityPointer for persistent references\",\"text\":\"**Never store entity references long-term - use EntityPointer for persistent references** \\nYou MUST NOT store WorkspaceEntity references in class fields or any persistent data structures that outlive the current operation. Entities are tied to a specific storage version and become invalid after ANY storage mutation. Within a single method or operation, you can freely use entities in local variables and pass them between functions. For any long-term storage (fields, caches, cross-operation references), you MUST use EntityPointer. This is non-obvious because entities look like regular objects but are actually ephemeral views into versioned storage.\\n\\n - **OK**: Use entities in local variables, method parameters, temporary collections within an operation\\n - **NEVER**: Store entities in class fields, persistent caches, or cross-operation data structures\\n - **ALWAYS**: Convert to EntityPointer for any reference that needs to survive beyond current operation\\n - Create pointer: `entity.createPointer()` when you need persistent reference\\n - Entities are valid only within their storage version lifetime (single operation scope)\\n\\n **Evidence:**\\n ```kotlin\\n // WRONG - Never store entity references\\n class MyService {\\n private var moduleEntity: ModuleEntity? = null // NEVER DO THIS!\\n private val entityCache = mutableListOf() // WRONG!\\n private val entityMap = HashMap() // WRONG!\\n }\\n \\n // CORRECT - Always use EntityPointer for storage\\n class MyService {\\n private var modulePointer: EntityPointer? = null // CORRECT\\n private val pointerCache = mutableListOf>() // CORRECT\\n private val pointerMap = HashMap>() // CORRECT\\n \\n fun storeReference(entity: ModuleEntity) {\\n // Convert to pointer immediately\\n modulePointer = entity.createPointer()\\n }\\n \\n fun useReference(storage: EntityStorage) {\\n // Resolve pointer to get current entity\\n val currentEntity = modulePointer?.resolve(storage)\\n currentEntity?.name // Safe - either current or null\\n }\\n }\\n ```\",\"evidence\":\"kotlin\\n// WRONG - Never store entity references\\nclass MyService {\\n private var moduleEntity: ModuleEntity? = null // NEVER DO THIS!\\n private val entityCache = mutableListOf() // WRONG!\\n private val entityMap = HashMap() // WRONG!\\n}\\n\\n// CORRECT - Always use EntityPointer for storage\\nclass MyService {\\n private var modulePointer: EntityPointer? = null // CORRECT\\n private val pointerCache = mutableListOf>() // CORRECT\\n private val pointerMap = HashMap>() // CORRECT\\n \\n fun storeReference(entity: ModuleEntity) {\\n // Convert to pointer immediately\\n modulePointer = entity.createPointer()\\n }\\n \\n fun useReference(storage: EntityStorage) {\\n // Resolve pointer to get current entity\\n val currentEntity = modulePointer?.resolve(storage)\\n currentEntity?.name // Safe - either current or null\\n }\\n}\"}]}" + } +]