MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-04-20 13:31:28 +07:00
Code Issues Packages Projects Releases Wiki Activity

[ml-api] JBAI-14600 unify ML API version

Browse Source 
* Switched to the project-level dependency everywhere
* Introduced a new module intellij.platform.ml.logs to separate ml-api library usages from intellij.platform.ml and intellij.platform.ml.impl modules
* The following models has been adapted to the latest ML API version: python imports ranking, fuzzy resolve, completion trigger model, jetenry inline prompt detection, jetenry fleet code classification

Merge-request: IJ-MR-166415
Merged-by: Vladimir Fedorov <890readrid@gmail.com>

(cherry picked from commit 9056efba5a5397a700daabb453ee1477cfaabdf3)

GitOrigin-RevId: 7cfc63d35b55ac968d5a3b789d2ed5f29d4f12d6
This commit is contained in:
Vladimir Fedorov
2025-06-23 18:32:35 +02:00
committed by intellij-monorepo-bot
parent afc2f51f4f
commit 0ac4ba9405
42 changed files with 280 additions and 366 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
.idea/libraries/jetbrains_ml_models_python_imports_ranking_model.xml generated
View File

@@ -1,18 +0,0 @@
<component name="libraryTable">
<library name="jetbrains.ml.models.python.imports.ranking.model" type="repository">
<properties include-transitive-deps="false" maven-id="com.jetbrains.ml.models:python-imports-ranking-model:nebulous-albatross">
<verification>
<artifact url="file://$MAVEN_REPOSITORY$/com/jetbrains/ml/models/python-imports-ranking-model/nebulous-albatross/python-imports-ranking-model-nebulous-albatross.jar">
<sha256sum>5c5e1cc5fae2d8379a586b87ca580c677e65bdc4538ee0e0773cce141dc61b6a</sha256sum>
</artifact>
</verification>
</properties>
<CLASSES>
<root url="jar://$MAVEN_REPOSITORY$/com/jetbrains/ml/models/python-imports-ranking-model/nebulous-albatross/python-imports-ranking-model-nebulous-albatross.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES>
<root url="jar://$MAVEN_REPOSITORY$/com/jetbrains/ml/models/python-imports-ranking-model/nebulous-albatross/python-imports-ranking-model-nebulous-albatross-sources.jar!/" />
</SOURCES>
</library>
</component>

12
.idea/libraries/jetbrains_mlapi_ml_api.xml generated
View File

@@ -1,9 +1,9 @@
<component name="libraryTable">
<library name="jetbrains.mlapi.ml.api" type="repository">
<properties include-transitive-deps="false" maven-id="com.jetbrains.mlapi:ml-api:86">
<properties include-transitive-deps="false" maven-id="com.jetbrains.mlapi:ml-api:0.1.94">
<verification>
<artifact url="file://$MAVEN_REPOSITORY$/com/jetbrains/mlapi/ml-api/86/ml-api-86.jar">
<sha256sum>a52b66fc6cafd0f9f8e295e9748de49903ad662d0d1bebb9d6df60e672e4d4cb</sha256sum>
<artifact url="file://$MAVEN_REPOSITORY$/com/jetbrains/mlapi/ml-api/0.1.94/ml-api-0.1.94.jar">
<sha256sum>1050f9b58c8d71833d075ae7302881cb9d905dcda4025a21c59657ac823ac105</sha256sum>
</artifact>
</verification>
<exclude>
@@ -11,13 +11,13 @@
</exclude>
</properties>
<CLASSES>
<root url="jar://$MAVEN_REPOSITORY$/com/jetbrains/mlapi/ml-api/86/ml-api-86.jar!/" />
<root url="jar://$MAVEN_REPOSITORY$/com/jetbrains/mlapi/ml-api/0.1.94/ml-api-0.1.94.jar!/" />
</CLASSES>
<JAVADOC>
<root url="jar://$MAVEN_REPOSITORY$/com/jetbrains/mlapi/ml-api/86/ml-api-86-javadoc.jar!/" />
<root url="jar://$MAVEN_REPOSITORY$/com/jetbrains/mlapi/ml-api/0.1.94/ml-api-0.1.94-javadoc.jar!/" />
</JAVADOC>
<SOURCES>
<root url="jar://$MAVEN_REPOSITORY$/com/jetbrains/mlapi/ml-api/86/ml-api-86-sources.jar!/" />
<root url="jar://$MAVEN_REPOSITORY$/com/jetbrains/mlapi/ml-api/0.1.94/ml-api-0.1.94-sources.jar!/" />
</SOURCES>
</library>
</component>

12
.idea/libraries/jetbrains_mlapi_ml_tools.xml generated
View File

@@ -1,9 +1,9 @@
<component name="libraryTable">
<library name="jetbrains.mlapi.ml.tools" type="repository">
<properties include-transitive-deps="false" maven-id="com.jetbrains.mlapi:ml-tools:86">
<properties include-transitive-deps="false" maven-id="com.jetbrains.mlapi:ml-tools:0.1.94">
<verification>
<artifact url="file://$MAVEN_REPOSITORY$/com/jetbrains/mlapi/ml-tools/86/ml-tools-86.jar">
<sha256sum>1fd34e39c77dad83957882a7b6ccf99c1d49403b4643efc85a4b041aa1108a13</sha256sum>
<artifact url="file://$MAVEN_REPOSITORY$/com/jetbrains/mlapi/ml-tools/0.1.94/ml-tools-0.1.94.jar">
<sha256sum>b2d36677ba73f2f7974199823b5f4323b2a16ea718ed506297402c6a7605af1c</sha256sum>
</artifact>
</verification>
<exclude>
@@ -11,13 +11,13 @@
</exclude>
</properties>
<CLASSES>
<root url="jar://$MAVEN_REPOSITORY$/com/jetbrains/mlapi/ml-tools/86/ml-tools-86.jar!/" />
<root url="jar://$MAVEN_REPOSITORY$/com/jetbrains/mlapi/ml-tools/0.1.94/ml-tools-0.1.94.jar!/" />
</CLASSES>
<JAVADOC>
<root url="jar://$MAVEN_REPOSITORY$/com/jetbrains/mlapi/ml-tools/86/ml-tools-86-javadoc.jar!/" />
<root url="jar://$MAVEN_REPOSITORY$/com/jetbrains/mlapi/ml-tools/0.1.94/ml-tools-0.1.94-javadoc.jar!/" />
</JAVADOC>
<SOURCES>
<root url="jar://$MAVEN_REPOSITORY$/com/jetbrains/mlapi/ml-tools/86/ml-tools-86-sources.jar!/" />
<root url="jar://$MAVEN_REPOSITORY$/com/jetbrains/mlapi/ml-tools/0.1.94/ml-tools-0.1.94-sources.jar!/" />
</SOURCES>
</library>
</component>

20
.idea/libraries/jetbrains_mlapi_ml_tools_suspendable.xml generated
View File

@@ -1,20 +0,0 @@
<component name="libraryTable">
<library name="jetbrains.mlapi.ml.tools.suspendable" type="repository">
<properties include-transitive-deps="false" maven-id="com.jetbrains.mlapi:ml-tools-suspendable:76">
<verification>
<artifact url="file://$MAVEN_REPOSITORY$/com/jetbrains/mlapi/ml-tools-suspendable/76/ml-tools-suspendable-76.jar">
<sha256sum>98be4a00226838f7afc9dd054ee40ed03e1379672b343258480b8b72f1b25e0f</sha256sum>
</artifact>
</verification>
</properties>
<CLASSES>
<root url="jar://$MAVEN_REPOSITORY$/com/jetbrains/mlapi/ml-tools-suspendable/76/ml-tools-suspendable-76.jar!/" />
</CLASSES>
<JAVADOC>
<root url="jar://$MAVEN_REPOSITORY$/com/jetbrains/mlapi/ml-tools-suspendable/76/ml-tools-suspendable-76-javadoc.jar!/" />
</JAVADOC>
<SOURCES>
<root url="jar://$MAVEN_REPOSITORY$/com/jetbrains/mlapi/ml-tools-suspendable/76/ml-tools-suspendable-76-sources.jar!/" />
</SOURCES>
</library>
</component>

1
.idea/modules.xml generated
View File

@@ -872,6 +872,7 @@
<module fileurl="file://$PROJECT_DIR$/platform/markdown-utils/intellij.platform.markdown.utils.iml" filepath="$PROJECT_DIR$/platform/markdown-utils/intellij.platform.markdown.utils.iml" />
<module fileurl="file://$PROJECT_DIR$/platform/ml-api/intellij.platform.ml.iml" filepath="$PROJECT_DIR$/platform/ml-api/intellij.platform.ml.iml" />
<module fileurl="file://$PROJECT_DIR$/platform/ml-impl/intellij.platform.ml.impl.iml" filepath="$PROJECT_DIR$/platform/ml-impl/intellij.platform.ml.impl.iml" />
<module fileurl="file://$PROJECT_DIR$/platform/ml-logs/intellij.platform.ml.logs.iml" filepath="$PROJECT_DIR$/platform/ml-logs/intellij.platform.ml.logs.iml" />
<module fileurl="file://$PROJECT_DIR$/platform/monolith/intellij.platform.monolith.iml" filepath="$PROJECT_DIR$/platform/monolith/intellij.platform.monolith.iml" />
<module fileurl="file://$PROJECT_DIR$/platform/main/intellij.platform.monolith.main/intellij.platform.monolith.main.iml" filepath="$PROJECT_DIR$/platform/main/intellij.platform.monolith.main/intellij.platform.monolith.main.iml" />
<module fileurl="file://$PROJECT_DIR$/platform/navbar/shared/intellij.platform.navbar.iml" filepath="$PROJECT_DIR$/platform/navbar/shared/intellij.platform.navbar.iml" />

29
lib/BUILD.bazel
View File

@@ -2839,13 +2839,6 @@ jvm_import(
visibility = ["//visibility:public"]
)
jvm_import(
name = "jetbrains-ml-models-python-imports-ranking-model",
jar = "@python-imports-ranking-model-nebulous-albatross_http//file",
source_jar = "@python-imports-ranking-model-nebulous-albatross-sources_http//file",
visibility = ["//visibility:public"]
)
jvm_import(
name = "jetbrains-mlapi-catboost-shadow-need-slf4j",
jar = "@catboost-shadow-need-slf4j-1_2_5_http//file",
@@ -2854,22 +2847,15 @@ jvm_import(
jvm_import(
name = "jetbrains-mlapi-ml-api",
jar = "@ml-api-86_http//file",
source_jar = "@ml-api-86-sources_http//file",
jar = "@ml-api-0_1_94_http//file",
source_jar = "@ml-api-0_1_94-sources_http//file",
visibility = ["//visibility:public"]
)
jvm_import(
name = "jetbrains-mlapi-ml-tools",
jar = "@ml-tools-86_http//file",
source_jar = "@ml-tools-86-sources_http//file",
visibility = ["//visibility:public"]
)
jvm_import(
name = "jetbrains-mlapi-ml-tools-suspendable",
jar = "@ml-tools-suspendable-76_http//file",
source_jar = "@ml-tools-suspendable-76-sources_http//file",
jar = "@ml-tools-0_1_94_http//file",
source_jar = "@ml-tools-0_1_94-sources_http//file",
visibility = ["//visibility:public"]
)
@@ -8587,6 +8573,13 @@ jvm_import(
source_jar = "@byte-buddy-agent-1_17_4-sources_http//file"
)
jvm_import(
name = "python-ml-features-jetbrains-ml-models-python-imports-ranking-model",
jar = "@python-imports-ranking-model-coral-panda-republished-4_http//file",
source_jar = "@python-imports-ranking-model-coral-panda-republished-4-sources_http//file",
visibility = ["//visibility:public"]
)
jvm_import(
name = "rd-core",
jar = "@rd-core-2025_2_2_http//file",

74
lib/MODULE.bazel
View File

@@ -3897,20 +3897,6 @@ http_file(
downloaded_file_path = "markdown-jvm-0.7.2-sources.jar"
)
http_file(
name = "python-imports-ranking-model-nebulous-albatross_http",
url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/ml/models/python-imports-ranking-model/nebulous-albatross/python-imports-ranking-model-nebulous-albatross.jar",
sha256 = "5c5e1cc5fae2d8379a586b87ca580c677e65bdc4538ee0e0773cce141dc61b6a",
downloaded_file_path = "python-imports-ranking-model-nebulous-albatross.jar"
)
http_file(
name = "python-imports-ranking-model-nebulous-albatross-sources_http",
url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/ml/models/python-imports-ranking-model/nebulous-albatross/python-imports-ranking-model-nebulous-albatross-sources.jar",
sha256 = "45f0027b5a3ed622fb77830f7f98caaa53aed34ba3a8c7d32c85c0c39223d612",
downloaded_file_path = "python-imports-ranking-model-nebulous-albatross-sources.jar"
)
http_file(
name = "catboost-shadow-need-slf4j-1_2_5_http",
url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/mlapi/catboost-shadow-need-slf4j/1.2.5/catboost-shadow-need-slf4j-1.2.5.jar",
@@ -3919,45 +3905,31 @@ http_file(
)
http_file(
name = "ml-api-86_http",
url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/mlapi/ml-api/86/ml-api-86.jar",
sha256 = "a52b66fc6cafd0f9f8e295e9748de49903ad662d0d1bebb9d6df60e672e4d4cb",
downloaded_file_path = "ml-api-86.jar"
name = "ml-api-0_1_94_http",
url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/mlapi/ml-api/0.1.94/ml-api-0.1.94.jar",
sha256 = "1050f9b58c8d71833d075ae7302881cb9d905dcda4025a21c59657ac823ac105",
downloaded_file_path = "ml-api-0.1.94.jar"
)
http_file(
name = "ml-api-86-sources_http",
url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/mlapi/ml-api/86/ml-api-86-sources.jar",
sha256 = "71001504fe5e7f21d83b717fa689bf6a2f3818428626eb4c0ff09eef688e03f6",
downloaded_file_path = "ml-api-86-sources.jar"
name = "ml-api-0_1_94-sources_http",
url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/mlapi/ml-api/0.1.94/ml-api-0.1.94-sources.jar",
sha256 = "79eb26365ce9f536733d96fc16e40e7e89e1daba9312a8d06449718430d7d499",
downloaded_file_path = "ml-api-0.1.94-sources.jar"
)
http_file(
name = "ml-tools-86_http",
url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/mlapi/ml-tools/86/ml-tools-86.jar",
sha256 = "1fd34e39c77dad83957882a7b6ccf99c1d49403b4643efc85a4b041aa1108a13",
downloaded_file_path = "ml-tools-86.jar"
name = "ml-tools-0_1_94_http",
url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/mlapi/ml-tools/0.1.94/ml-tools-0.1.94.jar",
sha256 = "b2d36677ba73f2f7974199823b5f4323b2a16ea718ed506297402c6a7605af1c",
downloaded_file_path = "ml-tools-0.1.94.jar"
)
http_file(
name = "ml-tools-86-sources_http",
url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/mlapi/ml-tools/86/ml-tools-86-sources.jar",
sha256 = "3780b7e4c6fad6945812f4729b661d54acc85068fd7a4b106e972bbd7d4dd0b1",
downloaded_file_path = "ml-tools-86-sources.jar"
)
http_file(
name = "ml-tools-suspendable-76_http",
url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/mlapi/ml-tools-suspendable/76/ml-tools-suspendable-76.jar",
sha256 = "98be4a00226838f7afc9dd054ee40ed03e1379672b343258480b8b72f1b25e0f",
downloaded_file_path = "ml-tools-suspendable-76.jar"
)
http_file(
name = "ml-tools-suspendable-76-sources_http",
url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/mlapi/ml-tools-suspendable/76/ml-tools-suspendable-76-sources.jar",
sha256 = "11b562e584a7b01596dc9fd14ca7c710a27f48c242b597a90af9ff32f663ace8",
downloaded_file_path = "ml-tools-suspendable-76-sources.jar"
name = "ml-tools-0_1_94-sources_http",
url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/mlapi/ml-tools/0.1.94/ml-tools-0.1.94-sources.jar",
sha256 = "8c5b021724a9b8dab25e33b09812aadbd160cf8e64e89a3dd17c40449dbdedae",
downloaded_file_path = "ml-tools-0.1.94-sources.jar"
)
http_file(
@@ -12325,6 +12297,20 @@ http_file(
downloaded_file_path = "byte-buddy-agent-1.17.4-sources.jar"
)
http_file(
name = "python-imports-ranking-model-coral-panda-republished-4_http",
url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/ml/models/python-imports-ranking-model/coral-panda-republished-4/python-imports-ranking-model-coral-panda-republished-4.jar",
sha256 = "cff0591184b48356a622f7635c6de1399fdbeb161f98d68206d09807c83fa98c",
downloaded_file_path = "python-imports-ranking-model-coral-panda-republished-4.jar"
)
http_file(
name = "python-imports-ranking-model-coral-panda-republished-4-sources_http",
url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/ml/models/python-imports-ranking-model/coral-panda-republished-4/python-imports-ranking-model-coral-panda-republished-4-sources.jar",
sha256 = "4aecfba9498dff7e199c7db2a718b53c77834347daf8b96cafc8b02b392c5e6c",
downloaded_file_path = "python-imports-ranking-model-coral-panda-republished-4-sources.jar"
)
http_file(
name = "rd-core-2025_2_2_http",
url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/com/jetbrains/rd/rd-core/2025.2.2/rd-core-2025.2.2.jar",

7
platform/build-scripts/src/org/jetbrains/intellij/build/CommunityLibraryLicenses.kt
View File

@@ -1381,20 +1381,15 @@ object CommunityLibraryLicenses {
jetbrainsLibrary("intellij.remoterobot.remote.fixtures"),
jetbrainsLibrary("intellij.remoterobot.robot.server.core"),
jetbrainsLibrary("jetbrains.compose.hot.reload.gradle.idea"),
jetbrainsLibrary("jetbrains.fullLine.mlapi.ml.api"),
jetbrainsLibrary("jetbrains.fullLine.mlapi.ml.tools"),
jetbrainsLibrary("jetbrains.fus.reporting.ap.validation"),
jetbrainsLibrary("jetbrains.intellij.deps.rwmutex.idea"),
jetbrainsLibrary("jetbrains.kotlin.compose.compiler.plugin"),
jetbrainsLibrary("jetbrains.llm.mlapi.catboost.shadow.need.slf4j"),
jetbrainsLibrary("jetbrains.llm.mlapi.ml.api"),
jetbrainsLibrary("jetbrains.llm.mlapi.ml.tools"),
jetbrainsLibrary("jetbrains.ml.completion.trigger_model_kotlin_cloud"),
jetbrainsLibrary("jetbrains.ml.models.jetenry.inline.prompt.detection.model"),
jetbrainsLibrary("jetbrains.ml.models.python.imports.ranking.model"),
jetbrainsLibrary("jetbrains.mlapi.catboost.shadow.need.slf4j"),
jetbrainsLibrary("jetbrains.mlapi.ml.api"),
jetbrainsLibrary("jetbrains.mlapi.ml.tools"),
jetbrainsLibrary("jetbrains.mlapi.ml.tools.suspendable"),
jetbrainsLibrary("jshell-frontend"),
jetbrainsLibrary("jvm-native-trusted-roots"),
jetbrainsLibrary("kotlin-gradle-plugin-idea"),

2
platform/ml-api/BUILD.bazel
View File

@@ -18,9 +18,7 @@ jvm_library(
"//platform/core-api:core",
"//platform/util",
"@lib//:jetbrains-annotations",
"@lib//:jetbrains-mlapi-ml-api",
],
exports = ["@lib//:jetbrains-mlapi-ml-api"],
runtime_deps = [":ml_resources"]
)
### auto-generated section `build intellij.platform.ml` end

1
platform/ml-api/intellij.platform.ml.iml
View File

@@ -13,6 +13,5 @@
<orderEntry type="module" module-name="intellij.platform.core" />
<orderEntry type="module" module-name="intellij.platform.util" />
<orderEntry type="library" name="jetbrains-annotations" level="project" />
<orderEntry type="library" exported="" name="jetbrains.mlapi.ml.api" level="project" />
</component>
</module>

13
platform/ml-impl/BUILD.bazel
View File

@@ -35,17 +35,9 @@ jvm_library(
"//platform/ml-api:ml",
"//platform/statistics",
"@lib//:kotlin-reflect",
"@lib//:jetbrains-mlapi-catboost-shadow-need-slf4j",
"//platform/ide-core-impl",
"@lib//:jetbrains-mlapi-ml-tools",
"@lib//:jetbrains-mlapi-ml-tools-suspendable",
],
exports = [
"@lib//:ngram-slp",
"@lib//:jetbrains-mlapi-catboost-shadow-need-slf4j",
"@lib//:jetbrains-mlapi-ml-tools",
"@lib//:jetbrains-mlapi-ml-tools-suspendable",
],
exports = ["@lib//:ngram-slp"],
runtime_deps = [
":ml-impl_resources",
"//platform/backend",
@@ -80,10 +72,7 @@ jvm_library(
"//platform/statistics",
"//platform/statistics:statistics_test_lib",
"@lib//:kotlin-reflect",
"@lib//:jetbrains-mlapi-catboost-shadow-need-slf4j",
"//platform/ide-core-impl",
"@lib//:jetbrains-mlapi-ml-tools",
"@lib//:jetbrains-mlapi-ml-tools-suspendable",
"//platform/statistics/testFramework",
],
runtime_deps = [

3
platform/ml-impl/intellij.platform.ml.impl.iml
View File

@@ -52,10 +52,7 @@
<orderEntry type="module" module-name="intellij.platform.ml" />
<orderEntry type="module" module-name="intellij.platform.statistics" />
<orderEntry type="library" name="kotlin-reflect" level="project" />
<orderEntry type="library" exported="" name="jetbrains.mlapi.catboost.shadow.need.slf4j" level="project" />
<orderEntry type="module" module-name="intellij.platform.ide.core.impl" />
<orderEntry type="library" exported="" name="jetbrains.mlapi.ml.tools" level="project" />
<orderEntry type="library" exported="" name="jetbrains.mlapi.ml.tools.suspendable" level="project" />
<orderEntry type="module" module-name="intellij.platform.statistics.testFramework" scope="TEST" />
</component>
</module>

15
platform/ml-impl/src/com/intellij/platform/ml/impl/fus/eventFields.kt
View File

@@ -1,15 +0,0 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.platform.ml.impl.fus
import com.intellij.internal.statistic.eventLog.events.EventFields
import com.intellij.lang.Language
import com.intellij.openapi.util.Version
import com.jetbrains.ml.api.logs.RuleBasedStringEventField
import org.jetbrains.annotations.ApiStatus
@ApiStatus.Internal
class VersionEventField(name: String, lazyDescription: () -> String) : RuleBasedStringEventField<Version>(name, EventFields.Version.validationRule.first(), lazyDescription, { it.toCompactString() })
@ApiStatus.Internal
class LanguageEventField(name: String, lazyDescription: () -> String) : RuleBasedStringEventField<Language>(name, EventFields.Language.validationRule.first(), lazyDescription, { it.id })

30
platform/ml-logs/BUILD.bazel Normal file
View File

@@ -0,0 +1,30 @@
### auto-generated section `build intellij.platform.ml.logs` start
load("@rules_jvm//:jvm.bzl", "jvm_library", "jvm_resources")
jvm_resources(
name = "ml-logs_resources",
files = glob(["resources/**/*"]),
strip_prefix = "resources"
)
jvm_library(
name = "ml-logs",
module_name = "intellij.platform.ml.logs",
visibility = ["//visibility:public"],
srcs = glob(["src/**/*.kt", "src/**/*.java"], allow_empty = True),
deps = [
"@lib//:kotlin-stdlib",
"//platform/statistics",
"@lib//:jetbrains-annotations",
"@lib//:jetbrains-mlapi-catboost-shadow-need-slf4j",
"@lib//:jetbrains-mlapi-ml-api",
"@lib//:jetbrains-mlapi-ml-tools",
],
exports = [
"@lib//:jetbrains-mlapi-catboost-shadow-need-slf4j",
"@lib//:jetbrains-mlapi-ml-api",
"@lib//:jetbrains-mlapi-ml-tools",
],
runtime_deps = [":ml-logs_resources"]
)
### auto-generated section `build intellij.platform.ml.logs` end

0
platform/ml-logs/api-dump.txt Normal file
View File

18
platform/ml-logs/intellij.platform.ml.logs.iml Normal file
View File

@@ -0,0 +1,18 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/resources" type="java-resource" />
<sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="kotlin-stdlib" level="project" />
<orderEntry type="module" module-name="intellij.platform.statistics" />
<orderEntry type="library" name="jetbrains-annotations" level="project" />
<orderEntry type="library" exported="" name="jetbrains.mlapi.catboost.shadow.need.slf4j" level="project" />
<orderEntry type="library" exported="" name="jetbrains.mlapi.ml.api" level="project" />
<orderEntry type="library" exported="" name="jetbrains.mlapi.ml.tools" level="project" />
</component>
</module>

5
platform/ml-logs/resources/intellij.platform.ml.logs.xml Normal file
View File

@@ -0,0 +1,5 @@
<idea-plugin package="com.intellij.platform.ml.logs">
<dependencies>
<module name="intellij.platform.backend"/>
</dependencies>
</idea-plugin>

85
platform/ml-impl/src/com/intellij/platform/ml/impl/fus/IJEventPairConverter.kt → platform/ml-logs/src/com/intellij/platform/ml/logs/IJEventPairConverter.kt
View File

@@ -1,22 +1,16 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.platform.ml.impl.fus
package com.intellij.platform.ml.logs
import com.intellij.internal.statistic.eventLog.FeatureUsageData
import com.intellij.internal.statistic.eventLog.events.EventPair
import com.intellij.internal.statistic.eventLog.events.PrimitiveEventField
import com.intellij.internal.statistic.eventLog.events.StringEventField
import com.intellij.lang.Language
import com.intellij.openapi.util.Version
import com.intellij.platform.ml.impl.fus.ConverterObjectDescription.Companion.asIJObjectDescription
import com.intellij.platform.ml.impl.fus.ConverterOfEnum.Companion.toIJConverter
import com.intellij.platform.ml.impl.fus.IJEventPairConverter.Companion.typedBuild
import com.jetbrains.ml.api.logs.RuleBasedStringEventField
import com.intellij.platform.ml.logs.ConverterObjectDescription.Companion.asIJObjectDescription
import com.intellij.platform.ml.logs.ConverterOfEnum.Companion.toIJConverter
import com.intellij.platform.ml.logs.IJEventPairConverter.Companion.typedBuild
import com.intellij.internal.statistic.eventLog.events.BooleanEventField as IJBooleanEventField
import com.intellij.internal.statistic.eventLog.events.ClassEventField as IJClassEventField
import com.intellij.internal.statistic.eventLog.events.DoubleEventField as IJDoubleEventField
import com.intellij.internal.statistic.eventLog.events.EnumEventField as IJEnumEventField
import com.intellij.internal.statistic.eventLog.events.EventField as IJEventField
import com.intellij.internal.statistic.eventLog.events.EventFields as IJEventFields
import com.intellij.internal.statistic.eventLog.events.EventPair as IJEventPair
import com.intellij.internal.statistic.eventLog.events.FloatEventField as IJFloatEventField1
import com.intellij.internal.statistic.eventLog.events.FloatListEventField as IJFloatListEventField
@@ -28,7 +22,6 @@ import com.intellij.internal.statistic.eventLog.events.ObjectDescription as IJOb
import com.intellij.internal.statistic.eventLog.events.ObjectEventData as IJObjectEventData
import com.intellij.internal.statistic.eventLog.events.ObjectEventField as IJObjectEventField
import com.intellij.internal.statistic.eventLog.events.ObjectListEventField as IJObjectListEventField
import com.intellij.internal.statistic.eventLog.events.StringEventField as IJStringEventField
import com.jetbrains.ml.api.logs.BooleanEventField as MLBooleanEventField
import com.jetbrains.ml.api.logs.ClassEventField as MLClassEventField
import com.jetbrains.ml.api.logs.DoubleEventField as MLDoubleEventField
@@ -87,21 +80,18 @@ internal class ConverterObjectDescription(mlObjectDescription: MLObjectDescripti
is MLDoubleEventField -> ConverterOfPrimitiveType(mlEventField) { n, d -> IJDoubleEventField(n, d) } as IJEventPairConverter<L, *>
is MLStringEventField -> ConverterOfString(mlEventField) as IJEventPairConverter<L, *>
is MLFloatListEventField -> object : IJEventPairConverter<List<Float>, List<Float>> {
override val ijEventField: IJEventField<List<Float>> = IJFloatListEventField(mlEventField.name, mlEventField.lazyDescription())
override val ijEventField: IJEventField<List<Float>> = IJFloatListEventField(mlEventField.name, mlEventField.lazyDescription?.invoke())
override fun buildEventPair(mlEventPair: MLEventPair<List<Float>>): IJEventPair<List<Float>> = ijEventField with mlEventPair.data
} as IJEventPairConverter<L, *>
is MLIntListEventField -> object : IJEventPairConverter<List<Int>, List<Int>> {
override val ijEventField: IJEventField<List<Int>> = IJIntListEventField(mlEventField.name, mlEventField.lazyDescription())
override val ijEventField: IJEventField<List<Int>> = IJIntListEventField(mlEventField.name, mlEventField.lazyDescription?.invoke())
override fun buildEventPair(mlEventPair: MLEventPair<List<Int>>): IJEventPair<List<Int>> = ijEventField with mlEventPair.data
} as IJEventPairConverter<L, *>
is MLLongListEventField -> object : IJEventPairConverter<List<Long>, List<Long>> {
override val ijEventField: IJEventField<List<Long>> = IJLongListEventField(mlEventField.name, mlEventField.lazyDescription())
override val ijEventField: IJEventField<List<Long>> = IJLongListEventField(mlEventField.name, mlEventField.lazyDescription?.invoke())
override fun buildEventPair(mlEventPair: MLEventPair<List<Long>>): IJEventPair<List<Long>> = ijEventField with mlEventPair.data
} as IJEventPairConverter<L, *>
is VersionEventField -> ConverterOfVersion(mlEventField) as IJEventPairConverter<L, *>
is LanguageEventField -> ConverterOfLanguage(mlEventField) as IJEventPairConverter<L, *>
is RuleBasedStringEventField<L> -> ConverterSensitiveString(mlEventField) as IJEventPairConverter<L, *>
else -> throw NotImplementedError("Please implement converter for $mlEventField from ML to IJ event fields")
}
@@ -131,45 +121,6 @@ internal class ConverterObjectDescription(mlObjectDescription: MLObjectDescripti
private class ConverterOfString(mlEventField: MLStringEventField) : IJEventPairConverter<String, String?> {
override val ijEventField: IJEventField<String?> = IJStringEventField.ValidatedByAllowedValues(
mlEventField.name,
allowedValues = mlEventField.possibleValues,
description = mlEventField.lazyDescription()
)
override fun buildEventPair(mlEventPair: MLEventPair<String>): IJEventPair<String?> {
return ijEventField with mlEventPair.data
}
}
private class ConverterOfLanguage(mlEventField: LanguageEventField) : IJEventPairConverter<Language, Language?> {
override val ijEventField: IJEventField<Language?> = IJEventFields.Language(mlEventField.name, mlEventField.lazyDescription())
override fun buildEventPair(mlEventPair: MLEventPair<Language>): IJEventPair<Language?> {
return ijEventField with mlEventPair.data
}
}
private class ConverterOfVersion(mlEventField: com.intellij.platform.ml.impl.fus.VersionEventField) : IJEventPairConverter<Version, Version?> {
private class VersionEventField(override val name: String, override val description: String?) : PrimitiveEventField<Version?>() {
override val validationRule: List<String>
get() = listOf("{regexp#version}")
override fun addData(fuData: FeatureUsageData, value: Version?) {
fuData.addVersion(value)
}
}
override val ijEventField: IJEventField<Version?> = VersionEventField(mlEventField.name, mlEventField.lazyDescription())
override fun buildEventPair(mlEventPair: MLEventPair<Version>): IJEventPair<Version?> {
return ijEventField with mlEventPair.data
}
}
private class ConverterSensitiveString<T>(private val mlEventField: RuleBasedStringEventField<T>) : IJEventPairConverter<T, String?> {
private class SensitiveStringEventField(
name: String,
@@ -180,11 +131,15 @@ private class ConverterSensitiveString<T>(private val mlEventField: RuleBasedStr
}
override val ijEventField: IJEventField<String?> =
SensitiveStringEventField(mlEventField.name, mlEventField.ruleId!!, mlEventField.lazyDescription())
SensitiveStringEventField(
mlEventField.name,
requireNotNull(mlEventField.ruleId) { "Error for $mlEventField: it must have a validation rule" },
mlEventField.lazyDescription?.invoke()
)
override fun buildEventPair(mlEventPair: MLEventPair<String>): EventPair<String?> =
ijEventField with mlEventPair.data
override fun buildEventPair(mlEventPair: MLEventPair<T>): EventPair<String?> {
return ijEventField with mlEventField.serialize(mlEventPair.data)
}
}
@@ -207,7 +162,7 @@ private class ConvertObjectList(mlEventField: MLObjectListEventField) :
private class ConverterOfEnum<T : Enum<*>>(mlEnumField: MLEnumEventField<T>) : IJEventPairConverter<T, T> {
override val ijEventField: IJEventField<T> = IJEnumEventField(mlEnumField.name, mlEnumField.enumClass, mlEnumField.transform)
override val ijEventField: IJEventField<T> = IJEnumEventField(mlEnumField.name, mlEnumField.enumClass) { it.name }
override fun buildEventPair(mlEventPair: MLEventPair<T>): IJEventPair<T> {
return ijEventField with mlEventPair.data
@@ -223,12 +178,12 @@ private class ConverterOfEnum<T : Enum<*>>(mlEnumField: MLEnumEventField<T>) : I
private class ConverterOfObject(
name: String,
lazyDescription: () -> String,
lazyDescription: (() -> String)?,
mlObjectDescription: MLObjectDescription,
) : IJEventPairConverter<MLObjectEventData, IJObjectEventData> {
val ijObjectDescription = mlObjectDescription.asIJObjectDescription()
override val ijEventField: IJEventField<IJObjectEventData> = IJObjectEventField(name, lazyDescription(), ijObjectDescription)
override val ijEventField: IJEventField<IJObjectEventData> = IJObjectEventField(name, lazyDescription?.invoke(), ijObjectDescription)
fun buildObjectEventData(mlObject: MLObjectEventData): IJObjectEventData {
return ijObjectDescription.buildObjectEventData(mlObject)
@@ -244,7 +199,7 @@ private class ConverterOfPrimitiveType<T>(
mlEventField: MLEventField<T>,
createIJField: (String, String?) -> IJEventField<T>
) : IJEventPairConverter<T, T> {
override val ijEventField: IJEventField<T> = createIJField(mlEventField.name, mlEventField.lazyDescription())
override val ijEventField: IJEventField<T> = createIJField(mlEventField.name, mlEventField.lazyDescription?.invoke())
override fun buildEventPair(mlEventPair: MLEventPair<T>): IJEventPair<T> {
return ijEventField with mlEventPair.data
@@ -255,7 +210,7 @@ private class ConverterOfPrimitiveType<T>(
private class ConverterOfClass(
mlEventField: MLClassEventField,
) : IJEventPairConverter<Class<*>, Class<*>?> {
override val ijEventField: IJEventField<Class<*>?> = IJClassEventField(mlEventField.name, mlEventField.lazyDescription())
override val ijEventField: IJEventField<Class<*>?> = IJClassEventField(mlEventField.name, mlEventField.lazyDescription?.invoke())
override fun buildEventPair(mlEventPair: MLEventPair<Class<*>>): IJEventPair<Class<*>?> {
return ijEventField with mlEventPair.data

2
platform/ml-impl/src/com/intellij/platform/ml/impl/fus/IntelliJFusEventRegister.kt → platform/ml-logs/src/com/intellij/platform/ml/logs/IntelliJFusEventRegister.kt
View File

@@ -1,5 +1,5 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.platform.ml.impl.fus
package com.intellij.platform.ml.logs
import com.intellij.internal.statistic.eventLog.events.VarargEventId
import com.jetbrains.ml.tools.logs.FusEventLogger

1
platform/platform-resources/src/META-INF/common-ide-modules.xml
View File

@@ -10,6 +10,7 @@
<module name="intellij.platform.tasks"/>
<module name="intellij.platform.tasks.impl"/>
<module name="intellij.platform.ml.impl"/>
<module name="intellij.platform.ml.logs"/>
<module name="intellij.libraries.microba"/>
<module name="intellij.libraries.skiko"/>
<module name="intellij.platform.compose"/>

8
python/intellij.python.ml.features/BUILD.bazel
View File

@@ -16,17 +16,19 @@ jvm_library(
"@lib//:kotlin-stdlib",
"//platform/core-api:core",
"//platform/editor-ui-api:editor-ui",
"//platform/analysis-api:analysis",
"//platform/indexing-api:indexing",
"//platform/statistics",
"//platform/ml-impl",
"//platform/projectModel-api:projectModel",
"//platform/ml-logs",
"//python/python-psi-impl:psi-impl",
"//platform/platform-api:ide",
"//platform/platform-impl:ide-impl",
"//python:python-community-impl",
"//platform/util/concurrency",
"//platform/core-ui",
"//platform/ml-api:ml",
"//platform/platform-impl/codeinsight-inline:ide-codeinsight-inline",
"@lib//:jetbrains-ml-models-python-imports-ranking-model",
"@lib//:python-ml-features-jetbrains-ml-models-python-imports-ranking-model",
],
runtime_deps = [":ml-features_resources"]
)

25
python/intellij.python.ml.features/intellij.python.ml.features.iml
View File

@@ -11,16 +11,35 @@
<orderEntry type="library" name="kotlin-stdlib" level="project" />
<orderEntry type="module" module-name="intellij.platform.core" />
<orderEntry type="module" module-name="intellij.platform.editor.ui" />
<orderEntry type="module" module-name="intellij.platform.analysis" />
<orderEntry type="module" module-name="intellij.platform.indexing" />
<orderEntry type="module" module-name="intellij.platform.statistics" />
<orderEntry type="module" module-name="intellij.platform.ml.impl" />
<orderEntry type="module" module-name="intellij.platform.projectModel" />
<orderEntry type="module" module-name="intellij.platform.ml.logs" />
<orderEntry type="module" module-name="intellij.python.psi.impl" />
<orderEntry type="module" module-name="intellij.platform.ide" />
<orderEntry type="module" module-name="intellij.platform.ide.impl" />
<orderEntry type="module" module-name="intellij.python.community.impl" />
<orderEntry type="module" module-name="intellij.platform.concurrency" />
<orderEntry type="module" module-name="intellij.platform.core.ui" />
<orderEntry type="module" module-name="intellij.platform.ml" />
<orderEntry type="module" module-name="intellij.platform.ide.codeinsight.inline" />
<orderEntry type="library" name="jetbrains.ml.models.python.imports.ranking.model" level="project" />
<orderEntry type="module-library">
<library name="jetbrains.ml.models.python.imports.ranking.model" type="repository">
<properties include-transitive-deps="false" maven-id="com.jetbrains.ml.models:python-imports-ranking-model:coral-panda-republished-4">
<verification>
<artifact url="file://$MAVEN_REPOSITORY$/com/jetbrains/ml/models/python-imports-ranking-model/coral-panda-republished-4/python-imports-ranking-model-coral-panda-republished-4.jar">
<sha256sum>cff0591184b48356a622f7635c6de1399fdbeb161f98d68206d09807c83fa98c</sha256sum>
</artifact>
</verification>
</properties>
<CLASSES>
<root url="jar://$MAVEN_REPOSITORY$/com/jetbrains/ml/models/python-imports-ranking-model/coral-panda-republished-4/python-imports-ranking-model-coral-panda-republished-4.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES>
<root url="jar://$MAVEN_REPOSITORY$/com/jetbrains/ml/models/python-imports-ranking-model/coral-panda-republished-4/python-imports-ranking-model-coral-panda-republished-4-sources.jar!/" />
</SOURCES>
</library>
</orderEntry>
</component>
</module>

6
python/intellij.python.ml.features/resources/intellij.python.ml.features.xml
View File

@@ -1,13 +1,15 @@
<idea-plugin package="com.intellij.python.ml.features">
<dependencies>
<module name="intellij.platform.ml.impl"/>
<module name="intellij.platform.ml.logs"/>
<module name="intellij.python.psi.impl"/>
</dependencies>
<extensions defaultExtensionNs="com.intellij">
<statistics.counterUsagesCollector implementationClass="com.intellij.python.ml.features.imports.PyCharmImportsRankingLogs"/>
<registryKey
key="quickfix.ranking.ml"
defaultValue="[DISABLED*]"
defaultValue="[IN_EXPERIMENT|ENABLED|DISABLED*]"
description="Enable ML ranking in quick fix for missing imports"/>
<postStartupActivity implementation="com.intellij.python.ml.features.imports.QuickfixRankingModelLoading"/>
<applicationService serviceInterface="com.jetbrains.python.codeInsight.imports.ImportChooser"
serviceImplementation="com.intellij.python.ml.features.imports.PyMLImportChooser" overrides="true"/>
</extensions>

8
python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/BaseProjectFeatures.kt → python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/BaseProjectFeatures.kt
View File

@@ -1,6 +1,6 @@
// Optimized and improved Kotlin code
package com.jetbrains.python.codeInsight.imports.mlapi.features
package com.intellij.python.ml.features.imports.features
import com.intellij.openapi.application.readAction
import com.intellij.openapi.fileEditor.FileEditorManager
@@ -12,8 +12,6 @@ import com.intellij.psi.search.GlobalSearchScope
import com.intellij.psi.util.PsiTreeUtil
import com.jetbrains.ml.api.feature.*
import com.jetbrains.python.PythonFileType
import com.jetbrains.python.codeInsight.imports.mlapi.ImportRankingContext
import com.jetbrains.python.codeInsight.imports.mlapi.ImportRankingContextFeatures
import com.jetbrains.python.psi.*
private val interestingClasses = arrayOf(
@@ -39,14 +37,14 @@ object BaseProjectFeatures : ImportRankingContextFeatures() {
"The estimated amount of files in the project (by a power of 2)"
}.nullable()
val PSI_PARENT_OF_ORIG: List<FeatureDeclaration<Class<*>?>> = (1..5).map { i -> FeatureDeclaration.aClass("psi_parent_of_orig_$i") { "PSI parent of original element #$i" }.nullable() }
val FILE_EXTENSION_TYPE: FeatureDeclaration<FileExtensionType?> = FeatureDeclaration.enum<FileExtensionType>("file_extension_type") { "extension of the original python file" }.nullable()
val FILE_EXTENSION_TYPE: FeatureDeclaration<Enum<*>?> = FeatureDeclaration.enum<FileExtensionType>("file_extension_type") { "extension of the original python file" }.nullable()
}
override val featureComputationPolicy: FeatureComputationPolicy = FeatureComputationPolicy(true, true)
override val namespaceFeatureDeclarations: List<FeatureDeclaration<*>> = extractFeatureDeclarations(Features)
override suspend fun computeNamespaceFeatures(instance: ImportRankingContext, filter: FeatureFilter): List<Feature> = buildList {
override suspend fun computeNamespaceFeatures(instance: ImportRankingContext, filter: FeatureSet): List<Feature> = buildList {
val candidates = instance.candidates
if (candidates.isEmpty()) return@buildList

6
python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/CandidatesListFeatures.kt → python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/CandidatesListFeatures.kt
View File

@@ -1,9 +1,7 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.jetbrains.python.codeInsight.imports.mlapi.features
package com.intellij.python.ml.features.imports.features
import com.jetbrains.ml.api.feature.*
import com.jetbrains.python.codeInsight.imports.mlapi.ImportRankingContext
import com.jetbrains.python.codeInsight.imports.mlapi.ImportRankingContextFeatures
object CandidatesListFeatures : ImportRankingContextFeatures() {
object Features {
@@ -19,7 +17,7 @@ object CandidatesListFeatures : ImportRankingContextFeatures() {
override val featureComputationPolicy: FeatureComputationPolicy = FeatureComputationPolicy(tolerateRedundantFeatures = true, putNullImplicitly = true)
override suspend fun computeNamespaceFeatures(instance: ImportRankingContext, filter: FeatureFilter): List<Feature> = buildList {
override suspend fun computeNamespaceFeatures(instance: ImportRankingContext, filter: FeatureSet): List<Feature> = buildList {
add(Features.LENGTH with instance.candidates.size)
add(Features.HIGHEST_OLD_RELEVANCE with instance.candidates.maxOfOrNull { it.relevance })
}

6
python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/ImportCandidateRelativeFeatures.kt → python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/ImportCandidateRelativeFeatures.kt
View File

@@ -1,9 +1,7 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.jetbrains.python.codeInsight.imports.mlapi.features
package com.intellij.python.ml.features.imports.features
import com.jetbrains.ml.api.feature.*
import com.jetbrains.python.codeInsight.imports.mlapi.ImportCandidateContext
import com.jetbrains.python.codeInsight.imports.mlapi.ImportCandidateFeatures
object ImportCandidateRelativeFeatures : ImportCandidateFeatures() {
object Features {
@@ -16,7 +14,7 @@ object ImportCandidateRelativeFeatures : ImportCandidateFeatures() {
override val featureComputationPolicy: FeatureComputationPolicy = FeatureComputationPolicy(tolerateRedundantFeatures = true, putNullImplicitly = true)
override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureFilter): List<Feature> = buildList<Feature> {
override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureSet): List<Feature> = buildList<Feature> {
add(Features.RELATIVE_POSITION with instance.candidates.indexOf(instance.candidate))
}
}

6
python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/ImportsFeatures.kt → python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/ImportsFeatures.kt
View File

@@ -1,5 +1,5 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.jetbrains.python.codeInsight.imports.mlapi.features
package com.intellij.python.ml.features.imports.features
import com.intellij.openapi.application.readAction
import com.intellij.openapi.fileEditor.FileEditorManager
@@ -7,8 +7,6 @@ import com.intellij.psi.PsiFile
import com.intellij.psi.PsiManager
import com.intellij.psi.util.QualifiedName
import com.jetbrains.ml.api.feature.*
import com.jetbrains.python.codeInsight.imports.mlapi.ImportCandidateContext
import com.jetbrains.python.codeInsight.imports.mlapi.ImportCandidateFeatures
import com.jetbrains.python.psi.PyFile
import com.jetbrains.python.psi.PyFromImportStatement
import com.jetbrains.python.psi.PyImportElement
@@ -25,7 +23,7 @@ object ImportsFeatures : ImportCandidateFeatures() {
override val namespaceFeatureDeclarations: List<FeatureDeclaration<*>> = extractFeatureDeclarations(Features)
override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureFilter): List<Feature> = buildList {
override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureSet): List<Feature> = buildList {
val importCandidate = instance.candidate
val project = readAction { importCandidate.importable?.project } ?: return@buildList
val editor = readAction { FileEditorManager.getInstance(project).selectedEditor } ?: return@buildList

6
python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/NeighborFilesImportsFeatures.kt → python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/NeighborFilesImportsFeatures.kt
View File

@@ -1,13 +1,11 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.jetbrains.python.codeInsight.imports.mlapi.features
package com.intellij.python.ml.features.imports.features
import com.intellij.openapi.application.readAction
import com.intellij.openapi.fileEditor.FileEditorManager
import com.intellij.openapi.vfs.VirtualFile
import com.intellij.psi.PsiManager
import com.jetbrains.ml.api.feature.*
import com.jetbrains.python.codeInsight.imports.mlapi.ImportCandidateContext
import com.jetbrains.python.codeInsight.imports.mlapi.ImportCandidateFeatures
import com.jetbrains.python.psi.PyFile
import kotlinx.coroutines.async
import kotlinx.coroutines.coroutineScope
@@ -26,7 +24,7 @@ object NeighborFilesImportsFeatures : ImportCandidateFeatures() {
override val namespaceFeatureDeclarations: List<FeatureDeclaration<*>> = extractFeatureDeclarations(Features)
override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureFilter): List<Feature> = coroutineScope {
override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureSet): List<Feature> = coroutineScope {
val importCandidate = instance.candidate
if (importCandidate.path == null) {
return@coroutineScope emptyList()

6
python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/OpenFilesImportsFeatures.kt → python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/OpenFilesImportsFeatures.kt
View File

@@ -1,12 +1,10 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.jetbrains.python.codeInsight.imports.mlapi.features
package com.intellij.python.ml.features.imports.features
import com.intellij.openapi.application.readAction
import com.intellij.openapi.fileEditor.FileEditorManager
import com.intellij.psi.PsiManager
import com.jetbrains.ml.api.feature.*
import com.jetbrains.python.codeInsight.imports.mlapi.ImportCandidateContext
import com.jetbrains.python.codeInsight.imports.mlapi.ImportCandidateFeatures
import com.jetbrains.python.psi.PyFile
import kotlinx.coroutines.async
import kotlinx.coroutines.coroutineScope
@@ -25,7 +23,7 @@ object OpenFilesImportsFeatures : ImportCandidateFeatures() {
override val namespaceFeatureDeclarations: List<FeatureDeclaration<*>> = extractFeatureDeclarations(Features)
override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureFilter): List<Feature> = coroutineScope {
override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureSet): List<Feature> = coroutineScope {
val importCandidate = instance.candidate
if (importCandidate.path == null) {
return@coroutineScope emptyList()

6
python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/PrimitiveImportFeatures.kt → python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/PrimitiveImportFeatures.kt
View File

@@ -1,9 +1,7 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.jetbrains.python.codeInsight.imports.mlapi.features
package com.intellij.python.ml.features.imports.features
import com.jetbrains.ml.api.feature.*
import com.jetbrains.python.codeInsight.imports.mlapi.ImportCandidateContext
import com.jetbrains.python.codeInsight.imports.mlapi.ImportCandidateFeatures
object PrimitiveImportFeatures : ImportCandidateFeatures() {
@@ -22,7 +20,7 @@ object PrimitiveImportFeatures : ImportCandidateFeatures() {
override val featureComputationPolicy: FeatureComputationPolicy = FeatureComputationPolicy(tolerateRedundantFeatures = true, putNullImplicitly = true)
override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureFilter): List<Feature> = buildList {
override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureSet): List<Feature> = buildList {
add(Features.RELEVANCE with instance.candidate.relevance)
add(Features.COMPONENT_COUNT with instance.candidate.path?.componentCount)
}

6
python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/PsiStructureFeatures.kt → python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/PsiStructureFeatures.kt
View File

@@ -1,11 +1,9 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.jetbrains.python.codeInsight.imports.mlapi.features
package com.intellij.python.ml.features.imports.features
import com.intellij.openapi.application.readAction
import com.intellij.psi.PsiElement
import com.jetbrains.ml.api.feature.*
import com.jetbrains.python.codeInsight.imports.mlapi.ImportCandidateContext
import com.jetbrains.python.codeInsight.imports.mlapi.ImportCandidateFeatures
object PsiStructureFeatures : ImportCandidateFeatures() {
@@ -18,7 +16,7 @@ object PsiStructureFeatures : ImportCandidateFeatures() {
override val featureComputationPolicy: FeatureComputationPolicy = FeatureComputationPolicy(tolerateRedundantFeatures = true, putNullImplicitly = true)
override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureFilter): List<Feature> = buildList {
override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureSet): List<Feature> = buildList {
readAction {
add(Features.PSI_CLASS with (instance.candidate.importable?.javaClass))
Features.PSI_PARENT.withIndex().forEach { (i, featureDeclaration) ->

10
python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/RelevanceEvaluationFeatures.kt → python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/RelevanceEvaluationFeatures.kt
View File

@@ -1,5 +1,5 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.jetbrains.python.codeInsight.imports.mlapi.features
package com.intellij.python.ml.features.imports.features
import com.intellij.openapi.application.readAction
import com.intellij.openapi.module.ModuleUtilCore
@@ -7,8 +7,6 @@ import com.intellij.openapi.projectRoots.Sdk
import com.intellij.openapi.vfs.VirtualFile
import com.intellij.psi.util.QualifiedName
import com.jetbrains.ml.api.feature.*
import com.jetbrains.python.codeInsight.imports.mlapi.ImportCandidateContext
import com.jetbrains.python.codeInsight.imports.mlapi.ImportCandidateFeatures
import com.jetbrains.python.sdk.PythonSdkUtil
enum class UnderscoresType {
@@ -32,15 +30,15 @@ enum class ModuleSourceType {
object RelevanceEvaluationFeatures : ImportCandidateFeatures() {
object Features {
val UNDERSCORES_IN_PATH: FeatureDeclaration<Int?> = FeatureDeclaration.int("underscores_in_path") { "number of prefix and suffix underscores in path" }.nullable()
val MODULE_SOURCE_TYPE: FeatureDeclaration<ModuleSourceType?> = FeatureDeclaration.enum<ModuleSourceType>("module_source_type") { "info about lib being std, local, or external" }.nullable()
val UNDERSCORES_TYPES_OF_PACKAGES: List<FeatureDeclaration<UnderscoresType?>> = (1..4).map { i -> FeatureDeclaration.enum<UnderscoresType>("underscores_types_of_package_$i") { "underscores types of package #$i" }.nullable() }
val MODULE_SOURCE_TYPE: FeatureDeclaration<Enum<*>?> = FeatureDeclaration.enum<ModuleSourceType>("module_source_type") { "info about lib being std, local, or external" }.nullable()
val UNDERSCORES_TYPES_OF_PACKAGES: List<FeatureDeclaration<Enum<*>?>> = (1..4).map { i -> FeatureDeclaration.enum<UnderscoresType>("underscores_types_of_package_$i") { "underscores types of package #$i" }.nullable() }
}
override val featureComputationPolicy: FeatureComputationPolicy = FeatureComputationPolicy(true, true)
override val namespaceFeatureDeclarations: List<FeatureDeclaration<*>> = extractFeatureDeclarations(Features)
override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureFilter): List<Feature> = buildList {
override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureSet): List<Feature> = buildList {
val importCandidate = instance.candidate
add(Features.UNDERSCORES_IN_PATH with countBoundaryUnderscores(importCandidate.path))
readAction {

21
python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlFeatures.kt → python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/mlFeatures.kt
View File

@@ -1,17 +1,12 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.python.ml.features.imports
package com.intellij.python.ml.features.imports.features
import com.jetbrains.ml.api.feature.Feature
import com.jetbrains.ml.api.feature.FeatureDeclaration
import com.jetbrains.ml.api.feature.FeatureFilter
import com.jetbrains.ml.api.feature.FeatureValueType
import com.jetbrains.ml.api.feature.FeatureSet
import com.jetbrains.ml.api.feature.suspendable.FeatureProvider
import com.jetbrains.python.codeInsight.imports.mlapi.ImportCandidateContext
import com.jetbrains.python.codeInsight.imports.mlapi.ImportRankingContext
import com.jetbrains.python.codeInsight.imports.mlapi.features.*
import kotlinx.coroutines.async
import kotlinx.coroutines.coroutineScope
import kotlin.collections.plus
internal object FeaturesRegistry {
@@ -30,7 +25,7 @@ internal object FeaturesRegistry {
OpenFilesImportsFeatures,
)
suspend fun computeContextFeatures(instance: ImportRankingContext, filter: FeatureFilter): List<Feature> = coroutineScope {
suspend fun computeContextFeatures(instance: ImportRankingContext, filter: FeatureSet): List<Feature> = coroutineScope {
val jobs = context.map { provider ->
async {
provider.computeFeaturesWithImplicitNull(instance, filter)
@@ -39,7 +34,7 @@ internal object FeaturesRegistry {
jobs.flatMap { it.await() }
}
suspend fun computeCandidateFeatures(instance: ImportCandidateContext, filter: FeatureFilter): List<Feature> = coroutineScope {
suspend fun computeCandidateFeatures(instance: ImportCandidateContext, filter: FeatureSet): List<Feature> = coroutineScope {
val jobs = candidate.map { provider ->
async {
provider.computeFeaturesWithImplicitNull(instance, filter)
@@ -50,23 +45,21 @@ internal object FeaturesRegistry {
private suspend fun <T : Any> FeatureProvider<T>.computeFeaturesWithImplicitNull(
instance: T,
filter: FeatureFilter
): List<Feature> {
filter: FeatureSet
): Collection<Feature> {
val features = provideFeatures(instance, filter)
if (!featureComputationPolicy.putNullImplicitly) {
return features
}
val implicitNullFeatures = mutableListOf<Feature>()
for (declaration in featureDeclarations) {
if (declaration.isNullable() && features.all { feature -> feature.declaration != declaration }) {
if (declaration.isNullable && !filter.contains(declaration)) {
implicitNullFeatures.add(declaration.asNullable() with null)
}
}
return features + implicitNullFeatures
}
private fun FeatureDeclaration<*>.isNullable(): Boolean = type is FeatureValueType.Nullable<*>
@Suppress("UNCHECKED_CAST")
private fun <T> FeatureDeclaration<T>.asNullable(): FeatureDeclaration<T?> = this as FeatureDeclaration<T?>

2
python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/mlUnits.kt → python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/mlUnits.kt
View File

@@ -1,5 +1,5 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.jetbrains.python.codeInsight.imports.mlapi
package com.intellij.python.ml.features.imports.features
import com.jetbrains.ml.api.feature.suspendable.FeatureProvider
import com.jetbrains.python.codeInsight.imports.ImportCandidateHolder

12
python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlAB.kt
View File

@@ -5,9 +5,9 @@ import com.intellij.internal.statistic.eventLog.EventLogConfiguration
import com.intellij.openapi.components.Service
import com.intellij.openapi.components.service
import com.intellij.openapi.util.registry.Registry
import com.intellij.platform.ml.impl.logs.MLEventLoggerProvider.Companion.ML_RECORDER_ID
import com.intellij.python.ml.features.imports.FinalImportRankingStatusService.RegistryOption
import com.jetbrains.ml.api.model.MLModel
import com.jetbrains.ml.tools.model.MLModel
import com.jetbrains.ml.tools.model.catboost.prediction.CatBoostRegressionResult
@Service
internal class FinalImportRankingStatusService {
@@ -20,9 +20,7 @@ internal class FinalImportRankingStatusService {
}
private val bucket: Int
by lazy { service<EventLogConfiguration>().getOrCreate(ML_RECORDER_ID,
// ids between ML and FUS recorders should match
"FUS").bucket }
by lazy { service<EventLogConfiguration>().getOrCreate("ML").bucket }
private val mlEnabledOnBucket: Boolean
by lazy { bucket % 2 == 0 }
@@ -31,7 +29,7 @@ internal class FinalImportRankingStatusService {
val status: FinalImportRankingStatus
get() {
val mlModel = service<ImportsRankingModelService>().getModelOwnership()
val mlModel = service<ImportsRankingModelService>().model
val registryOption = getRegistryOption()
val registryOptionAllowsEnabling = registryOption == RegistryOption.ENABLED || (registryOption == RegistryOption.IN_EXPERIMENT && mlEnabledOnBucket)
return if (mlModel != null && registryOptionAllowsEnabling)
@@ -49,6 +47,6 @@ internal sealed class FinalImportRankingStatus(
val mlModelUnavailable: Boolean,
val registryOption: RegistryOption,
) {
class Enabled(val mlModel: MLModel<Double>, registryOption: RegistryOption) : FinalImportRankingStatus(true, false, registryOption)
class Enabled(val mlModel: MLModel<CatBoostRegressionResult>, registryOption: RegistryOption) : FinalImportRankingStatus(true, false, registryOption)
class Disabled(mlModelUnavailable: Boolean, registryOption: RegistryOption) : FinalImportRankingStatus(false, mlModelUnavailable, registryOption)
}

8
python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlAnalysis.kt
View File

@@ -8,15 +8,15 @@ import com.jetbrains.ml.api.logs.LongEventField
internal object ContextAnalysis {
val CANCELLED = BooleanEventField("selection_cancelled") { "No item has been selected" }
val CANCELLED = BooleanEventField("selection_cancelled", null) { "No item has been selected" }
val SELECTED_POSITION = IntEventField("selected_position") { "The position of the selected import statement" }
val MODEL_UNAVAILABLE = BooleanEventField("model_unavailable") { "ML model was unavailable" }
val MODEL_UNAVAILABLE = BooleanEventField("model_unavailable", null) { "ML model was unavailable" }
val SELECTED_POSITION_INITIAL = IntEventField("selected_position_initial") { "The position of the selected import statement, if the final ML ranking would not have happened" }
val TIME_MS_TO_DISPLAY = LongEventField("time_ms_before_displayed") { "Duration from the quickfix start until when the imports were displayed" }
val TIME_MS_BEFORE_CLOSED = LongEventField("time_ms_before_closed") { "Duration from the quickfix start until the pop-up was closed" }
val ML_LOGGING_STATE = EnumEventField.of<LoggingOption>("ml_logging_state", { "State of the ML session logging" })
val ML_LOGGING_STATE = EnumEventField("ml_logging_state", null, { "State of the ML session logging" }, LoggingOption::class.java)
val ML_ENABLED = BooleanEventField("ml_enabled") { "Machine Learning ranking is enabled" }
val REGISTRY_OPTION = EnumEventField.of<FinalImportRankingStatusService.RegistryOption>("registry_option", { "Registry option of the experiment status" })
val REGISTRY_OPTION = EnumEventField("registry_option", null, { "Registry option of the experiment status" }, FinalImportRankingStatusService.RegistryOption::class.java)
}
internal object CandidateAnalysis {

37
python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlImplementation.kt
View File

@@ -6,14 +6,17 @@ import com.intellij.openapi.components.Service
import com.intellij.openapi.components.service
import com.intellij.openapi.diagnostic.debug
import com.intellij.openapi.diagnostic.logger
import com.intellij.python.ml.features.imports.features.FeaturesRegistry
import com.intellij.python.ml.features.imports.features.ImportCandidateContext
import com.intellij.python.ml.features.imports.features.ImportRankingContext
import com.intellij.util.application
import com.jetbrains.ml.api.feature.Feature
import com.jetbrains.ml.api.feature.FeatureDeclaration
import com.jetbrains.ml.api.logs.EventPair
import com.jetbrains.ml.api.model.MLModel
import com.jetbrains.ml.tools.logs.MLLogsTree
import com.jetbrains.ml.tools.model.MLModel
import com.jetbrains.ml.tools.model.catboost.prediction.CatBoostRegressionResult
import com.jetbrains.python.codeInsight.imports.ImportCandidateHolder
import com.jetbrains.python.codeInsight.imports.mlapi.ImportCandidateContext
import com.jetbrains.python.codeInsight.imports.mlapi.ImportRankingContext
import kotlinx.coroutines.*
import java.util.concurrent.atomic.AtomicBoolean
import kotlin.time.Duration.Companion.seconds
@@ -39,23 +42,23 @@ private class ExperimentalMLRanker(
contextFeatures: MutableList<Feature>,
contextAnalysis: MutableList<EventPair<*>>,
timestampStarted: Long,
private val mlModel: MLModel<Double>,
private val mlModel: MLModel<CatBoostRegressionResult>,
) : FinalCandidatesRanker(contextFeatures, contextAnalysis, timestampStarted) {
override val mlEnabled = true
override fun launchMLRanking(initialCandidatesOrder: MutableList<out ImportCandidateHolder>, displayResult: (RateableRankingResult) -> Unit) {
service<MLApiComputations>().coroutineScope.launch(Dispatchers.Default) {
contextFeatures.addAll(FeaturesRegistry.computeContextFeatures(ImportRankingContext(initialCandidatesOrder), mlModel.knownFeatures))
contextFeatures.addAll(FeaturesRegistry.computeContextFeatures(ImportRankingContext(initialCandidatesOrder), mlModel.inputFeatures))
val importCandidatesFeatures = initialCandidatesOrder.associateWith { candidate ->
async { FeaturesRegistry.computeCandidateFeatures(ImportCandidateContext(initialCandidatesOrder, candidate), mlModel.knownFeatures) }
async { FeaturesRegistry.computeCandidateFeatures(ImportCandidateContext(initialCandidatesOrder, candidate), mlModel.inputFeatures) }
}.mapValues { it.value.await() }
val scoreByCandidate = mlModel.predictBatch(contextFeatures, importCandidatesFeatures)
val scoreByCandidate = mlModel.predictBatch(contextFeatures + fillTypingFeatures(), importCandidatesFeatures)
val relevanceCandidateOrder = scoreByCandidate.toList()
.sortedByDescending { it.second }
.sortedByDescending { it.second.logit }
.map { it.first }
logger<ExperimentalMLRanker>().debug {
@@ -175,3 +178,21 @@ internal class RateableRankingResult(
}
}
}
private object TypingFeatures {
val SINCE_LAST_TYPING = FeatureDeclaration.int("typing_speed_tracker_time_since_last_typing") { "Deprecated" }
val TYPING_SPEED_1S = FeatureDeclaration.double("typing_speed_tracker_typing_speed_1s") { "Deprecated" }
val TYPING_SPEED_2S = FeatureDeclaration.double("typing_speed_tracker_typing_speed_2s") { "Deprecated" }
val TYPING_SPEED_5S = FeatureDeclaration.double("typing_speed_tracker_typing_speed_5s") { "Deprecated" }
val TYPING_SPEED_30S = FeatureDeclaration.double("typing_speed_tracker_typing_speed_30s") { "Deprecated" }
}
private fun fillTypingFeatures(): List<Feature> {
return listOf(
TypingFeatures.SINCE_LAST_TYPING with 1,
TypingFeatures.TYPING_SPEED_2S with 0.0,
TypingFeatures.TYPING_SPEED_30S with 0.0,
TypingFeatures.TYPING_SPEED_5S with 0.0,
TypingFeatures.TYPING_SPEED_1S with 0.0,
)
}

6
python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlLogs.kt
View File

@@ -3,8 +3,8 @@ package com.intellij.python.ml.features.imports
import com.intellij.internal.statistic.eventLog.EventLogGroup
import com.intellij.internal.statistic.service.fus.collectors.CounterUsagesCollector
import com.intellij.platform.ml.impl.fus.IntelliJFusEventRegister
import com.intellij.platform.ml.impl.logs.MLEventLoggerProvider.Companion.ML_RECORDER_ID
import com.intellij.platform.ml.logs.IntelliJFusEventRegister
import com.intellij.python.ml.features.imports.features.FeaturesRegistry
import com.intellij.util.application
import com.jetbrains.ml.tools.logs.MLTreeLoggers.withOneEvent
import com.jetbrains.ml.tools.logs.extractEventFields
@@ -12,7 +12,7 @@ import kotlin.random.Random
internal object PyCharmImportsRankingLogs : CounterUsagesCollector() {
private val GROUP = EventLogGroup("pycharm.quickfix.imports", 7, ML_RECORDER_ID)
private val GROUP = EventLogGroup("pycharm.quickfix.imports", 7, "ML")
val mlLogger = withOneEvent(
fusEventName = "pycharm_import_statements_ranking",
fusEventRegister = IntelliJFusEventRegister(GROUP),

99
python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlModel.kt
View File

@@ -2,73 +2,58 @@
package com.intellij.python.ml.features.imports
import com.intellij.openapi.components.Service
import com.jetbrains.ml.api.feature.Feature
import com.jetbrains.ml.api.feature.FeatureDeclaration
import com.jetbrains.ml.api.feature.FeatureFilter
import com.jetbrains.ml.api.feature.extractFeatureDeclarations
import com.jetbrains.ml.api.model.MLModel
import com.jetbrains.ml.api.model.MLModelLoader
import com.intellij.openapi.components.service
import com.intellij.openapi.diagnostic.logger
import com.intellij.openapi.project.Project
import com.intellij.openapi.startup.ProjectActivity
import com.intellij.util.concurrency.AppExecutorUtil
import com.jetbrains.ml.models.PythonImportsRankingModelHolder
import com.jetbrains.ml.tools.model.MLModelLoaders
import com.jetbrains.ml.tools.model.ModelDistributionReaders
import com.jetbrains.ml.tools.model.catboost.CatBoostDistributionFormat
import com.jetbrains.ml.tools.model.suspendable.MLModelSuspendableService
import com.jetbrains.ml.tools.model.MLModel
import com.jetbrains.ml.tools.model.catboost.prediction.CatBoostRegressionResult
import com.jetbrains.ml.tools.model.pipeline.ModelPipeline
import com.jetbrains.ml.tools.model.pipeline.ModelPipelineLoader
import java.util.concurrent.CompletableFuture
import java.util.concurrent.ExecutorService
@Service
class ImportsRankingModelService : MLModelSuspendableService<MLModel<Double>, Double>(
MissingTypingFeaturesLoader(MLModelLoaders.Regressor(
reader = ModelDistributionReaders.FromJavaResources(PythonImportsRankingModelHolder::class.java, PythonImportsRankingModelHolder.MODEL_ID),
format = CatBoostDistributionFormat(),
))
)
@Service(Service.Level.APP)
internal class ImportsRankingModelService {
private class MissingTypingFeaturesLoader(private val baseLoader: MLModelLoader<MLModel<Double>, Double>) : MLModelLoader<MLModel<Double>, Double> {
init {
loadModel()
}
override fun loadModel(executor: ExecutorService, parameters: Map<String, Any>?): CompletableFuture<MLModel<Double>> {
return baseLoader.loadModel(executor, parameters).thenApply { rawModel ->
return@thenApply object : MLModel<Double> {
override fun predict(features: List<Feature>, parameters: Map<String, Any>): Double {
return rawModel.predict(features + fillTypingFeatures(), parameters)
}
private var modelFuture: CompletableFuture<Void>? = null
override fun predictBatch(contextFeatures: List<Feature>, itemFeatures: List<List<Feature>>, parameters: Map<String, Any>): List<Double> {
return rawModel.predictBatch(contextFeatures + fillTypingFeatures(), itemFeatures, parameters)
}
var model: ModelPipeline<MLModel<CatBoostRegressionResult>, CatBoostRegressionResult>? = null
private set
override val knownFeatures: FeatureFilter = object : FeatureFilter {
override fun accept(featureDeclaration: FeatureDeclaration<*>): Boolean {
return if (featureDeclaration in allTypingFeatures) {
return false
}
else {
rawModel.knownFeatures.accept(featureDeclaration)
}
}
}
}
private fun loadModel() {
modelFuture?.cancel(true)
LOG.info("Loading CatBoost Imports Ranking model")
modelFuture = ModelPipelineLoader.load(
PythonImportsRankingModelHolder.getStream(),
AppExecutorUtil.getAppExecutorService()
).thenAccept { model ->
LOG.info("Successfully loaded imports ranking model")
this.model = model.predicting<CatBoostRegressionResult>()
}.exceptionally { e ->
LOG.warn("Failed to load CatBoost imports ranking model", e)
null
}
}
companion object {
@JvmStatic
fun getInstance(): ImportsRankingModelService = service()
private val LOG = logger<ImportsRankingModelService>()
}
}
private object TypingFeatures {
val SINCE_LAST_TYPING = FeatureDeclaration.int("typing_speed_tracker_time_since_last_typing") { "Deprecated" }
val TYPING_SPEED_1S = FeatureDeclaration.double("typing_speed_tracker_typing_speed_1s") { "Deprecated" }
val TYPING_SPEED_2S = FeatureDeclaration.double("typing_speed_tracker_typing_speed_2s") { "Deprecated" }
val TYPING_SPEED_5S = FeatureDeclaration.double("typing_speed_tracker_typing_speed_5s") { "Deprecated" }
val TYPING_SPEED_30S = FeatureDeclaration.double("typing_speed_tracker_typing_speed_30s") { "Deprecated" }
}
private val allTypingFeatures: List<FeatureDeclaration<*>> = extractFeatureDeclarations(TypingFeatures)
private fun fillTypingFeatures(): List<Feature> {
return listOf(
TypingFeatures.SINCE_LAST_TYPING with 1,
TypingFeatures.TYPING_SPEED_2S with 0.0,
TypingFeatures.TYPING_SPEED_30S with 0.0,
TypingFeatures.TYPING_SPEED_5S with 0.0,
TypingFeatures.TYPING_SPEED_1S with 0.0,
)
private class QuickfixRankingModelLoading : ProjectActivity {
override suspend fun execute(project: Project) {
if (!service<FinalImportRankingStatusService>().shouldLoadModel) return
ImportsRankingModelService.getInstance()
}
}

12
python/pluginCore/plugin-content.yaml
View File

@@ -8,11 +8,6 @@
files:
- name: $MAVEN_REPOSITORY$/io/github/z4kn4fein/semver-jvm/2/semver-jvm-2.jar
reason: <- intellij.python.community.impl
- name: lib/jetbrains.ml.models.python.imports.ranking.model.jar
library: jetbrains.ml.models.python.imports.ranking.model
files:
- name: $MAVEN_REPOSITORY$/com/jetbrains/ml/models/python-imports-ranking-model/nebulous-albatross/python-imports-ranking-model-nebulous-albatross.jar
reason: <- intellij.python.ml.features
- name: lib/jsr305.jar
library: jsr305
files:
@@ -35,6 +30,12 @@
module: intellij.python.community.impl
files:
- name: $MAVEN_REPOSITORY$/completion/ml/python/features/ml-completion-prev-exprs-models/1/ml-completion-prev-exprs-models-1.jar
- name: lib/modules/intellij.python.ml.features.jar
contentModules:
- name: intellij.python.ml.features
libraries:
jetbrains.ml.models.python.imports.ranking.model:
- name: $MAVEN_REPOSITORY$/com/jetbrains/ml/models/python-imports-ranking-model/coral-panda-republished-4/python-imports-ranking-model-coral-panda-republished-4.jar
- name: lib/python-ce-frontend.jar
contentModules:
- name: intellij.python.parser
@@ -75,7 +76,6 @@
- name: intellij.python.grazie
- name: intellij.python.langInjection
- name: intellij.python.markdown
- name: intellij.python.ml.features
- name: intellij.python.terminal
- name: lib/tuweni-toml.jar
library: tuweni-toml

1
python/python-psi-impl/BUILD.bazel
View File

@@ -48,7 +48,6 @@ jvm_library(
"//platform/backend/workspace",
"//python/python-parser:parser",
"//python/python-syntax-core:syntax-core",
"//platform/ml-api:ml",
"//python/impl.helperLocator:community-helpersLocator",
],
exports = ["//python/python-syntax-core:syntax-core"],

1
python/python-psi-impl/intellij.python.psi.impl.iml
View File

@@ -44,7 +44,6 @@
<orderEntry type="module" module-name="intellij.platform.backend.workspace" />
<orderEntry type="module" module-name="intellij.python.parser" />
<orderEntry type="module" module-name="intellij.python.syntax.core" exported="" />
<orderEntry type="module" module-name="intellij.platform.ml" />
<orderEntry type="module" module-name="intellij.python.community.helpersLocator" />
</component>
</module>