diff --git a/.idea/libraries/jetbrains_ml_models_python_imports_ranking_model.xml b/.idea/libraries/jetbrains_ml_models_python_imports_ranking_model.xml deleted file mode 100644 index 4f04be2bd2c3..000000000000 --- a/.idea/libraries/jetbrains_ml_models_python_imports_ranking_model.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - 5c5e1cc5fae2d8379a586b87ca580c677e65bdc4538ee0e0773cce141dc61b6a - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/jetbrains_mlapi_ml_api.xml b/.idea/libraries/jetbrains_mlapi_ml_api.xml index 795e3f342104..32783d55c678 100644 --- a/.idea/libraries/jetbrains_mlapi_ml_api.xml +++ b/.idea/libraries/jetbrains_mlapi_ml_api.xml @@ -1,9 +1,9 @@ - + - - a52b66fc6cafd0f9f8e295e9748de49903ad662d0d1bebb9d6df60e672e4d4cb + + 1050f9b58c8d71833d075ae7302881cb9d905dcda4025a21c59657ac823ac105 @@ -11,13 +11,13 @@ - + - + - + \ No newline at end of file diff --git a/.idea/libraries/jetbrains_mlapi_ml_tools.xml b/.idea/libraries/jetbrains_mlapi_ml_tools.xml index d43c0de4df8f..b23f52b39a5b 100644 --- a/.idea/libraries/jetbrains_mlapi_ml_tools.xml +++ b/.idea/libraries/jetbrains_mlapi_ml_tools.xml @@ -1,9 +1,9 @@ - + - - 1fd34e39c77dad83957882a7b6ccf99c1d49403b4643efc85a4b041aa1108a13 + + b2d36677ba73f2f7974199823b5f4323b2a16ea718ed506297402c6a7605af1c @@ -11,13 +11,13 @@ - + - + - + \ No newline at end of file diff --git a/.idea/libraries/jetbrains_mlapi_ml_tools_suspendable.xml b/.idea/libraries/jetbrains_mlapi_ml_tools_suspendable.xml deleted file mode 100644 index a146422563ce..000000000000 --- a/.idea/libraries/jetbrains_mlapi_ml_tools_suspendable.xml +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - 98be4a00226838f7afc9dd054ee40ed03e1379672b343258480b8b72f1b25e0f - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml index 4054ee0c5217..2457564a393f 100644 --- a/.idea/modules.xml +++ b/.idea/modules.xml @@ -872,6 +872,7 @@ + diff --git a/lib/BUILD.bazel b/lib/BUILD.bazel index d0e9373c929a..0178d5945d5b 100644 --- a/lib/BUILD.bazel +++ b/lib/BUILD.bazel @@ -2807,13 +2807,6 @@ jvm_import( visibility = ["//visibility:public"] ) -jvm_import( - name = "jetbrains-ml-models-python-imports-ranking-model", - jar = "@python-imports-ranking-model-nebulous-albatross_http//file", - source_jar = "@python-imports-ranking-model-nebulous-albatross-sources_http//file", - visibility = ["//visibility:public"] -) - jvm_import( name = "jetbrains-mlapi-catboost-shadow-need-slf4j", jar = "@catboost-shadow-need-slf4j-1_2_5_http//file", @@ -2822,22 +2815,15 @@ jvm_import( jvm_import( name = "jetbrains-mlapi-ml-api", - jar = "@ml-api-86_http//file", - source_jar = "@ml-api-86-sources_http//file", + jar = "@ml-api-0_1_94_http//file", + source_jar = "@ml-api-0_1_94-sources_http//file", visibility = ["//visibility:public"] ) jvm_import( name = "jetbrains-mlapi-ml-tools", - jar = "@ml-tools-86_http//file", - source_jar = "@ml-tools-86-sources_http//file", - visibility = ["//visibility:public"] -) - -jvm_import( - name = "jetbrains-mlapi-ml-tools-suspendable", - jar = "@ml-tools-suspendable-76_http//file", - source_jar = "@ml-tools-suspendable-76-sources_http//file", + jar = "@ml-tools-0_1_94_http//file", + source_jar = "@ml-tools-0_1_94-sources_http//file", visibility = ["//visibility:public"] ) @@ -8539,6 +8525,13 @@ jvm_import( source_jar = "@byte-buddy-agent-1_17_4-sources_http//file" ) +jvm_import( + name = "python-ml-features-jetbrains-ml-models-python-imports-ranking-model", + jar = "@python-imports-ranking-model-coral-panda-republished-4_http//file", + source_jar = "@python-imports-ranking-model-coral-panda-republished-4-sources_http//file", + visibility = ["//visibility:public"] +) + jvm_import( name = "rd-core", jar = "@rd-core-2025_2_2_http//file", diff --git a/lib/MODULE.bazel b/lib/MODULE.bazel index 0c9233e7136a..837fbd77ce65 100644 --- a/lib/MODULE.bazel +++ b/lib/MODULE.bazel @@ -3897,20 +3897,6 @@ http_file( downloaded_file_path = "markdown-jvm-0.7.2-sources.jar" ) -http_file( - name = "python-imports-ranking-model-nebulous-albatross_http", - url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/ml/models/python-imports-ranking-model/nebulous-albatross/python-imports-ranking-model-nebulous-albatross.jar", - sha256 = "5c5e1cc5fae2d8379a586b87ca580c677e65bdc4538ee0e0773cce141dc61b6a", - downloaded_file_path = "python-imports-ranking-model-nebulous-albatross.jar" -) - -http_file( - name = "python-imports-ranking-model-nebulous-albatross-sources_http", - url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/ml/models/python-imports-ranking-model/nebulous-albatross/python-imports-ranking-model-nebulous-albatross-sources.jar", - sha256 = "45f0027b5a3ed622fb77830f7f98caaa53aed34ba3a8c7d32c85c0c39223d612", - downloaded_file_path = "python-imports-ranking-model-nebulous-albatross-sources.jar" -) - http_file( name = "catboost-shadow-need-slf4j-1_2_5_http", url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/mlapi/catboost-shadow-need-slf4j/1.2.5/catboost-shadow-need-slf4j-1.2.5.jar", @@ -3919,45 +3905,31 @@ http_file( ) http_file( - name = "ml-api-86_http", - url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/mlapi/ml-api/86/ml-api-86.jar", - sha256 = "a52b66fc6cafd0f9f8e295e9748de49903ad662d0d1bebb9d6df60e672e4d4cb", - downloaded_file_path = "ml-api-86.jar" + name = "ml-api-0_1_94_http", + url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/mlapi/ml-api/0.1.94/ml-api-0.1.94.jar", + sha256 = "1050f9b58c8d71833d075ae7302881cb9d905dcda4025a21c59657ac823ac105", + downloaded_file_path = "ml-api-0.1.94.jar" ) http_file( - name = "ml-api-86-sources_http", - url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/mlapi/ml-api/86/ml-api-86-sources.jar", - sha256 = "71001504fe5e7f21d83b717fa689bf6a2f3818428626eb4c0ff09eef688e03f6", - downloaded_file_path = "ml-api-86-sources.jar" + name = "ml-api-0_1_94-sources_http", + url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/mlapi/ml-api/0.1.94/ml-api-0.1.94-sources.jar", + sha256 = "79eb26365ce9f536733d96fc16e40e7e89e1daba9312a8d06449718430d7d499", + downloaded_file_path = "ml-api-0.1.94-sources.jar" ) http_file( - name = "ml-tools-86_http", - url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/mlapi/ml-tools/86/ml-tools-86.jar", - sha256 = "1fd34e39c77dad83957882a7b6ccf99c1d49403b4643efc85a4b041aa1108a13", - downloaded_file_path = "ml-tools-86.jar" + name = "ml-tools-0_1_94_http", + url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/mlapi/ml-tools/0.1.94/ml-tools-0.1.94.jar", + sha256 = "b2d36677ba73f2f7974199823b5f4323b2a16ea718ed506297402c6a7605af1c", + downloaded_file_path = "ml-tools-0.1.94.jar" ) http_file( - name = "ml-tools-86-sources_http", - url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/mlapi/ml-tools/86/ml-tools-86-sources.jar", - sha256 = "3780b7e4c6fad6945812f4729b661d54acc85068fd7a4b106e972bbd7d4dd0b1", - downloaded_file_path = "ml-tools-86-sources.jar" -) - -http_file( - name = "ml-tools-suspendable-76_http", - url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/mlapi/ml-tools-suspendable/76/ml-tools-suspendable-76.jar", - sha256 = "98be4a00226838f7afc9dd054ee40ed03e1379672b343258480b8b72f1b25e0f", - downloaded_file_path = "ml-tools-suspendable-76.jar" -) - -http_file( - name = "ml-tools-suspendable-76-sources_http", - url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/mlapi/ml-tools-suspendable/76/ml-tools-suspendable-76-sources.jar", - sha256 = "11b562e584a7b01596dc9fd14ca7c710a27f48c242b597a90af9ff32f663ace8", - downloaded_file_path = "ml-tools-suspendable-76-sources.jar" + name = "ml-tools-0_1_94-sources_http", + url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/mlapi/ml-tools/0.1.94/ml-tools-0.1.94-sources.jar", + sha256 = "8c5b021724a9b8dab25e33b09812aadbd160cf8e64e89a3dd17c40449dbdedae", + downloaded_file_path = "ml-tools-0.1.94-sources.jar" ) http_file( @@ -12325,6 +12297,20 @@ http_file( downloaded_file_path = "byte-buddy-agent-1.17.4-sources.jar" ) +http_file( + name = "python-imports-ranking-model-coral-panda-republished-4_http", + url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/ml/models/python-imports-ranking-model/coral-panda-republished-4/python-imports-ranking-model-coral-panda-republished-4.jar", + sha256 = "cff0591184b48356a622f7635c6de1399fdbeb161f98d68206d09807c83fa98c", + downloaded_file_path = "python-imports-ranking-model-coral-panda-republished-4.jar" +) + +http_file( + name = "python-imports-ranking-model-coral-panda-republished-4-sources_http", + url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/ml/models/python-imports-ranking-model/coral-panda-republished-4/python-imports-ranking-model-coral-panda-republished-4-sources.jar", + sha256 = "4aecfba9498dff7e199c7db2a718b53c77834347daf8b96cafc8b02b392c5e6c", + downloaded_file_path = "python-imports-ranking-model-coral-panda-republished-4-sources.jar" +) + http_file( name = "rd-core-2025_2_2_http", url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/com/jetbrains/rd/rd-core/2025.2.2/rd-core-2025.2.2.jar", diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/CommunityLibraryLicenses.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/CommunityLibraryLicenses.kt index d4a374a17973..8ded8d96f9ae 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/CommunityLibraryLicenses.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/CommunityLibraryLicenses.kt @@ -1381,20 +1381,15 @@ object CommunityLibraryLicenses { jetbrainsLibrary("intellij.remoterobot.remote.fixtures"), jetbrainsLibrary("intellij.remoterobot.robot.server.core"), jetbrainsLibrary("jetbrains.compose.hot.reload.gradle.idea"), - jetbrainsLibrary("jetbrains.fullLine.mlapi.ml.api"), - jetbrainsLibrary("jetbrains.fullLine.mlapi.ml.tools"), jetbrainsLibrary("jetbrains.fus.reporting.ap.validation"), jetbrainsLibrary("jetbrains.intellij.deps.rwmutex.idea"), jetbrainsLibrary("jetbrains.kotlin.compose.compiler.plugin"), - jetbrainsLibrary("jetbrains.llm.mlapi.catboost.shadow.need.slf4j"), - jetbrainsLibrary("jetbrains.llm.mlapi.ml.api"), - jetbrainsLibrary("jetbrains.llm.mlapi.ml.tools"), jetbrainsLibrary("jetbrains.ml.completion.trigger_model_kotlin_cloud"), jetbrainsLibrary("jetbrains.ml.models.jetenry.inline.prompt.detection.model"), jetbrainsLibrary("jetbrains.ml.models.python.imports.ranking.model"), + jetbrainsLibrary("jetbrains.mlapi.catboost.shadow.need.slf4j"), jetbrainsLibrary("jetbrains.mlapi.ml.api"), jetbrainsLibrary("jetbrains.mlapi.ml.tools"), - jetbrainsLibrary("jetbrains.mlapi.ml.tools.suspendable"), jetbrainsLibrary("jshell-frontend"), jetbrainsLibrary("jvm-native-trusted-roots"), jetbrainsLibrary("kotlin-gradle-plugin-idea"), diff --git a/platform/ml-api/BUILD.bazel b/platform/ml-api/BUILD.bazel index 09db92977df8..4575b50b8aaf 100644 --- a/platform/ml-api/BUILD.bazel +++ b/platform/ml-api/BUILD.bazel @@ -18,9 +18,7 @@ jvm_library( "//platform/core-api:core", "//platform/util", "@lib//:jetbrains-annotations", - "@lib//:jetbrains-mlapi-ml-api", ], - exports = ["@lib//:jetbrains-mlapi-ml-api"], runtime_deps = [":ml_resources"] ) ### auto-generated section `build intellij.platform.ml` end \ No newline at end of file diff --git a/platform/ml-api/intellij.platform.ml.iml b/platform/ml-api/intellij.platform.ml.iml index db9cda46de5c..02f73ad35ecd 100644 --- a/platform/ml-api/intellij.platform.ml.iml +++ b/platform/ml-api/intellij.platform.ml.iml @@ -13,6 +13,5 @@ - \ No newline at end of file diff --git a/platform/ml-impl/BUILD.bazel b/platform/ml-impl/BUILD.bazel index 609c881abfd0..c0d920a0ee27 100644 --- a/platform/ml-impl/BUILD.bazel +++ b/platform/ml-impl/BUILD.bazel @@ -36,17 +36,9 @@ jvm_library( "//platform/ml-api:ml", "//platform/statistics", "@lib//:kotlin-reflect", - "@lib//:jetbrains-mlapi-catboost-shadow-need-slf4j", "//platform/ide-core-impl", - "@lib//:jetbrains-mlapi-ml-tools", - "@lib//:jetbrains-mlapi-ml-tools-suspendable", - ], - exports = [ - "@lib//:ngram-slp", - "@lib//:jetbrains-mlapi-catboost-shadow-need-slf4j", - "@lib//:jetbrains-mlapi-ml-tools", - "@lib//:jetbrains-mlapi-ml-tools-suspendable", ], + exports = ["@lib//:ngram-slp"], runtime_deps = [ ":ml-impl_resources", "//platform/backend", @@ -80,10 +72,7 @@ jvm_library( "//platform/statistics", "//platform/statistics:statistics_test_lib", "@lib//:kotlin-reflect", - "@lib//:jetbrains-mlapi-catboost-shadow-need-slf4j", "//platform/ide-core-impl", - "@lib//:jetbrains-mlapi-ml-tools", - "@lib//:jetbrains-mlapi-ml-tools-suspendable", "//platform/statistics/testFramework", ], runtime_deps = [ diff --git a/platform/ml-impl/intellij.platform.ml.impl.iml b/platform/ml-impl/intellij.platform.ml.impl.iml index 21345931fc05..f44e6ed1e56f 100644 --- a/platform/ml-impl/intellij.platform.ml.impl.iml +++ b/platform/ml-impl/intellij.platform.ml.impl.iml @@ -52,10 +52,7 @@ - - - \ No newline at end of file diff --git a/platform/ml-impl/src/com/intellij/platform/ml/impl/fus/eventFields.kt b/platform/ml-impl/src/com/intellij/platform/ml/impl/fus/eventFields.kt deleted file mode 100644 index cc229735c896..000000000000 --- a/platform/ml-impl/src/com/intellij/platform/ml/impl/fus/eventFields.kt +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. -package com.intellij.platform.ml.impl.fus - -import com.intellij.internal.statistic.eventLog.events.EventFields -import com.intellij.lang.Language -import com.intellij.openapi.util.Version -import com.jetbrains.ml.api.logs.RuleBasedStringEventField -import org.jetbrains.annotations.ApiStatus - - -@ApiStatus.Internal -class VersionEventField(name: String, lazyDescription: () -> String) : RuleBasedStringEventField(name, EventFields.Version.validationRule.first(), lazyDescription, { it.toCompactString() }) - -@ApiStatus.Internal -class LanguageEventField(name: String, lazyDescription: () -> String) : RuleBasedStringEventField(name, EventFields.Language.validationRule.first(), lazyDescription, { it.id }) diff --git a/platform/ml-logs/BUILD.bazel b/platform/ml-logs/BUILD.bazel new file mode 100644 index 000000000000..c5d7dc6cf463 --- /dev/null +++ b/platform/ml-logs/BUILD.bazel @@ -0,0 +1,30 @@ +### auto-generated section `build intellij.platform.ml.logs` start +load("@rules_jvm//:jvm.bzl", "jvm_library", "jvm_resources") + +jvm_resources( + name = "ml-logs_resources", + files = glob(["resources/**/*"]), + strip_prefix = "resources" +) + +jvm_library( + name = "ml-logs", + module_name = "intellij.platform.ml.logs", + visibility = ["//visibility:public"], + srcs = glob(["src/**/*.kt", "src/**/*.java"], allow_empty = True), + deps = [ + "@lib//:kotlin-stdlib", + "//platform/statistics", + "@lib//:jetbrains-annotations", + "@lib//:jetbrains-mlapi-catboost-shadow-need-slf4j", + "@lib//:jetbrains-mlapi-ml-api", + "@lib//:jetbrains-mlapi-ml-tools", + ], + exports = [ + "@lib//:jetbrains-mlapi-catboost-shadow-need-slf4j", + "@lib//:jetbrains-mlapi-ml-api", + "@lib//:jetbrains-mlapi-ml-tools", + ], + runtime_deps = [":ml-logs_resources"] +) +### auto-generated section `build intellij.platform.ml.logs` end \ No newline at end of file diff --git a/platform/ml-logs/api-dump.txt b/platform/ml-logs/api-dump.txt new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/platform/ml-logs/intellij.platform.ml.logs.iml b/platform/ml-logs/intellij.platform.ml.logs.iml new file mode 100644 index 000000000000..3c5c689fcc7c --- /dev/null +++ b/platform/ml-logs/intellij.platform.ml.logs.iml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/platform/ml-logs/resources/intellij.platform.ml.logs.xml b/platform/ml-logs/resources/intellij.platform.ml.logs.xml new file mode 100644 index 000000000000..b886081abee3 --- /dev/null +++ b/platform/ml-logs/resources/intellij.platform.ml.logs.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/platform/ml-impl/src/com/intellij/platform/ml/impl/fus/IJEventPairConverter.kt b/platform/ml-logs/src/com/intellij/platform/ml/logs/IJEventPairConverter.kt similarity index 75% rename from platform/ml-impl/src/com/intellij/platform/ml/impl/fus/IJEventPairConverter.kt rename to platform/ml-logs/src/com/intellij/platform/ml/logs/IJEventPairConverter.kt index 14cd47bce95f..981d473c7dae 100644 --- a/platform/ml-impl/src/com/intellij/platform/ml/impl/fus/IJEventPairConverter.kt +++ b/platform/ml-logs/src/com/intellij/platform/ml/logs/IJEventPairConverter.kt @@ -1,22 +1,16 @@ // Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. -package com.intellij.platform.ml.impl.fus +package com.intellij.platform.ml.logs -import com.intellij.internal.statistic.eventLog.FeatureUsageData import com.intellij.internal.statistic.eventLog.events.EventPair -import com.intellij.internal.statistic.eventLog.events.PrimitiveEventField import com.intellij.internal.statistic.eventLog.events.StringEventField -import com.intellij.lang.Language -import com.intellij.openapi.util.Version -import com.intellij.platform.ml.impl.fus.ConverterObjectDescription.Companion.asIJObjectDescription -import com.intellij.platform.ml.impl.fus.ConverterOfEnum.Companion.toIJConverter -import com.intellij.platform.ml.impl.fus.IJEventPairConverter.Companion.typedBuild -import com.jetbrains.ml.api.logs.RuleBasedStringEventField +import com.intellij.platform.ml.logs.ConverterObjectDescription.Companion.asIJObjectDescription +import com.intellij.platform.ml.logs.ConverterOfEnum.Companion.toIJConverter +import com.intellij.platform.ml.logs.IJEventPairConverter.Companion.typedBuild import com.intellij.internal.statistic.eventLog.events.BooleanEventField as IJBooleanEventField import com.intellij.internal.statistic.eventLog.events.ClassEventField as IJClassEventField import com.intellij.internal.statistic.eventLog.events.DoubleEventField as IJDoubleEventField import com.intellij.internal.statistic.eventLog.events.EnumEventField as IJEnumEventField import com.intellij.internal.statistic.eventLog.events.EventField as IJEventField -import com.intellij.internal.statistic.eventLog.events.EventFields as IJEventFields import com.intellij.internal.statistic.eventLog.events.EventPair as IJEventPair import com.intellij.internal.statistic.eventLog.events.FloatEventField as IJFloatEventField1 import com.intellij.internal.statistic.eventLog.events.FloatListEventField as IJFloatListEventField @@ -28,7 +22,6 @@ import com.intellij.internal.statistic.eventLog.events.ObjectDescription as IJOb import com.intellij.internal.statistic.eventLog.events.ObjectEventData as IJObjectEventData import com.intellij.internal.statistic.eventLog.events.ObjectEventField as IJObjectEventField import com.intellij.internal.statistic.eventLog.events.ObjectListEventField as IJObjectListEventField -import com.intellij.internal.statistic.eventLog.events.StringEventField as IJStringEventField import com.jetbrains.ml.api.logs.BooleanEventField as MLBooleanEventField import com.jetbrains.ml.api.logs.ClassEventField as MLClassEventField import com.jetbrains.ml.api.logs.DoubleEventField as MLDoubleEventField @@ -87,21 +80,18 @@ internal class ConverterObjectDescription(mlObjectDescription: MLObjectDescripti is MLDoubleEventField -> ConverterOfPrimitiveType(mlEventField) { n, d -> IJDoubleEventField(n, d) } as IJEventPairConverter is MLStringEventField -> ConverterOfString(mlEventField) as IJEventPairConverter is MLFloatListEventField -> object : IJEventPairConverter, List> { - override val ijEventField: IJEventField> = IJFloatListEventField(mlEventField.name, mlEventField.lazyDescription()) + override val ijEventField: IJEventField> = IJFloatListEventField(mlEventField.name, mlEventField.lazyDescription?.invoke()) override fun buildEventPair(mlEventPair: MLEventPair>): IJEventPair> = ijEventField with mlEventPair.data } as IJEventPairConverter is MLIntListEventField -> object : IJEventPairConverter, List> { - override val ijEventField: IJEventField> = IJIntListEventField(mlEventField.name, mlEventField.lazyDescription()) + override val ijEventField: IJEventField> = IJIntListEventField(mlEventField.name, mlEventField.lazyDescription?.invoke()) override fun buildEventPair(mlEventPair: MLEventPair>): IJEventPair> = ijEventField with mlEventPair.data } as IJEventPairConverter is MLLongListEventField -> object : IJEventPairConverter, List> { - override val ijEventField: IJEventField> = IJLongListEventField(mlEventField.name, mlEventField.lazyDescription()) + override val ijEventField: IJEventField> = IJLongListEventField(mlEventField.name, mlEventField.lazyDescription?.invoke()) override fun buildEventPair(mlEventPair: MLEventPair>): IJEventPair> = ijEventField with mlEventPair.data } as IJEventPairConverter - is VersionEventField -> ConverterOfVersion(mlEventField) as IJEventPairConverter - is LanguageEventField -> ConverterOfLanguage(mlEventField) as IJEventPairConverter - is RuleBasedStringEventField -> ConverterSensitiveString(mlEventField) as IJEventPairConverter else -> throw NotImplementedError("Please implement converter for $mlEventField from ML to IJ event fields") } @@ -131,45 +121,6 @@ internal class ConverterObjectDescription(mlObjectDescription: MLObjectDescripti private class ConverterOfString(mlEventField: MLStringEventField) : IJEventPairConverter { - override val ijEventField: IJEventField = IJStringEventField.ValidatedByAllowedValues( - mlEventField.name, - allowedValues = mlEventField.possibleValues, - description = mlEventField.lazyDescription() - ) - - override fun buildEventPair(mlEventPair: MLEventPair): IJEventPair { - return ijEventField with mlEventPair.data - } -} - -private class ConverterOfLanguage(mlEventField: LanguageEventField) : IJEventPairConverter { - override val ijEventField: IJEventField = IJEventFields.Language(mlEventField.name, mlEventField.lazyDescription()) - - override fun buildEventPair(mlEventPair: MLEventPair): IJEventPair { - return ijEventField with mlEventPair.data - } -} - - -private class ConverterOfVersion(mlEventField: com.intellij.platform.ml.impl.fus.VersionEventField) : IJEventPairConverter { - private class VersionEventField(override val name: String, override val description: String?) : PrimitiveEventField() { - override val validationRule: List - get() = listOf("{regexp#version}") - - override fun addData(fuData: FeatureUsageData, value: Version?) { - fuData.addVersion(value) - } - } - - override val ijEventField: IJEventField = VersionEventField(mlEventField.name, mlEventField.lazyDescription()) - - override fun buildEventPair(mlEventPair: MLEventPair): IJEventPair { - return ijEventField with mlEventPair.data - } -} - - -private class ConverterSensitiveString(private val mlEventField: RuleBasedStringEventField) : IJEventPairConverter { private class SensitiveStringEventField( name: String, @@ -180,11 +131,15 @@ private class ConverterSensitiveString(private val mlEventField: RuleBasedStr } override val ijEventField: IJEventField = - SensitiveStringEventField(mlEventField.name, mlEventField.ruleId!!, mlEventField.lazyDescription()) + SensitiveStringEventField( + mlEventField.name, + requireNotNull(mlEventField.ruleId) { "Error for $mlEventField: it must have a validation rule" }, + mlEventField.lazyDescription?.invoke() + ) + + override fun buildEventPair(mlEventPair: MLEventPair): EventPair = + ijEventField with mlEventPair.data - override fun buildEventPair(mlEventPair: MLEventPair): EventPair { - return ijEventField with mlEventField.serialize(mlEventPair.data) - } } @@ -207,7 +162,7 @@ private class ConvertObjectList(mlEventField: MLObjectListEventField) : private class ConverterOfEnum>(mlEnumField: MLEnumEventField) : IJEventPairConverter { - override val ijEventField: IJEventField = IJEnumEventField(mlEnumField.name, mlEnumField.enumClass, mlEnumField.transform) + override val ijEventField: IJEventField = IJEnumEventField(mlEnumField.name, mlEnumField.enumClass) { it.name } override fun buildEventPair(mlEventPair: MLEventPair): IJEventPair { return ijEventField with mlEventPair.data @@ -223,12 +178,12 @@ private class ConverterOfEnum>(mlEnumField: MLEnumEventField) : I private class ConverterOfObject( name: String, - lazyDescription: () -> String, + lazyDescription: (() -> String)?, mlObjectDescription: MLObjectDescription, ) : IJEventPairConverter { val ijObjectDescription = mlObjectDescription.asIJObjectDescription() - override val ijEventField: IJEventField = IJObjectEventField(name, lazyDescription(), ijObjectDescription) + override val ijEventField: IJEventField = IJObjectEventField(name, lazyDescription?.invoke(), ijObjectDescription) fun buildObjectEventData(mlObject: MLObjectEventData): IJObjectEventData { return ijObjectDescription.buildObjectEventData(mlObject) @@ -244,7 +199,7 @@ private class ConverterOfPrimitiveType( mlEventField: MLEventField, createIJField: (String, String?) -> IJEventField ) : IJEventPairConverter { - override val ijEventField: IJEventField = createIJField(mlEventField.name, mlEventField.lazyDescription()) + override val ijEventField: IJEventField = createIJField(mlEventField.name, mlEventField.lazyDescription?.invoke()) override fun buildEventPair(mlEventPair: MLEventPair): IJEventPair { return ijEventField with mlEventPair.data @@ -255,7 +210,7 @@ private class ConverterOfPrimitiveType( private class ConverterOfClass( mlEventField: MLClassEventField, ) : IJEventPairConverter, Class<*>?> { - override val ijEventField: IJEventField?> = IJClassEventField(mlEventField.name, mlEventField.lazyDescription()) + override val ijEventField: IJEventField?> = IJClassEventField(mlEventField.name, mlEventField.lazyDescription?.invoke()) override fun buildEventPair(mlEventPair: MLEventPair>): IJEventPair?> { return ijEventField with mlEventPair.data diff --git a/platform/ml-impl/src/com/intellij/platform/ml/impl/fus/IntelliJFusEventRegister.kt b/platform/ml-logs/src/com/intellij/platform/ml/logs/IntelliJFusEventRegister.kt similarity index 97% rename from platform/ml-impl/src/com/intellij/platform/ml/impl/fus/IntelliJFusEventRegister.kt rename to platform/ml-logs/src/com/intellij/platform/ml/logs/IntelliJFusEventRegister.kt index 20a791a6b5b4..91eaae49469c 100644 --- a/platform/ml-impl/src/com/intellij/platform/ml/impl/fus/IntelliJFusEventRegister.kt +++ b/platform/ml-logs/src/com/intellij/platform/ml/logs/IntelliJFusEventRegister.kt @@ -1,5 +1,5 @@ // Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. -package com.intellij.platform.ml.impl.fus +package com.intellij.platform.ml.logs import com.intellij.internal.statistic.eventLog.events.VarargEventId import com.jetbrains.ml.tools.logs.FusEventLogger diff --git a/platform/platform-resources/src/META-INF/common-ide-modules.xml b/platform/platform-resources/src/META-INF/common-ide-modules.xml index 4ab10d7b4b0b..bef1cf898ccf 100644 --- a/platform/platform-resources/src/META-INF/common-ide-modules.xml +++ b/platform/platform-resources/src/META-INF/common-ide-modules.xml @@ -10,6 +10,7 @@ + diff --git a/python/intellij.python.ml.features/BUILD.bazel b/python/intellij.python.ml.features/BUILD.bazel index 8e95592919ab..8cd02256b193 100644 --- a/python/intellij.python.ml.features/BUILD.bazel +++ b/python/intellij.python.ml.features/BUILD.bazel @@ -16,17 +16,19 @@ jvm_library( "@lib//:kotlin-stdlib", "//platform/core-api:core", "//platform/editor-ui-api:editor-ui", + "//platform/analysis-api:analysis", + "//platform/indexing-api:indexing", "//platform/statistics", - "//platform/ml-impl", + "//platform/projectModel-api:projectModel", + "//platform/ml-logs", "//python/python-psi-impl:psi-impl", "//platform/platform-api:ide", "//platform/platform-impl:ide-impl", "//python:python-community-impl", "//platform/util/concurrency", "//platform/core-ui", - "//platform/ml-api:ml", "//platform/platform-impl/codeinsight-inline:ide-codeinsight-inline", - "@lib//:jetbrains-ml-models-python-imports-ranking-model", + "@lib//:python-ml-features-jetbrains-ml-models-python-imports-ranking-model", ], runtime_deps = [":ml-features_resources"] ) diff --git a/python/intellij.python.ml.features/intellij.python.ml.features.iml b/python/intellij.python.ml.features/intellij.python.ml.features.iml index f297d0482ec3..4f18f73ebb4e 100644 --- a/python/intellij.python.ml.features/intellij.python.ml.features.iml +++ b/python/intellij.python.ml.features/intellij.python.ml.features.iml @@ -11,16 +11,35 @@ + + - + + - - + + + + + + cff0591184b48356a622f7635c6de1399fdbeb161f98d68206d09807c83fa98c + + + + + + + + + + + + \ No newline at end of file diff --git a/python/intellij.python.ml.features/resources/intellij.python.ml.features.xml b/python/intellij.python.ml.features/resources/intellij.python.ml.features.xml index 42240ed72736..fdca7e7ecdda 100644 --- a/python/intellij.python.ml.features/resources/intellij.python.ml.features.xml +++ b/python/intellij.python.ml.features/resources/intellij.python.ml.features.xml @@ -1,13 +1,15 @@ - + + + diff --git a/python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/BaseProjectFeatures.kt b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/BaseProjectFeatures.kt similarity index 86% rename from python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/BaseProjectFeatures.kt rename to python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/BaseProjectFeatures.kt index e61d4da4bd21..b216b651e961 100644 --- a/python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/BaseProjectFeatures.kt +++ b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/BaseProjectFeatures.kt @@ -1,6 +1,6 @@ // Optimized and improved Kotlin code -package com.jetbrains.python.codeInsight.imports.mlapi.features +package com.intellij.python.ml.features.imports.features import com.intellij.openapi.application.readAction import com.intellij.openapi.fileEditor.FileEditorManager @@ -12,8 +12,6 @@ import com.intellij.psi.search.GlobalSearchScope import com.intellij.psi.util.PsiTreeUtil import com.jetbrains.ml.api.feature.* import com.jetbrains.python.PythonFileType -import com.jetbrains.python.codeInsight.imports.mlapi.ImportRankingContext -import com.jetbrains.python.codeInsight.imports.mlapi.ImportRankingContextFeatures import com.jetbrains.python.psi.* private val interestingClasses = arrayOf( @@ -39,14 +37,14 @@ object BaseProjectFeatures : ImportRankingContextFeatures() { "The estimated amount of files in the project (by a power of 2)" }.nullable() val PSI_PARENT_OF_ORIG: List?>> = (1..5).map { i -> FeatureDeclaration.aClass("psi_parent_of_orig_$i") { "PSI parent of original element #$i" }.nullable() } - val FILE_EXTENSION_TYPE: FeatureDeclaration = FeatureDeclaration.enum("file_extension_type") { "extension of the original python file" }.nullable() + val FILE_EXTENSION_TYPE: FeatureDeclaration?> = FeatureDeclaration.enum("file_extension_type") { "extension of the original python file" }.nullable() } override val featureComputationPolicy: FeatureComputationPolicy = FeatureComputationPolicy(true, true) override val namespaceFeatureDeclarations: List> = extractFeatureDeclarations(Features) - override suspend fun computeNamespaceFeatures(instance: ImportRankingContext, filter: FeatureFilter): List = buildList { + override suspend fun computeNamespaceFeatures(instance: ImportRankingContext, filter: FeatureSet): List = buildList { val candidates = instance.candidates if (candidates.isEmpty()) return@buildList diff --git a/python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/CandidatesListFeatures.kt b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/CandidatesListFeatures.kt similarity index 79% rename from python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/CandidatesListFeatures.kt rename to python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/CandidatesListFeatures.kt index 54c440066820..f95f0473f252 100644 --- a/python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/CandidatesListFeatures.kt +++ b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/CandidatesListFeatures.kt @@ -1,9 +1,7 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. -package com.jetbrains.python.codeInsight.imports.mlapi.features +package com.intellij.python.ml.features.imports.features import com.jetbrains.ml.api.feature.* -import com.jetbrains.python.codeInsight.imports.mlapi.ImportRankingContext -import com.jetbrains.python.codeInsight.imports.mlapi.ImportRankingContextFeatures object CandidatesListFeatures : ImportRankingContextFeatures() { object Features { @@ -19,7 +17,7 @@ object CandidatesListFeatures : ImportRankingContextFeatures() { override val featureComputationPolicy: FeatureComputationPolicy = FeatureComputationPolicy(tolerateRedundantFeatures = true, putNullImplicitly = true) - override suspend fun computeNamespaceFeatures(instance: ImportRankingContext, filter: FeatureFilter): List = buildList { + override suspend fun computeNamespaceFeatures(instance: ImportRankingContext, filter: FeatureSet): List = buildList { add(Features.LENGTH with instance.candidates.size) add(Features.HIGHEST_OLD_RELEVANCE with instance.candidates.maxOfOrNull { it.relevance }) } diff --git a/python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/ImportCandidateRelativeFeatures.kt b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/ImportCandidateRelativeFeatures.kt similarity index 73% rename from python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/ImportCandidateRelativeFeatures.kt rename to python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/ImportCandidateRelativeFeatures.kt index 856fb24e9bbb..3187c6b16514 100644 --- a/python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/ImportCandidateRelativeFeatures.kt +++ b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/ImportCandidateRelativeFeatures.kt @@ -1,9 +1,7 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. -package com.jetbrains.python.codeInsight.imports.mlapi.features +package com.intellij.python.ml.features.imports.features import com.jetbrains.ml.api.feature.* -import com.jetbrains.python.codeInsight.imports.mlapi.ImportCandidateContext -import com.jetbrains.python.codeInsight.imports.mlapi.ImportCandidateFeatures object ImportCandidateRelativeFeatures : ImportCandidateFeatures() { object Features { @@ -16,7 +14,7 @@ object ImportCandidateRelativeFeatures : ImportCandidateFeatures() { override val featureComputationPolicy: FeatureComputationPolicy = FeatureComputationPolicy(tolerateRedundantFeatures = true, putNullImplicitly = true) - override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureFilter): List = buildList { + override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureSet): List = buildList { add(Features.RELATIVE_POSITION with instance.candidates.indexOf(instance.candidate)) } } diff --git a/python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/ImportsFeatures.kt b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/ImportsFeatures.kt similarity index 92% rename from python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/ImportsFeatures.kt rename to python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/ImportsFeatures.kt index 868614bd4056..9040aebc4683 100644 --- a/python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/ImportsFeatures.kt +++ b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/ImportsFeatures.kt @@ -1,5 +1,5 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. -package com.jetbrains.python.codeInsight.imports.mlapi.features +package com.intellij.python.ml.features.imports.features import com.intellij.openapi.application.readAction import com.intellij.openapi.fileEditor.FileEditorManager @@ -7,8 +7,6 @@ import com.intellij.psi.PsiFile import com.intellij.psi.PsiManager import com.intellij.psi.util.QualifiedName import com.jetbrains.ml.api.feature.* -import com.jetbrains.python.codeInsight.imports.mlapi.ImportCandidateContext -import com.jetbrains.python.codeInsight.imports.mlapi.ImportCandidateFeatures import com.jetbrains.python.psi.PyFile import com.jetbrains.python.psi.PyFromImportStatement import com.jetbrains.python.psi.PyImportElement @@ -25,7 +23,7 @@ object ImportsFeatures : ImportCandidateFeatures() { override val namespaceFeatureDeclarations: List> = extractFeatureDeclarations(Features) - override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureFilter): List = buildList { + override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureSet): List = buildList { val importCandidate = instance.candidate val project = readAction { importCandidate.importable?.project } ?: return@buildList val editor = readAction { FileEditorManager.getInstance(project).selectedEditor } ?: return@buildList diff --git a/python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/NeighborFilesImportsFeatures.kt b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/NeighborFilesImportsFeatures.kt similarity index 93% rename from python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/NeighborFilesImportsFeatures.kt rename to python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/NeighborFilesImportsFeatures.kt index 9c57669a8768..78ee32c5f859 100644 --- a/python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/NeighborFilesImportsFeatures.kt +++ b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/NeighborFilesImportsFeatures.kt @@ -1,13 +1,11 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. -package com.jetbrains.python.codeInsight.imports.mlapi.features +package com.intellij.python.ml.features.imports.features import com.intellij.openapi.application.readAction import com.intellij.openapi.fileEditor.FileEditorManager import com.intellij.openapi.vfs.VirtualFile import com.intellij.psi.PsiManager import com.jetbrains.ml.api.feature.* -import com.jetbrains.python.codeInsight.imports.mlapi.ImportCandidateContext -import com.jetbrains.python.codeInsight.imports.mlapi.ImportCandidateFeatures import com.jetbrains.python.psi.PyFile import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope @@ -26,7 +24,7 @@ object NeighborFilesImportsFeatures : ImportCandidateFeatures() { override val namespaceFeatureDeclarations: List> = extractFeatureDeclarations(Features) - override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureFilter): List = coroutineScope { + override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureSet): List = coroutineScope { val importCandidate = instance.candidate if (importCandidate.path == null) { return@coroutineScope emptyList() diff --git a/python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/OpenFilesImportsFeatures.kt b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/OpenFilesImportsFeatures.kt similarity index 92% rename from python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/OpenFilesImportsFeatures.kt rename to python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/OpenFilesImportsFeatures.kt index 30251c22b34f..44a6beede7d7 100644 --- a/python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/OpenFilesImportsFeatures.kt +++ b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/OpenFilesImportsFeatures.kt @@ -1,12 +1,10 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. -package com.jetbrains.python.codeInsight.imports.mlapi.features +package com.intellij.python.ml.features.imports.features import com.intellij.openapi.application.readAction import com.intellij.openapi.fileEditor.FileEditorManager import com.intellij.psi.PsiManager import com.jetbrains.ml.api.feature.* -import com.jetbrains.python.codeInsight.imports.mlapi.ImportCandidateContext -import com.jetbrains.python.codeInsight.imports.mlapi.ImportCandidateFeatures import com.jetbrains.python.psi.PyFile import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope @@ -25,7 +23,7 @@ object OpenFilesImportsFeatures : ImportCandidateFeatures() { override val namespaceFeatureDeclarations: List> = extractFeatureDeclarations(Features) - override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureFilter): List = coroutineScope { + override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureSet): List = coroutineScope { val importCandidate = instance.candidate if (importCandidate.path == null) { return@coroutineScope emptyList() diff --git a/python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/PrimitiveImportFeatures.kt b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/PrimitiveImportFeatures.kt similarity index 79% rename from python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/PrimitiveImportFeatures.kt rename to python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/PrimitiveImportFeatures.kt index 20a01c45caeb..7895917b108a 100644 --- a/python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/PrimitiveImportFeatures.kt +++ b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/PrimitiveImportFeatures.kt @@ -1,9 +1,7 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. -package com.jetbrains.python.codeInsight.imports.mlapi.features +package com.intellij.python.ml.features.imports.features import com.jetbrains.ml.api.feature.* -import com.jetbrains.python.codeInsight.imports.mlapi.ImportCandidateContext -import com.jetbrains.python.codeInsight.imports.mlapi.ImportCandidateFeatures object PrimitiveImportFeatures : ImportCandidateFeatures() { @@ -22,7 +20,7 @@ object PrimitiveImportFeatures : ImportCandidateFeatures() { override val featureComputationPolicy: FeatureComputationPolicy = FeatureComputationPolicy(tolerateRedundantFeatures = true, putNullImplicitly = true) - override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureFilter): List = buildList { + override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureSet): List = buildList { add(Features.RELEVANCE with instance.candidate.relevance) add(Features.COMPONENT_COUNT with instance.candidate.path?.componentCount) } diff --git a/python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/PsiStructureFeatures.kt b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/PsiStructureFeatures.kt similarity index 82% rename from python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/PsiStructureFeatures.kt rename to python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/PsiStructureFeatures.kt index 4c587bc13fbe..982d3b015045 100644 --- a/python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/PsiStructureFeatures.kt +++ b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/PsiStructureFeatures.kt @@ -1,11 +1,9 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. -package com.jetbrains.python.codeInsight.imports.mlapi.features +package com.intellij.python.ml.features.imports.features import com.intellij.openapi.application.readAction import com.intellij.psi.PsiElement import com.jetbrains.ml.api.feature.* -import com.jetbrains.python.codeInsight.imports.mlapi.ImportCandidateContext -import com.jetbrains.python.codeInsight.imports.mlapi.ImportCandidateFeatures object PsiStructureFeatures : ImportCandidateFeatures() { @@ -18,7 +16,7 @@ object PsiStructureFeatures : ImportCandidateFeatures() { override val featureComputationPolicy: FeatureComputationPolicy = FeatureComputationPolicy(tolerateRedundantFeatures = true, putNullImplicitly = true) - override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureFilter): List = buildList { + override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureSet): List = buildList { readAction { add(Features.PSI_CLASS with (instance.candidate.importable?.javaClass)) Features.PSI_PARENT.withIndex().forEach { (i, featureDeclaration) -> diff --git a/python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/RelevanceEvaluationFeatures.kt b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/RelevanceEvaluationFeatures.kt similarity index 84% rename from python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/RelevanceEvaluationFeatures.kt rename to python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/RelevanceEvaluationFeatures.kt index 5809ece1f6fa..8625c39f03cd 100644 --- a/python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/features/RelevanceEvaluationFeatures.kt +++ b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/RelevanceEvaluationFeatures.kt @@ -1,5 +1,5 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. -package com.jetbrains.python.codeInsight.imports.mlapi.features +package com.intellij.python.ml.features.imports.features import com.intellij.openapi.application.readAction import com.intellij.openapi.module.ModuleUtilCore @@ -7,8 +7,6 @@ import com.intellij.openapi.projectRoots.Sdk import com.intellij.openapi.vfs.VirtualFile import com.intellij.psi.util.QualifiedName import com.jetbrains.ml.api.feature.* -import com.jetbrains.python.codeInsight.imports.mlapi.ImportCandidateContext -import com.jetbrains.python.codeInsight.imports.mlapi.ImportCandidateFeatures import com.jetbrains.python.sdk.PythonSdkUtil enum class UnderscoresType { @@ -32,15 +30,15 @@ enum class ModuleSourceType { object RelevanceEvaluationFeatures : ImportCandidateFeatures() { object Features { val UNDERSCORES_IN_PATH: FeatureDeclaration = FeatureDeclaration.int("underscores_in_path") { "number of prefix and suffix underscores in path" }.nullable() - val MODULE_SOURCE_TYPE: FeatureDeclaration = FeatureDeclaration.enum("module_source_type") { "info about lib being std, local, or external" }.nullable() - val UNDERSCORES_TYPES_OF_PACKAGES: List> = (1..4).map { i -> FeatureDeclaration.enum("underscores_types_of_package_$i") { "underscores types of package #$i" }.nullable() } + val MODULE_SOURCE_TYPE: FeatureDeclaration?> = FeatureDeclaration.enum("module_source_type") { "info about lib being std, local, or external" }.nullable() + val UNDERSCORES_TYPES_OF_PACKAGES: List?>> = (1..4).map { i -> FeatureDeclaration.enum("underscores_types_of_package_$i") { "underscores types of package #$i" }.nullable() } } override val featureComputationPolicy: FeatureComputationPolicy = FeatureComputationPolicy(true, true) override val namespaceFeatureDeclarations: List> = extractFeatureDeclarations(Features) - override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureFilter): List = buildList { + override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureSet): List = buildList { val importCandidate = instance.candidate add(Features.UNDERSCORES_IN_PATH with countBoundaryUnderscores(importCandidate.path)) readAction { diff --git a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlFeatures.kt b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/mlFeatures.kt similarity index 71% rename from python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlFeatures.kt rename to python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/mlFeatures.kt index df25ac9b6ab5..d4e15fdeca74 100644 --- a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlFeatures.kt +++ b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/mlFeatures.kt @@ -1,17 +1,12 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. -package com.intellij.python.ml.features.imports +package com.intellij.python.ml.features.imports.features import com.jetbrains.ml.api.feature.Feature import com.jetbrains.ml.api.feature.FeatureDeclaration -import com.jetbrains.ml.api.feature.FeatureFilter -import com.jetbrains.ml.api.feature.FeatureValueType +import com.jetbrains.ml.api.feature.FeatureSet import com.jetbrains.ml.api.feature.suspendable.FeatureProvider -import com.jetbrains.python.codeInsight.imports.mlapi.ImportCandidateContext -import com.jetbrains.python.codeInsight.imports.mlapi.ImportRankingContext -import com.jetbrains.python.codeInsight.imports.mlapi.features.* import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope -import kotlin.collections.plus internal object FeaturesRegistry { @@ -30,7 +25,7 @@ internal object FeaturesRegistry { OpenFilesImportsFeatures, ) - suspend fun computeContextFeatures(instance: ImportRankingContext, filter: FeatureFilter): List = coroutineScope { + suspend fun computeContextFeatures(instance: ImportRankingContext, filter: FeatureSet): List = coroutineScope { val jobs = context.map { provider -> async { provider.computeFeaturesWithImplicitNull(instance, filter) @@ -39,7 +34,7 @@ internal object FeaturesRegistry { jobs.flatMap { it.await() } } - suspend fun computeCandidateFeatures(instance: ImportCandidateContext, filter: FeatureFilter): List = coroutineScope { + suspend fun computeCandidateFeatures(instance: ImportCandidateContext, filter: FeatureSet): List = coroutineScope { val jobs = candidate.map { provider -> async { provider.computeFeaturesWithImplicitNull(instance, filter) @@ -50,23 +45,21 @@ internal object FeaturesRegistry { private suspend fun FeatureProvider.computeFeaturesWithImplicitNull( instance: T, - filter: FeatureFilter - ): List { + filter: FeatureSet + ): Collection { val features = provideFeatures(instance, filter) if (!featureComputationPolicy.putNullImplicitly) { return features } val implicitNullFeatures = mutableListOf() for (declaration in featureDeclarations) { - if (declaration.isNullable() && features.all { feature -> feature.declaration != declaration }) { + if (declaration.isNullable && !filter.contains(declaration)) { implicitNullFeatures.add(declaration.asNullable() with null) } } return features + implicitNullFeatures } - private fun FeatureDeclaration<*>.isNullable(): Boolean = type is FeatureValueType.Nullable<*> - @Suppress("UNCHECKED_CAST") private fun FeatureDeclaration.asNullable(): FeatureDeclaration = this as FeatureDeclaration diff --git a/python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/mlUnits.kt b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/mlUnits.kt similarity index 92% rename from python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/mlUnits.kt rename to python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/mlUnits.kt index 5c58b93b7cc3..19154354dc6d 100644 --- a/python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/mlapi/mlUnits.kt +++ b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/mlUnits.kt @@ -1,5 +1,5 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. -package com.jetbrains.python.codeInsight.imports.mlapi +package com.intellij.python.ml.features.imports.features import com.jetbrains.ml.api.feature.suspendable.FeatureProvider import com.jetbrains.python.codeInsight.imports.ImportCandidateHolder diff --git a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlAB.kt b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlAB.kt index 0dc7345c3842..ee93f73e9440 100644 --- a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlAB.kt +++ b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlAB.kt @@ -5,9 +5,9 @@ import com.intellij.internal.statistic.eventLog.EventLogConfiguration import com.intellij.openapi.components.Service import com.intellij.openapi.components.service import com.intellij.openapi.util.registry.Registry -import com.intellij.platform.ml.impl.logs.MLEventLoggerProvider.Companion.ML_RECORDER_ID import com.intellij.python.ml.features.imports.FinalImportRankingStatusService.RegistryOption -import com.jetbrains.ml.api.model.MLModel +import com.jetbrains.ml.tools.model.MLModel +import com.jetbrains.ml.tools.model.catboost.prediction.CatBoostRegressionResult @Service internal class FinalImportRankingStatusService { @@ -20,9 +20,7 @@ internal class FinalImportRankingStatusService { } private val bucket: Int - by lazy { service().getOrCreate(ML_RECORDER_ID, - // ids between ML and FUS recorders should match - "FUS").bucket } + by lazy { service().getOrCreate("ML").bucket } private val mlEnabledOnBucket: Boolean by lazy { bucket % 2 == 0 } @@ -31,7 +29,7 @@ internal class FinalImportRankingStatusService { val status: FinalImportRankingStatus get() { - val mlModel = service().getModelOwnership() + val mlModel = service().model val registryOption = getRegistryOption() val registryOptionAllowsEnabling = registryOption == RegistryOption.ENABLED || (registryOption == RegistryOption.IN_EXPERIMENT && mlEnabledOnBucket) return if (mlModel != null && registryOptionAllowsEnabling) @@ -49,6 +47,6 @@ internal sealed class FinalImportRankingStatus( val mlModelUnavailable: Boolean, val registryOption: RegistryOption, ) { - class Enabled(val mlModel: MLModel, registryOption: RegistryOption) : FinalImportRankingStatus(true, false, registryOption) + class Enabled(val mlModel: MLModel, registryOption: RegistryOption) : FinalImportRankingStatus(true, false, registryOption) class Disabled(mlModelUnavailable: Boolean, registryOption: RegistryOption) : FinalImportRankingStatus(false, mlModelUnavailable, registryOption) } diff --git a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlAnalysis.kt b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlAnalysis.kt index 141fb6dc1826..f5b34b1a2c22 100644 --- a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlAnalysis.kt +++ b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlAnalysis.kt @@ -8,15 +8,15 @@ import com.jetbrains.ml.api.logs.LongEventField internal object ContextAnalysis { - val CANCELLED = BooleanEventField("selection_cancelled") { "No item has been selected" } + val CANCELLED = BooleanEventField("selection_cancelled", null) { "No item has been selected" } val SELECTED_POSITION = IntEventField("selected_position") { "The position of the selected import statement" } - val MODEL_UNAVAILABLE = BooleanEventField("model_unavailable") { "ML model was unavailable" } + val MODEL_UNAVAILABLE = BooleanEventField("model_unavailable", null) { "ML model was unavailable" } val SELECTED_POSITION_INITIAL = IntEventField("selected_position_initial") { "The position of the selected import statement, if the final ML ranking would not have happened" } val TIME_MS_TO_DISPLAY = LongEventField("time_ms_before_displayed") { "Duration from the quickfix start until when the imports were displayed" } val TIME_MS_BEFORE_CLOSED = LongEventField("time_ms_before_closed") { "Duration from the quickfix start until the pop-up was closed" } - val ML_LOGGING_STATE = EnumEventField.of("ml_logging_state", { "State of the ML session logging" }) + val ML_LOGGING_STATE = EnumEventField("ml_logging_state", null, { "State of the ML session logging" }, LoggingOption::class.java) val ML_ENABLED = BooleanEventField("ml_enabled") { "Machine Learning ranking is enabled" } - val REGISTRY_OPTION = EnumEventField.of("registry_option", { "Registry option of the experiment status" }) + val REGISTRY_OPTION = EnumEventField("registry_option", null, { "Registry option of the experiment status" }, FinalImportRankingStatusService.RegistryOption::class.java) } internal object CandidateAnalysis { diff --git a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlImplementation.kt b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlImplementation.kt index 9f3c0acc90b6..ba3dfe058e8f 100644 --- a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlImplementation.kt +++ b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlImplementation.kt @@ -6,14 +6,17 @@ import com.intellij.openapi.components.Service import com.intellij.openapi.components.service import com.intellij.openapi.diagnostic.debug import com.intellij.openapi.diagnostic.logger +import com.intellij.python.ml.features.imports.features.FeaturesRegistry +import com.intellij.python.ml.features.imports.features.ImportCandidateContext +import com.intellij.python.ml.features.imports.features.ImportRankingContext import com.intellij.util.application import com.jetbrains.ml.api.feature.Feature +import com.jetbrains.ml.api.feature.FeatureDeclaration import com.jetbrains.ml.api.logs.EventPair -import com.jetbrains.ml.api.model.MLModel import com.jetbrains.ml.tools.logs.MLLogsTree +import com.jetbrains.ml.tools.model.MLModel +import com.jetbrains.ml.tools.model.catboost.prediction.CatBoostRegressionResult import com.jetbrains.python.codeInsight.imports.ImportCandidateHolder -import com.jetbrains.python.codeInsight.imports.mlapi.ImportCandidateContext -import com.jetbrains.python.codeInsight.imports.mlapi.ImportRankingContext import kotlinx.coroutines.* import java.util.concurrent.atomic.AtomicBoolean import kotlin.time.Duration.Companion.seconds @@ -39,23 +42,23 @@ private class ExperimentalMLRanker( contextFeatures: MutableList, contextAnalysis: MutableList>, timestampStarted: Long, - private val mlModel: MLModel, + private val mlModel: MLModel, ) : FinalCandidatesRanker(contextFeatures, contextAnalysis, timestampStarted) { override val mlEnabled = true override fun launchMLRanking(initialCandidatesOrder: MutableList, displayResult: (RateableRankingResult) -> Unit) { service().coroutineScope.launch(Dispatchers.Default) { - contextFeatures.addAll(FeaturesRegistry.computeContextFeatures(ImportRankingContext(initialCandidatesOrder), mlModel.knownFeatures)) + contextFeatures.addAll(FeaturesRegistry.computeContextFeatures(ImportRankingContext(initialCandidatesOrder), mlModel.inputFeatures)) val importCandidatesFeatures = initialCandidatesOrder.associateWith { candidate -> - async { FeaturesRegistry.computeCandidateFeatures(ImportCandidateContext(initialCandidatesOrder, candidate), mlModel.knownFeatures) } + async { FeaturesRegistry.computeCandidateFeatures(ImportCandidateContext(initialCandidatesOrder, candidate), mlModel.inputFeatures) } }.mapValues { it.value.await() } - val scoreByCandidate = mlModel.predictBatch(contextFeatures, importCandidatesFeatures) + val scoreByCandidate = mlModel.predictBatch(contextFeatures + fillTypingFeatures(), importCandidatesFeatures) val relevanceCandidateOrder = scoreByCandidate.toList() - .sortedByDescending { it.second } + .sortedByDescending { it.second.logit } .map { it.first } logger().debug { @@ -175,3 +178,21 @@ internal class RateableRankingResult( } } } + +private object TypingFeatures { + val SINCE_LAST_TYPING = FeatureDeclaration.int("typing_speed_tracker_time_since_last_typing") { "Deprecated" } + val TYPING_SPEED_1S = FeatureDeclaration.double("typing_speed_tracker_typing_speed_1s") { "Deprecated" } + val TYPING_SPEED_2S = FeatureDeclaration.double("typing_speed_tracker_typing_speed_2s") { "Deprecated" } + val TYPING_SPEED_5S = FeatureDeclaration.double("typing_speed_tracker_typing_speed_5s") { "Deprecated" } + val TYPING_SPEED_30S = FeatureDeclaration.double("typing_speed_tracker_typing_speed_30s") { "Deprecated" } +} + +private fun fillTypingFeatures(): List { + return listOf( + TypingFeatures.SINCE_LAST_TYPING with 1, + TypingFeatures.TYPING_SPEED_2S with 0.0, + TypingFeatures.TYPING_SPEED_30S with 0.0, + TypingFeatures.TYPING_SPEED_5S with 0.0, + TypingFeatures.TYPING_SPEED_1S with 0.0, + ) +} \ No newline at end of file diff --git a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlLogs.kt b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlLogs.kt index b0aeb15f3cc6..40f88aef5264 100644 --- a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlLogs.kt +++ b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlLogs.kt @@ -3,8 +3,8 @@ package com.intellij.python.ml.features.imports import com.intellij.internal.statistic.eventLog.EventLogGroup import com.intellij.internal.statistic.service.fus.collectors.CounterUsagesCollector -import com.intellij.platform.ml.impl.fus.IntelliJFusEventRegister -import com.intellij.platform.ml.impl.logs.MLEventLoggerProvider.Companion.ML_RECORDER_ID +import com.intellij.platform.ml.logs.IntelliJFusEventRegister +import com.intellij.python.ml.features.imports.features.FeaturesRegistry import com.intellij.util.application import com.jetbrains.ml.tools.logs.MLTreeLoggers.withOneEvent import com.jetbrains.ml.tools.logs.extractEventFields @@ -12,7 +12,7 @@ import kotlin.random.Random internal object PyCharmImportsRankingLogs : CounterUsagesCollector() { - private val GROUP = EventLogGroup("pycharm.quickfix.imports", 7, ML_RECORDER_ID) + private val GROUP = EventLogGroup("pycharm.quickfix.imports", 7, "ML") val mlLogger = withOneEvent( fusEventName = "pycharm_import_statements_ranking", fusEventRegister = IntelliJFusEventRegister(GROUP), diff --git a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlModel.kt b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlModel.kt index a3bfa6af092c..4014b4f72cef 100644 --- a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlModel.kt +++ b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlModel.kt @@ -2,73 +2,58 @@ package com.intellij.python.ml.features.imports import com.intellij.openapi.components.Service -import com.jetbrains.ml.api.feature.Feature -import com.jetbrains.ml.api.feature.FeatureDeclaration -import com.jetbrains.ml.api.feature.FeatureFilter -import com.jetbrains.ml.api.feature.extractFeatureDeclarations -import com.jetbrains.ml.api.model.MLModel -import com.jetbrains.ml.api.model.MLModelLoader +import com.intellij.openapi.components.service +import com.intellij.openapi.diagnostic.logger +import com.intellij.openapi.project.Project +import com.intellij.openapi.startup.ProjectActivity +import com.intellij.util.concurrency.AppExecutorUtil import com.jetbrains.ml.models.PythonImportsRankingModelHolder -import com.jetbrains.ml.tools.model.MLModelLoaders -import com.jetbrains.ml.tools.model.ModelDistributionReaders -import com.jetbrains.ml.tools.model.catboost.CatBoostDistributionFormat -import com.jetbrains.ml.tools.model.suspendable.MLModelSuspendableService +import com.jetbrains.ml.tools.model.MLModel +import com.jetbrains.ml.tools.model.catboost.prediction.CatBoostRegressionResult +import com.jetbrains.ml.tools.model.pipeline.ModelPipeline +import com.jetbrains.ml.tools.model.pipeline.ModelPipelineLoader import java.util.concurrent.CompletableFuture -import java.util.concurrent.ExecutorService -@Service -class ImportsRankingModelService : MLModelSuspendableService, Double>( - MissingTypingFeaturesLoader(MLModelLoaders.Regressor( - reader = ModelDistributionReaders.FromJavaResources(PythonImportsRankingModelHolder::class.java, PythonImportsRankingModelHolder.MODEL_ID), - format = CatBoostDistributionFormat(), - )) -) +@Service(Service.Level.APP) +internal class ImportsRankingModelService { -private class MissingTypingFeaturesLoader(private val baseLoader: MLModelLoader, Double>) : MLModelLoader, Double> { + init { + loadModel() + } - override fun loadModel(executor: ExecutorService, parameters: Map?): CompletableFuture> { - return baseLoader.loadModel(executor, parameters).thenApply { rawModel -> - return@thenApply object : MLModel { - override fun predict(features: List, parameters: Map): Double { - return rawModel.predict(features + fillTypingFeatures(), parameters) - } + private var modelFuture: CompletableFuture? = null - override fun predictBatch(contextFeatures: List, itemFeatures: List>, parameters: Map): List { - return rawModel.predictBatch(contextFeatures + fillTypingFeatures(), itemFeatures, parameters) - } + var model: ModelPipeline, CatBoostRegressionResult>? = null + private set - override val knownFeatures: FeatureFilter = object : FeatureFilter { - override fun accept(featureDeclaration: FeatureDeclaration<*>): Boolean { - return if (featureDeclaration in allTypingFeatures) { - return false - } - else { - rawModel.knownFeatures.accept(featureDeclaration) - } - } - } - } + private fun loadModel() { + modelFuture?.cancel(true) + + LOG.info("Loading CatBoost Imports Ranking model") + modelFuture = ModelPipelineLoader.load( + PythonImportsRankingModelHolder.getStream(), + AppExecutorUtil.getAppExecutorService() + ).thenAccept { model -> + LOG.info("Successfully loaded imports ranking model") + this.model = model.predicting() + }.exceptionally { e -> + LOG.warn("Failed to load CatBoost imports ranking model", e) + null } } + + companion object { + @JvmStatic + fun getInstance(): ImportsRankingModelService = service() + + private val LOG = logger() + } } -private object TypingFeatures { - val SINCE_LAST_TYPING = FeatureDeclaration.int("typing_speed_tracker_time_since_last_typing") { "Deprecated" } - val TYPING_SPEED_1S = FeatureDeclaration.double("typing_speed_tracker_typing_speed_1s") { "Deprecated" } - val TYPING_SPEED_2S = FeatureDeclaration.double("typing_speed_tracker_typing_speed_2s") { "Deprecated" } - val TYPING_SPEED_5S = FeatureDeclaration.double("typing_speed_tracker_typing_speed_5s") { "Deprecated" } - val TYPING_SPEED_30S = FeatureDeclaration.double("typing_speed_tracker_typing_speed_30s") { "Deprecated" } -} - -private val allTypingFeatures: List> = extractFeatureDeclarations(TypingFeatures) - -private fun fillTypingFeatures(): List { - return listOf( - TypingFeatures.SINCE_LAST_TYPING with 1, - TypingFeatures.TYPING_SPEED_2S with 0.0, - TypingFeatures.TYPING_SPEED_30S with 0.0, - TypingFeatures.TYPING_SPEED_5S with 0.0, - TypingFeatures.TYPING_SPEED_1S with 0.0, - ) +private class QuickfixRankingModelLoading : ProjectActivity { + override suspend fun execute(project: Project) { + if (!service().shouldLoadModel) return + ImportsRankingModelService.getInstance() + } } diff --git a/python/pluginCore/plugin-content.yaml b/python/pluginCore/plugin-content.yaml index 201d5e11dde4..b863e0a37bfb 100644 --- a/python/pluginCore/plugin-content.yaml +++ b/python/pluginCore/plugin-content.yaml @@ -8,11 +8,6 @@ files: - name: $MAVEN_REPOSITORY$/io/github/z4kn4fein/semver-jvm/2/semver-jvm-2.jar reason: <- intellij.python.community.impl -- name: lib/jetbrains.ml.models.python.imports.ranking.model.jar - library: jetbrains.ml.models.python.imports.ranking.model - files: - - name: $MAVEN_REPOSITORY$/com/jetbrains/ml/models/python-imports-ranking-model/nebulous-albatross/python-imports-ranking-model-nebulous-albatross.jar - reason: <- intellij.python.ml.features - name: lib/jsr305.jar library: jsr305 files: @@ -35,6 +30,12 @@ module: intellij.python.community.impl files: - name: $MAVEN_REPOSITORY$/completion/ml/python/features/ml-completion-prev-exprs-models/1/ml-completion-prev-exprs-models-1.jar +- name: lib/modules/intellij.python.ml.features.jar + contentModules: + - name: intellij.python.ml.features + libraries: + jetbrains.ml.models.python.imports.ranking.model: + - name: $MAVEN_REPOSITORY$/com/jetbrains/ml/models/python-imports-ranking-model/coral-panda-republished-4/python-imports-ranking-model-coral-panda-republished-4.jar - name: lib/python-ce-frontend.jar contentModules: - name: intellij.python.parser @@ -75,7 +76,6 @@ - name: intellij.python.grazie - name: intellij.python.langInjection - name: intellij.python.markdown - - name: intellij.python.ml.features - name: intellij.python.terminal - name: lib/tuweni-toml.jar library: tuweni-toml diff --git a/python/python-psi-impl/BUILD.bazel b/python/python-psi-impl/BUILD.bazel index 4a748571bc23..e14a4129f68c 100644 --- a/python/python-psi-impl/BUILD.bazel +++ b/python/python-psi-impl/BUILD.bazel @@ -48,7 +48,6 @@ jvm_library( "//platform/backend/workspace", "//python/python-parser:parser", "//python/python-syntax-core:syntax-core", - "//platform/ml-api:ml", "//python/impl.helperLocator:community-helpersLocator", ], exports = ["//python/python-syntax-core:syntax-core"], diff --git a/python/python-psi-impl/intellij.python.psi.impl.iml b/python/python-psi-impl/intellij.python.psi.impl.iml index f91a76e2b130..834c3024683e 100644 --- a/python/python-psi-impl/intellij.python.psi.impl.iml +++ b/python/python-psi-impl/intellij.python.psi.impl.iml @@ -44,7 +44,6 @@ - \ No newline at end of file