diff --git a/.idea/libraries/jetbrains_mlapi_ml_api.xml b/.idea/libraries/jetbrains_mlapi_catboost.xml similarity index 55% rename from .idea/libraries/jetbrains_mlapi_ml_api.xml rename to .idea/libraries/jetbrains_mlapi_catboost.xml index 32783d55c678..d5b858594ef0 100644 --- a/.idea/libraries/jetbrains_mlapi_ml_api.xml +++ b/.idea/libraries/jetbrains_mlapi_catboost.xml @@ -1,23 +1,20 @@ - - + + - - 1050f9b58c8d71833d075ae7302881cb9d905dcda4025a21c59657ac823ac105 + + 81d40889aba4f259f59787bf9d953501668fc4f49081c75792618bba1cc6c661 - - - - + - + - + \ No newline at end of file diff --git a/.idea/libraries/jetbrains_mlapi_ml_tools.xml b/.idea/libraries/jetbrains_mlapi_core.xml similarity index 53% rename from .idea/libraries/jetbrains_mlapi_ml_tools.xml rename to .idea/libraries/jetbrains_mlapi_core.xml index b23f52b39a5b..9381e6551688 100644 --- a/.idea/libraries/jetbrains_mlapi_ml_tools.xml +++ b/.idea/libraries/jetbrains_mlapi_core.xml @@ -1,23 +1,20 @@ - - + + - - b2d36677ba73f2f7974199823b5f4323b2a16ea718ed506297402c6a7605af1c + + 6e07a48c74982f31a55b5099791e45b6af42a8e540c591732a10f8493abb56fe - - - - + - + - + \ No newline at end of file diff --git a/lib/BUILD.bazel b/lib/BUILD.bazel index 709934f6425f..c0865a6d3147 100644 --- a/lib/BUILD.bazel +++ b/lib/BUILD.bazel @@ -2827,6 +2827,13 @@ jvm_import( visibility = ["//visibility:public"] ) +jvm_import( + name = "jetbrains-mlapi-catboost", + jar = "@com_jetbrains_mlapi-mlapi-catboost-0_2_0_http//file", + source_jar = "@com_jetbrains_mlapi-mlapi-catboost-0_2_0_http//file", + visibility = ["//visibility:public"] +) + jvm_import( name = "jetbrains-mlapi-catboost-shadow-need-slf4j", jar = "@com_jetbrains_mlapi-catboost-shadow-need-slf4j-1_2_5_http//file", @@ -2834,16 +2841,9 @@ jvm_import( ) jvm_import( - name = "jetbrains-mlapi-ml-api", - jar = "@com_jetbrains_mlapi-ml-api-0_1_94_http//file", - source_jar = "@com_jetbrains_mlapi-ml-api-0_1_94_http//file", - visibility = ["//visibility:public"] -) - -jvm_import( - name = "jetbrains-mlapi-ml-tools", - jar = "@com_jetbrains_mlapi-ml-tools-0_1_94_http//file", - source_jar = "@com_jetbrains_mlapi-ml-tools-0_1_94_http//file", + name = "jetbrains-mlapi-core", + jar = "@com_jetbrains_mlapi-mlapi-core-0_2_0_http//file", + source_jar = "@com_jetbrains_mlapi-mlapi-core-0_2_0_http//file", visibility = ["//visibility:public"] ) @@ -9024,8 +9024,8 @@ jvm_import( jvm_import( name = "python-ml-features-jetbrains-ml-models-python-imports-ranking-model", - jar = "@com_jetbrains_ml_models-python-imports-ranking-model-coral-panda-republished-4_http//file", - source_jar = "@com_jetbrains_ml_models-python-imports-ranking-model-coral-panda-republished-4_http//file", + jar = "@com_jetbrains_ml_models-python-imports-ranking-model-coral-panda-republished-6_http//file", + source_jar = "@com_jetbrains_ml_models-python-imports-ranking-model-coral-panda-republished-6_http//file", visibility = ["//visibility:public"] ) diff --git a/lib/MODULE.bazel b/lib/MODULE.bazel index ab8ab81fa478..32b12899858f 100644 --- a/lib/MODULE.bazel +++ b/lib/MODULE.bazel @@ -3982,6 +3982,20 @@ http_file( downloaded_file_path = "markdown-jvm-0.7.2-sources.jar" ) +http_file( + name = "com_jetbrains_mlapi-mlapi-catboost-0_2_0_http", + url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/mlapi/mlapi-catboost/0.2.0/mlapi-catboost-0.2.0.jar", + sha256 = "81d40889aba4f259f59787bf9d953501668fc4f49081c75792618bba1cc6c661", + downloaded_file_path = "mlapi-catboost-0.2.0.jar" +) + +http_file( + name = "com_jetbrains_mlapi-mlapi-catboost-0_2_0-sources_http", + url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/mlapi/mlapi-catboost/0.2.0/mlapi-catboost-0.2.0-sources.jar", + sha256 = "7da5d2518a30d2777d0238eb88cb6ebb7327d957889d6ba259d1e73afc264660", + downloaded_file_path = "mlapi-catboost-0.2.0-sources.jar" +) + http_file( name = "com_jetbrains_mlapi-catboost-shadow-need-slf4j-1_2_5_http", url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/mlapi/catboost-shadow-need-slf4j/1.2.5/catboost-shadow-need-slf4j-1.2.5.jar", @@ -3990,31 +4004,17 @@ http_file( ) http_file( - name = "com_jetbrains_mlapi-ml-api-0_1_94_http", - url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/mlapi/ml-api/0.1.94/ml-api-0.1.94.jar", - sha256 = "1050f9b58c8d71833d075ae7302881cb9d905dcda4025a21c59657ac823ac105", - downloaded_file_path = "ml-api-0.1.94.jar" + name = "com_jetbrains_mlapi-mlapi-core-0_2_0_http", + url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/mlapi/mlapi-core/0.2.0/mlapi-core-0.2.0.jar", + sha256 = "6e07a48c74982f31a55b5099791e45b6af42a8e540c591732a10f8493abb56fe", + downloaded_file_path = "mlapi-core-0.2.0.jar" ) http_file( - name = "com_jetbrains_mlapi-ml-api-0_1_94-sources_http", - url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/mlapi/ml-api/0.1.94/ml-api-0.1.94-sources.jar", - sha256 = "79eb26365ce9f536733d96fc16e40e7e89e1daba9312a8d06449718430d7d499", - downloaded_file_path = "ml-api-0.1.94-sources.jar" -) - -http_file( - name = "com_jetbrains_mlapi-ml-tools-0_1_94_http", - url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/mlapi/ml-tools/0.1.94/ml-tools-0.1.94.jar", - sha256 = "b2d36677ba73f2f7974199823b5f4323b2a16ea718ed506297402c6a7605af1c", - downloaded_file_path = "ml-tools-0.1.94.jar" -) - -http_file( - name = "com_jetbrains_mlapi-ml-tools-0_1_94-sources_http", - url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/mlapi/ml-tools/0.1.94/ml-tools-0.1.94-sources.jar", - sha256 = "8c5b021724a9b8dab25e33b09812aadbd160cf8e64e89a3dd17c40449dbdedae", - downloaded_file_path = "ml-tools-0.1.94-sources.jar" + name = "com_jetbrains_mlapi-mlapi-core-0_2_0-sources_http", + url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/mlapi/mlapi-core/0.2.0/mlapi-core-0.2.0-sources.jar", + sha256 = "ac917198906a6dfac091713d55d29ebe0ce347df9b3537bd458f6fcdab3af5c8", + downloaded_file_path = "mlapi-core-0.2.0-sources.jar" ) http_file( @@ -13160,17 +13160,17 @@ http_file( ) http_file( - name = "com_jetbrains_ml_models-python-imports-ranking-model-coral-panda-republished-4_http", - url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/ml/models/python-imports-ranking-model/coral-panda-republished-4/python-imports-ranking-model-coral-panda-republished-4.jar", - sha256 = "cff0591184b48356a622f7635c6de1399fdbeb161f98d68206d09807c83fa98c", - downloaded_file_path = "python-imports-ranking-model-coral-panda-republished-4.jar" + name = "com_jetbrains_ml_models-python-imports-ranking-model-coral-panda-republished-6_http", + url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/ml/models/python-imports-ranking-model/coral-panda-republished-6/python-imports-ranking-model-coral-panda-republished-6.jar", + sha256 = "2dbd0ae71bac24a83db76238f0a0cf2bd6b0fd1963b6827371ae8813a0434827", + downloaded_file_path = "python-imports-ranking-model-coral-panda-republished-6.jar" ) http_file( - name = "com_jetbrains_ml_models-python-imports-ranking-model-coral-panda-republished-4-sources_http", - url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/ml/models/python-imports-ranking-model/coral-panda-republished-4/python-imports-ranking-model-coral-panda-republished-4-sources.jar", - sha256 = "4aecfba9498dff7e199c7db2a718b53c77834347daf8b96cafc8b02b392c5e6c", - downloaded_file_path = "python-imports-ranking-model-coral-panda-republished-4-sources.jar" + name = "com_jetbrains_ml_models-python-imports-ranking-model-coral-panda-republished-6-sources_http", + url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/jetbrains/ml/models/python-imports-ranking-model/coral-panda-republished-6/python-imports-ranking-model-coral-panda-republished-6-sources.jar", + sha256 = "3b0e9358cb7dfa787a28ce3502655f39cd73c6f65a2b1eed7074f04acb562584", + downloaded_file_path = "python-imports-ranking-model-coral-panda-republished-6-sources.jar" ) http_file( diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/CommunityLibraryLicenses.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/CommunityLibraryLicenses.kt index d4916005dcd4..d90d1e9288f4 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/CommunityLibraryLicenses.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/CommunityLibraryLicenses.kt @@ -1414,9 +1414,9 @@ object CommunityLibraryLicenses { jetbrainsLibrary("jetbrains.kotlin.jps.plugin.classpath"), jetbrainsLibrary("jetbrains.ml.models.jetenry.inline.prompt.detection.model"), jetbrainsLibrary("jetbrains.ml.models.python.imports.ranking.model"), + jetbrainsLibrary("jetbrains.mlapi.catboost"), jetbrainsLibrary("jetbrains.mlapi.catboost.shadow.need.slf4j"), - jetbrainsLibrary("jetbrains.mlapi.ml.api"), - jetbrainsLibrary("jetbrains.mlapi.ml.tools"), + jetbrainsLibrary("jetbrains.mlapi.core"), jetbrainsLibrary("jshell-frontend"), jetbrainsLibrary("jvm-native-trusted-roots"), jetbrainsLibrary("kotlin-gradle-plugin-idea"), diff --git a/platform/ml-logs/BUILD.bazel b/platform/ml-logs/BUILD.bazel index 46ba52d00681..d2018670ec52 100644 --- a/platform/ml-logs/BUILD.bazel +++ b/platform/ml-logs/BUILD.bazel @@ -15,16 +15,11 @@ jvm_library( resources = [":ml-logs_resources"], deps = [ "@lib//:kotlin-stdlib", + "//platform/util", "//platform/statistics", "@lib//:jetbrains-annotations", - "@lib//:jetbrains-mlapi-catboost-shadow-need-slf4j", - "@lib//:jetbrains-mlapi-ml-api", - "@lib//:jetbrains-mlapi-ml-tools", + "@lib//:jetbrains-mlapi-core", ], - exports = [ - "@lib//:jetbrains-mlapi-catboost-shadow-need-slf4j", - "@lib//:jetbrains-mlapi-ml-api", - "@lib//:jetbrains-mlapi-ml-tools", - ] + exports = ["@lib//:jetbrains-mlapi-core"] ) ### auto-generated section `build intellij.platform.ml.logs` end \ No newline at end of file diff --git a/platform/ml-logs/intellij.platform.ml.logs.iml b/platform/ml-logs/intellij.platform.ml.logs.iml index 3c5c689fcc7c..fd290fc45d84 100644 --- a/platform/ml-logs/intellij.platform.ml.logs.iml +++ b/platform/ml-logs/intellij.platform.ml.logs.iml @@ -9,10 +9,9 @@ + - - - + \ No newline at end of file diff --git a/platform/ml-logs/src/com/intellij/platform/ml/logs/IJEventPairConverter.kt b/platform/ml-logs/src/com/intellij/platform/ml/logs/IJEventPairConverter.kt index 981d473c7dae..b95ef866d5f3 100644 --- a/platform/ml-logs/src/com/intellij/platform/ml/logs/IJEventPairConverter.kt +++ b/platform/ml-logs/src/com/intellij/platform/ml/logs/IJEventPairConverter.kt @@ -1,126 +1,86 @@ // Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.platform.ml.logs -import com.intellij.internal.statistic.eventLog.events.EventPair -import com.intellij.internal.statistic.eventLog.events.StringEventField -import com.intellij.platform.ml.logs.ConverterObjectDescription.Companion.asIJObjectDescription -import com.intellij.platform.ml.logs.ConverterOfEnum.Companion.toIJConverter +import com.intellij.internal.statistic.eventLog.events.* import com.intellij.platform.ml.logs.IJEventPairConverter.Companion.typedBuild -import com.intellij.internal.statistic.eventLog.events.BooleanEventField as IJBooleanEventField -import com.intellij.internal.statistic.eventLog.events.ClassEventField as IJClassEventField -import com.intellij.internal.statistic.eventLog.events.DoubleEventField as IJDoubleEventField -import com.intellij.internal.statistic.eventLog.events.EnumEventField as IJEnumEventField -import com.intellij.internal.statistic.eventLog.events.EventField as IJEventField -import com.intellij.internal.statistic.eventLog.events.EventPair as IJEventPair -import com.intellij.internal.statistic.eventLog.events.FloatEventField as IJFloatEventField1 -import com.intellij.internal.statistic.eventLog.events.FloatListEventField as IJFloatListEventField -import com.intellij.internal.statistic.eventLog.events.IntEventField as IJIntEventField -import com.intellij.internal.statistic.eventLog.events.IntListEventField as IJIntListEventField -import com.intellij.internal.statistic.eventLog.events.LongEventField as IJLongEventField -import com.intellij.internal.statistic.eventLog.events.LongListEventField as IJLongListEventField -import com.intellij.internal.statistic.eventLog.events.ObjectDescription as IJObjectDescription -import com.intellij.internal.statistic.eventLog.events.ObjectEventData as IJObjectEventData -import com.intellij.internal.statistic.eventLog.events.ObjectEventField as IJObjectEventField -import com.intellij.internal.statistic.eventLog.events.ObjectListEventField as IJObjectListEventField -import com.jetbrains.ml.api.logs.BooleanEventField as MLBooleanEventField -import com.jetbrains.ml.api.logs.ClassEventField as MLClassEventField -import com.jetbrains.ml.api.logs.DoubleEventField as MLDoubleEventField -import com.jetbrains.ml.api.logs.EnumEventField as MLEnumEventField -import com.jetbrains.ml.api.logs.EventField as MLEventField -import com.jetbrains.ml.api.logs.EventPair as MLEventPair -import com.jetbrains.ml.api.logs.FloatEventField as MLFloatEventField -import com.jetbrains.ml.api.logs.FloatListEventField as MLFloatListEventField -import com.jetbrains.ml.api.logs.IntEventField as MLIntEventField -import com.jetbrains.ml.api.logs.IntListEventField as MLIntListEventField -import com.jetbrains.ml.api.logs.LongEventField as MLLongEventField -import com.jetbrains.ml.api.logs.LongListEventField as MLLongListEventField -import com.jetbrains.ml.api.logs.ObjectDescription as MLObjectDescription -import com.jetbrains.ml.api.logs.ObjectEventData as MLObjectEventData -import com.jetbrains.ml.api.logs.ObjectEventField as MLObjectEventField -import com.jetbrains.ml.api.logs.ObjectListEventField as MLObjectListEventField -import com.jetbrains.ml.api.logs.StringEventField as MLStringEventField +import com.jetbrains.mlapi.feature.* +import com.jetbrains.mlapi.logs.ObjectArrayDeclaration +import com.jetbrains.mlapi.logs.ObjectDeclaration +import com.jetbrains.mlapi.logs.ObjectFeatureDeclaration -internal interface IJEventPairConverter { - val ijEventField: IJEventField +internal interface IJEventPairConverter { + val ijEventField: EventField - fun buildEventPair(mlEventPair: MLEventPair): IJEventPair + fun buildEventPair(feature: FeatureT): EventPair companion object { - fun IJEventPairConverter.typedBuild(mlEventPair: MLEventPair<*>): IJEventPair { + fun IJEventPairConverter.typedBuild(feature: Feature): EventPair { @Suppress("UNCHECKED_CAST") - return buildEventPair(mlEventPair as MLEventPair) + return buildEventPair(feature as F) } } } -internal class ConverterObjectDescription(mlObjectDescription: MLObjectDescription) : IJObjectDescription() { - private val toIJConverters: Map, IJEventPairConverter<*, *>> = mlObjectDescription.getFields().associateWith { mlField -> - val converter = createConverter(mlField) - checkNotNull(converter) { "Please implement converter for $mlField from ML to IJ event fields" } +internal class ConverterObjectDescription(signatures: List) : ObjectDescription() { + private val toIJConverters: Map> = signatures.associateWith { signature -> + val converter = createConverter(signature) + checkNotNull(converter) { "Please implement converter for $signature from ML to IJ event fields" } field(converter.ijEventField) converter } @Suppress("UNCHECKED_CAST") - private fun createConverter(mlEventField: MLEventField): IJEventPairConverter = when (mlEventField) { - is MLObjectEventField -> ConverterOfObject( - mlEventField.name, - mlEventField.lazyDescription, - mlEventField.objectDescription - ) as IJEventPairConverter - is MLBooleanEventField -> ConverterOfPrimitiveType(mlEventField) { n, d -> IJBooleanEventField(n, d) } as IJEventPairConverter - is MLIntEventField -> ConverterOfPrimitiveType(mlEventField) { n, d -> IJIntEventField(n, d) } as IJEventPairConverter - is MLLongEventField -> ConverterOfPrimitiveType(mlEventField) { n, d -> IJLongEventField(n, d) } as IJEventPairConverter - is MLFloatEventField -> ConverterOfPrimitiveType(mlEventField) { n, d -> IJFloatEventField1(n, d) } as IJEventPairConverter - is MLEnumEventField<*> -> mlEventField.toIJConverter() as IJEventPairConverter - is MLClassEventField -> ConverterOfClass(mlEventField) as IJEventPairConverter - is MLObjectListEventField -> ConvertObjectList(mlEventField) as IJEventPairConverter - is MLDoubleEventField -> ConverterOfPrimitiveType(mlEventField) { n, d -> IJDoubleEventField(n, d) } as IJEventPairConverter - is MLStringEventField -> ConverterOfString(mlEventField) as IJEventPairConverter - is MLFloatListEventField -> object : IJEventPairConverter, List> { - override val ijEventField: IJEventField> = IJFloatListEventField(mlEventField.name, mlEventField.lazyDescription?.invoke()) - override fun buildEventPair(mlEventPair: MLEventPair>): IJEventPair> = ijEventField with mlEventPair.data - } as IJEventPairConverter - is MLIntListEventField -> object : IJEventPairConverter, List> { - override val ijEventField: IJEventField> = IJIntListEventField(mlEventField.name, mlEventField.lazyDescription?.invoke()) - override fun buildEventPair(mlEventPair: MLEventPair>): IJEventPair> = ijEventField with mlEventPair.data - } as IJEventPairConverter - is MLLongListEventField -> object : IJEventPairConverter, List> { - override val ijEventField: IJEventField> = IJLongListEventField(mlEventField.name, mlEventField.lazyDescription?.invoke()) - override fun buildEventPair(mlEventPair: MLEventPair>): IJEventPair> = ijEventField with mlEventPair.data - } as IJEventPairConverter + private fun createConverter(signature: FeatureSignature): IJEventPairConverter<*, *>? = when (signature) { + is ObjectFeatureDeclaration -> ConverterOfObject(signature) + is ObjectArrayDeclaration -> ConvertObjectList(signature) + is PrimitiveFeatureDeclaration<*> -> { + when (signature.type) { + is PrimitiveType.Boolean -> ConverterOfBoolean(signature) + is PrimitiveType.Int32 -> ConverterOfInt32(signature) + is PrimitiveType.Int64 -> ConverterOfInt64(signature) + is PrimitiveType.Float -> ConverterOfFloat(signature) + is PrimitiveType.Double -> ConverterOfDouble(signature) + is PrimitiveType.String -> ConverterOfString(signature) - else -> throw NotImplementedError("Please implement converter for $mlEventField from ML to IJ event fields") + is PrimitiveType.EmbeddingInt32 -> ConverterOfIntList(signature) + is PrimitiveType.EmbeddingInt64 -> ConverterOfLongList(signature) + is PrimitiveType.EmbeddingFloat -> ConverterOfFloatList(signature) + is PrimitiveType.EmbeddingDouble -> ConverterOfFloatList(signature) + + is PrimitiveType.Null -> null + } + } + is ClassFeatureDeclaration -> ConverterOfClass(signature) + is EnumFeatureDeclaration<*> -> ConverterOfEnum(signature) + is NullableFeatureDeclaration<*> -> createConverter(signature.asNonNullable) + + else -> throw NotImplementedError("Please implement converter for $signature from ML to IJ event fields") } - fun buildEventPairs(mlEventPairs: List>): List> { - return mlEventPairs.map { mlEventPair -> - require(mlEventPair.field in toIJConverters) { + fun buildEventPairs(features: List): List> { + return features.map { feature -> + require(feature.signature in toIJConverters) { """ - Field ${mlEventPair.field} (name: ${mlEventPair.field.name}) was not found among + Field ${feature.signature} (name: ${feature.signature.name}) was not found among the registered ones: ${toIJConverters.keys.map { it.name }} """.trimIndent() } - val converter = requireNotNull(toIJConverters[mlEventPair.field]) - converter.typedBuild(mlEventPair) + val converter = requireNotNull(toIJConverters[feature.signature]) + converter.typedBuild(feature) } } - fun buildObjectEventData(mlObject: MLObjectEventData): IJObjectEventData { - return IJObjectEventData(buildEventPairs(mlObject.values)) + fun buildObjectEventData(features: List): ObjectEventData { + return ObjectEventData(buildEventPairs(features)) } - companion object { - fun MLObjectDescription.asIJObjectDescription(): ConverterObjectDescription = - ConverterObjectDescription(this) - } } -private class ConverterOfString(mlEventField: MLStringEventField) : IJEventPairConverter { +private class ConverterOfString(declaration: PrimitiveFeatureDeclaration<*>) : IJEventPairConverter { private class SensitiveStringEventField( name: String, @@ -130,89 +90,111 @@ private class ConverterOfString(mlEventField: MLStringEventField) : IJEventPairC override val validationRule: List = listOf("{util#${ruleId}}") } - override val ijEventField: IJEventField = + override val ijEventField: EventField = SensitiveStringEventField( - mlEventField.name, - requireNotNull(mlEventField.ruleId) { "Error for $mlEventField: it must have a validation rule" }, - mlEventField.lazyDescription?.invoke() + declaration.name, + requireNotNull(declaration.logsMetadata?.ruleId) { "Error for $declaration: it must have a validation rule" }, + declaration.logsMetadata?.lazyDescription?.invoke() ) - override fun buildEventPair(mlEventPair: MLEventPair): EventPair = - ijEventField with mlEventPair.data + override fun buildEventPair(feature: Feature.String): EventPair = + ijEventField with feature.stringValue } - -private class ConvertObjectList(mlEventField: MLObjectListEventField) : - IJEventPairConverter, List> { - private val innerObjectConverter = ConverterOfObject(mlEventField.name, mlEventField.lazyDescription, mlEventField.internalObjectDescription) - - // FIXME: description is not passed - override val ijEventField: IJEventField> = IJObjectListEventField( - mlEventField.name, - innerObjectConverter.ijObjectDescription - ) - - override fun buildEventPair(mlEventPair: MLEventPair>): IJEventPair> { - return ijEventField with mlEventPair.data.map { innerObjectFieldsValues -> - innerObjectConverter.buildObjectEventData(innerObjectFieldsValues) - } - } +private abstract class ConverterOfPrimitiveType(declaration: AbstractPrimitiveDeclaration<*, *>) : IJEventPairConverter { + abstract fun createEventField(name: String, description: String?): EventField + abstract fun getValue(feature: FeatureT): T + + override val ijEventField: EventField by lazy { createEventField(declaration.name, declaration.logsMetadata?.lazyDescription?.invoke()) } + override fun buildEventPair(feature: FeatureT): EventPair = ijEventField with getValue(feature) } +private class ConverterOfBoolean(declaration: PrimitiveFeatureDeclaration<*>) : ConverterOfPrimitiveType(declaration) { + override fun createEventField(name: String, description: String?): EventField = BooleanEventField(name, description) + override fun getValue(feature: Feature.Boolean) = feature.booleanValue +} -private class ConverterOfEnum>(mlEnumField: MLEnumEventField) : IJEventPairConverter { - override val ijEventField: IJEventField = IJEnumEventField(mlEnumField.name, mlEnumField.enumClass) { it.name } +private class ConverterOfFloat(declaration: PrimitiveFeatureDeclaration<*>) : ConverterOfPrimitiveType(declaration) { + override fun createEventField(name: String, description: String?): EventField = FloatEventField(name, description) + override fun getValue(feature: Feature.Float) = feature.floatValue +} - override fun buildEventPair(mlEventPair: MLEventPair): IJEventPair { - return ijEventField with mlEventPair.data - } +private class ConverterOfDouble(declaration: PrimitiveFeatureDeclaration<*>) : ConverterOfPrimitiveType(declaration) { + override fun createEventField(name: String, description: String?): EventField = DoubleEventField(name, description) + override fun getValue(feature: Feature.Double) = feature.doubleValue +} - companion object { - fun > MLEnumEventField.toIJConverter(): ConverterOfEnum { - return ConverterOfEnum(this) - } - } +private class ConverterOfInt32(declaration: PrimitiveFeatureDeclaration<*>) : ConverterOfPrimitiveType(declaration) { + override fun createEventField(name: String, description: String?): EventField = IntEventField(name, description) + override fun getValue(feature: Feature.Int32) = feature.int32Value +} + +private class ConverterOfInt64(declaration: PrimitiveFeatureDeclaration<*>) : ConverterOfPrimitiveType(declaration) { + override fun createEventField(name: String, description: String?): EventField = LongEventField(name, description) + override fun getValue(feature: Feature.Int64) = feature.int64Value +} + +private class ConverterOfIntList(declaration: PrimitiveFeatureDeclaration<*>) : ConverterOfPrimitiveType>(declaration) { + override fun createEventField(name: String, description: String?): EventField> = IntListEventField(name, description) + override fun getValue(feature: Feature.Int32Embedding): List = feature.int32Array.toList() +} + +private class ConverterOfLongList(declaration: PrimitiveFeatureDeclaration<*>) : ConverterOfPrimitiveType>(declaration) { + override fun createEventField(name: String, description: String?): EventField> = LongListEventField(name, description) + override fun getValue(feature: Feature.Int64Embedding): List = feature.int64Array.toList() +} + +private class ConverterOfFloatList(declaration: PrimitiveFeatureDeclaration<*>) : ConverterOfPrimitiveType>(declaration) { + override fun createEventField(name: String, description: String?): EventField> = FloatListEventField(name, description) + override fun getValue(feature: Feature.FloatEmbedding): List = feature.floatArray.toList() +} + +private class ConverterOfClass(declaration: ClassFeatureDeclaration) : ConverterOfPrimitiveType?>(declaration) { + override fun createEventField(name: String, description: String?): EventField?> = ClassEventField(name, description) + override fun getValue(feature: Feature.Class): Class<*> = feature.classValue +} + +private class ConverterOfEnum>( + private val enumDeclaration: EnumFeatureDeclaration +) : ConverterOfPrimitiveType, E>(enumDeclaration) { + override fun createEventField(name: String, description: String?): EventField = + EnumEventField(name, enumDeclaration.enumClass, description = description, transform = { it.name }) + override fun getValue(feature: Feature.Enum): E = feature.enumValue } private class ConverterOfObject( - name: String, - lazyDescription: (() -> String)?, - mlObjectDescription: MLObjectDescription, -) : IJEventPairConverter { - val ijObjectDescription = mlObjectDescription.asIJObjectDescription() + objectDeclaration: ObjectDeclaration<*>, +) : IJEventPairConverter { + val ijObjectDescription = ConverterObjectDescription(objectDeclaration.signatures) - override val ijEventField: IJEventField = IJObjectEventField(name, lazyDescription?.invoke(), ijObjectDescription) + override val ijEventField: EventField = + ObjectEventField(objectDeclaration.name, objectDeclaration.logsMetadata?.lazyDescription?.invoke(), ijObjectDescription) - fun buildObjectEventData(mlObject: MLObjectEventData): IJObjectEventData { - return ijObjectDescription.buildObjectEventData(mlObject) + fun buildObjectEventData(features: List): ObjectEventData { + return ijObjectDescription.buildObjectEventData(features) } - override fun buildEventPair(mlEventPair: MLEventPair): IJEventPair { - return ijEventField with buildObjectEventData(mlEventPair.data) + override fun buildEventPair(feature: Feature.Object): EventPair { + return ijEventField with buildObjectEventData(feature.values) } } -private class ConverterOfPrimitiveType( - mlEventField: MLEventField, - createIJField: (String, String?) -> IJEventField -) : IJEventPairConverter { - override val ijEventField: IJEventField = createIJField(mlEventField.name, mlEventField.lazyDescription?.invoke()) +private class ConvertObjectList(declaration: ObjectArrayDeclaration) : IJEventPairConverter> { - override fun buildEventPair(mlEventPair: MLEventPair): IJEventPair { - return ijEventField with mlEventPair.data - } -} + private val innerObjectConverter = ConverterOfObject(declaration.elementSignature) + // FIXME: description is not passed + override val ijEventField: EventField> = ObjectListEventField( + declaration.name, + innerObjectConverter.ijObjectDescription, + ) -private class ConverterOfClass( - mlEventField: MLClassEventField, -) : IJEventPairConverter, Class<*>?> { - override val ijEventField: IJEventField?> = IJClassEventField(mlEventField.name, mlEventField.lazyDescription?.invoke()) - - override fun buildEventPair(mlEventPair: MLEventPair>): IJEventPair?> { - return ijEventField with mlEventPair.data + override fun buildEventPair(feature: Feature.ObjectArray): EventPair> { + return ijEventField with feature.values.map { objectFeature -> + innerObjectConverter.buildObjectEventData(objectFeature.values) + } } } diff --git a/platform/ml-logs/src/com/intellij/platform/ml/logs/IJFeatureDeclarations.kt b/platform/ml-logs/src/com/intellij/platform/ml/logs/IJFeatureDeclarations.kt new file mode 100644 index 000000000000..b93eb02a1415 --- /dev/null +++ b/platform/ml-logs/src/com/intellij/platform/ml/logs/IJFeatureDeclarations.kt @@ -0,0 +1,22 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.platform.ml.logs + +import com.intellij.internal.statistic.utils.getPluginInfo +import com.intellij.openapi.util.text.StringUtil +import com.jetbrains.mlapi.feature.ClassFeatureDeclaration +import com.jetbrains.mlapi.feature.EnumFeatureDeclaration +import com.jetbrains.mlapi.feature.FeatureDeclaration +import org.jetbrains.annotations.ApiStatus + +@ApiStatus.Internal +object IJFeatureDeclarations { + inline fun > enum(name: String, noinline lazyDescription: (() -> String)? = null): EnumFeatureDeclaration = + FeatureDeclaration.enum(name, null, lazyDescription) + + fun aClass(name: String, lazyDescription: (() -> String)? = null): ClassFeatureDeclaration = + FeatureDeclaration.aClass(name, null, lazyDescription, classCheckAndTransform) + + private val classCheckAndTransform: (Class<*>) -> String = { + if (getPluginInfo(it).isSafeToReport()) StringUtil.substringBeforeLast(it.name, "$\$Lambda", true) else "third.party" + } +} \ No newline at end of file diff --git a/platform/ml-logs/src/com/intellij/platform/ml/logs/IntelliJFusEventRegister.kt b/platform/ml-logs/src/com/intellij/platform/ml/logs/IntelliJFusEventRegister.kt index 91eaae49469c..bf9724d6dfe2 100644 --- a/platform/ml-logs/src/com/intellij/platform/ml/logs/IntelliJFusEventRegister.kt +++ b/platform/ml-logs/src/com/intellij/platform/ml/logs/IntelliJFusEventRegister.kt @@ -1,30 +1,29 @@ // Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.platform.ml.logs +import com.intellij.internal.statistic.eventLog.EventLogGroup import com.intellij.internal.statistic.eventLog.events.VarargEventId -import com.jetbrains.ml.tools.logs.FusEventLogger -import com.jetbrains.ml.tools.logs.FusEventRegister +import com.jetbrains.mlapi.feature.Feature +import com.jetbrains.mlapi.feature.FeatureDeclaration +import com.jetbrains.mlapi.logs.LogsEventRegister +import com.jetbrains.mlapi.logs.MLEventLogger import org.jetbrains.annotations.ApiStatus -import com.intellij.internal.statistic.eventLog.EventLogGroup as IJEventLogGroup -import com.jetbrains.ml.api.logs.EventField as MLEventField -import com.jetbrains.ml.api.logs.EventPair as MLEventPair -import com.jetbrains.ml.api.logs.ObjectDescription as MLObjectDescription @ApiStatus.Internal -class IntelliJFusEventRegister(private val baseEventGroup: IJEventLogGroup) : FusEventRegister { +class IntelliJFusEventRegister(private val baseEventGroup: EventLogGroup) : LogsEventRegister { private class Logger( private val varargEventId: VarargEventId, private val objectDescription: ConverterObjectDescription - ) : FusEventLogger { - override fun log(eventPairs: List>) { - val ijEventPairs = objectDescription.buildEventPairs(eventPairs) + ) : MLEventLogger { + override fun log(features: List) { + val ijEventPairs = objectDescription.buildEventPairs(features) varargEventId.log(*ijEventPairs.toTypedArray()) } } - override fun registerEvent(name: String, eventFields: List>): FusEventLogger { - val objectDescription = ConverterObjectDescription(MLObjectDescription(eventFields)) + override fun registerEvent(name: String, declarations: List>): MLEventLogger { + val objectDescription = ConverterObjectDescription(declarations) val varargEventId = baseEventGroup.registerVarargEvent(name, null, *objectDescription.getFields()) return Logger(varargEventId, objectDescription) } diff --git a/plugins/findUsagesMl/BUILD.bazel b/plugins/findUsagesMl/BUILD.bazel index 012eb4f1214e..918bdaa209e6 100644 --- a/plugins/findUsagesMl/BUILD.bazel +++ b/plugins/findUsagesMl/BUILD.bazel @@ -15,7 +15,6 @@ jvm_library( resources = [":findUsagesMl_resources"], deps = [ "@lib//:kotlin-stdlib", - "@lib//:jetbrains-mlapi-ml-api", "@lib//:commons-text", "//platform/indexing-api:indexing", "//platform/util", @@ -35,7 +34,6 @@ jvm_library( associates = [":findUsagesMl"], deps = [ "@lib//:kotlin-stdlib", - "@lib//:jetbrains-mlapi-ml-api", "@lib//:commons-text", "//platform/indexing-api:indexing", "//platform/util", diff --git a/plugins/findUsagesMl/intellij.findUsagesMl.iml b/plugins/findUsagesMl/intellij.findUsagesMl.iml index 8a6fd95f8674..4da5457a4793 100644 --- a/plugins/findUsagesMl/intellij.findUsagesMl.iml +++ b/plugins/findUsagesMl/intellij.findUsagesMl.iml @@ -10,7 +10,6 @@ - diff --git a/plugins/findUsagesMl/src/com/intellij/findUsagesMl/AnalysisProviders.kt b/plugins/findUsagesMl/src/com/intellij/findUsagesMl/AnalysisProviders.kt index 34fef062f300..dab7e29dbe49 100644 --- a/plugins/findUsagesMl/src/com/intellij/findUsagesMl/AnalysisProviders.kt +++ b/plugins/findUsagesMl/src/com/intellij/findUsagesMl/AnalysisProviders.kt @@ -1,13 +1,11 @@ package com.intellij.findUsagesMl -import com.jetbrains.ml.api.logs.BooleanEventField -import com.jetbrains.ml.api.logs.EventField -import com.jetbrains.ml.api.logs.EventPair -import com.jetbrains.ml.api.logs.IntEventField -import com.jetbrains.ml.api.logs.LongEventField -import com.jetbrains.ml.tools.logs.extractEventFields +import com.jetbrains.mlapi.feature.Feature +import com.jetbrains.mlapi.feature.FeatureContainer +import com.jetbrains.mlapi.feature.FeatureDeclaration data class FindUsagesFileRankingAnalysisInfo( + val customSessionId: Long, val isUsage: Boolean, val timestamp: Long, val isSearchValid: Boolean = true, @@ -17,8 +15,9 @@ data class FindUsagesFileRankingAnalysisInfo( val activeSessionId: Long = -1, val finishSessionId: Long = -1, ) { - constructor(isUsage: Boolean, timestamp: Long, isSearchValid: Boolean, numberOfUsageFiles: Int?, numberOfCandidates: Int?, indexInOriginalOrder: Int?, activeSessionId: Long?, finishSessionId: Long?) - : this(isUsage, + constructor(customSessionId: Long, isUsage: Boolean, timestamp: Long, isSearchValid: Boolean, numberOfUsageFiles: Int?, numberOfCandidates: Int?, indexInOriginalOrder: Int?, activeSessionId: Long?, finishSessionId: Long?) + : this(customSessionId, + isUsage, timestamp, isSearchValid, numberOfUsageFiles ?: -1, @@ -28,27 +27,26 @@ data class FindUsagesFileRankingAnalysisInfo( finishSessionId ?: -1) } -object FindUsagesFileRankerAnalysisTargets { - val IS_USAGE: BooleanEventField = BooleanEventField("is_usage", lazyDescription = { "Is usage" }) - val SEARCH_TIMESTAMP: LongEventField = LongEventField("search_timestamp_ms", lazyDescription = { "Search timestamp" }) - val IS_VALID: BooleanEventField = BooleanEventField(name = "is_valid", lazyDescription = { "Is the search session valid (false if corrupted)" }) +object FindUsagesFileRankerAnalysisTargets : FeatureContainer { + val SESSION_ID: FeatureDeclaration = FeatureDeclaration.long("session_id") { "Id of the search session" } - val NUMBER_OF_USAGE_FILES: IntEventField = IntEventField(name = "number_of_usage_files", lazyDescription = { "Number of files containing a usage" }) - val NUMBER_OF_CANDIDATES: IntEventField = IntEventField(name = "number_of_candidates", lazyDescription = { "Number of candidates" }) - val INDEX_IN_ORIGINAL_ORDER: IntEventField = IntEventField(name = "index_in_original_order", lazyDescription = { "Index in original order" }) + val IS_USAGE: FeatureDeclaration = FeatureDeclaration.boolean("is_usage") { "Is usage" } + val SEARCH_TIMESTAMP: FeatureDeclaration = FeatureDeclaration.long("search_timestamp_ms") { "Search timestamp" } + val IS_VALID: FeatureDeclaration = FeatureDeclaration.boolean("is_valid") { "Is the search session valid (false if corrupted)" } - val ACTIVE_SESSION: LongEventField = LongEventField(name = "active_session", lazyDescription = { "Id of last active (started) session" }) - val FINISH_SESSION: LongEventField = LongEventField(name = "finish_session", lazyDescription = { "Id of finishing session" }) + val NUMBER_OF_USAGE_FILES: FeatureDeclaration = FeatureDeclaration.int("number_of_usage_files") { "Number of files containing a usage" } + val NUMBER_OF_CANDIDATES: FeatureDeclaration = FeatureDeclaration.int("number_of_candidates") { "Number of candidates" } + val INDEX_IN_ORIGINAL_ORDER: FeatureDeclaration = FeatureDeclaration.int("index_in_original_order") { "Index in original order" } - fun eventFields(): List>> { - return listOf(extractEventFields(FindUsagesFileRankerAnalysisTargets)) - } + val ACTIVE_SESSION: FeatureDeclaration = FeatureDeclaration.long("active_session") { "Id of last active (started) session" } + val FINISH_SESSION: FeatureDeclaration = FeatureDeclaration.long("finish_session") { "Id of finishing session" } } class FindUsagesFileRankerAnalysisProvider { - fun provideAnalysisTargets(info: FindUsagesFileRankingAnalysisInfo): List> { + fun provideAnalysisTargets(info: FindUsagesFileRankingAnalysisInfo): List { if (!info.isSearchValid) { return listOf( + FindUsagesFileRankerAnalysisTargets.SESSION_ID with info.customSessionId, FindUsagesFileRankerAnalysisTargets.SEARCH_TIMESTAMP with info.timestamp, FindUsagesFileRankerAnalysisTargets.IS_VALID with false, FindUsagesFileRankerAnalysisTargets.ACTIVE_SESSION with info.activeSessionId, @@ -56,6 +54,7 @@ class FindUsagesFileRankerAnalysisProvider { ) } return listOf( + FindUsagesFileRankerAnalysisTargets.SESSION_ID with info.customSessionId, FindUsagesFileRankerAnalysisTargets.IS_VALID with true, FindUsagesFileRankerAnalysisTargets.IS_USAGE with info.isUsage, FindUsagesFileRankerAnalysisTargets.SEARCH_TIMESTAMP with info.timestamp, diff --git a/plugins/findUsagesMl/src/com/intellij/findUsagesMl/FeatureCollector.kt b/plugins/findUsagesMl/src/com/intellij/findUsagesMl/FeatureCollector.kt index 814634206cf8..af852754c659 100644 --- a/plugins/findUsagesMl/src/com/intellij/findUsagesMl/FeatureCollector.kt +++ b/plugins/findUsagesMl/src/com/intellij/findUsagesMl/FeatureCollector.kt @@ -3,7 +3,7 @@ package com.intellij.findUsagesMl import com.intellij.internal.statistic.eventLog.EventLogGroup import com.intellij.internal.statistic.service.fus.collectors.CounterUsagesCollector import com.intellij.platform.ml.logs.IntelliJFusEventRegister -import com.jetbrains.ml.tools.logs.MLTreeLoggers.withOneEvent +import com.jetbrains.mlapi.logs.MLTreeLogger internal object FindUsagesFileRankerFeatureCollector : CounterUsagesCollector() { @@ -11,13 +11,13 @@ internal object FindUsagesFileRankerFeatureCollector : CounterUsagesCollector() const val eventGroupId: String = "findUsages.fileRanking" const val fusEventName: String = "find_usages_file_ranking" - private val GROUP = EventLogGroup(eventGroupId, 3, recorderId) + private val GROUP = EventLogGroup(eventGroupId, 4, recorderId) - val mlLogger = withOneEvent( - fusEventName = fusEventName, - fusEventRegister = IntelliJFusEventRegister(GROUP), - treeFeatures = FindUsagesFileRankerFeatures.declarations(), - treeAnalysis = FindUsagesFileRankerAnalysisTargets.eventFields() + val mlLogger = MLTreeLogger.withOneEvent( + eventName = fusEventName, + logsEventRegister = IntelliJFusEventRegister(GROUP), + treeFeatures = listOf(FindUsagesFileRankerFeatures.extractFeatureDeclarations()), + treeAnalysis = listOf(FindUsagesFileRankerAnalysisTargets.extractFeatureDeclarations()) ) override fun getGroup() = GROUP diff --git a/plugins/findUsagesMl/src/com/intellij/findUsagesMl/FeatureProviders.kt b/plugins/findUsagesMl/src/com/intellij/findUsagesMl/FeatureProviders.kt index d52f3978ff30..b3dd7dd2435e 100644 --- a/plugins/findUsagesMl/src/com/intellij/findUsagesMl/FeatureProviders.kt +++ b/plugins/findUsagesMl/src/com/intellij/findUsagesMl/FeatureProviders.kt @@ -1,7 +1,7 @@ package com.intellij.findUsagesMl import com.intellij.openapi.vfs.VirtualFile -import com.jetbrains.ml.api.feature.* +import com.jetbrains.mlapi.feature.* import org.apache.commons.text.similarity.JaroWinklerSimilarity @@ -14,7 +14,7 @@ data class FindUsagesRankingFileInfo( val projectPath: String, ) -object FindUsagesFileRankerFeatures { +object FindUsagesFileRankerFeatures : FeatureContainer { val QUERY_JARO_WINKLER_SIMILARITY: FeatureDeclaration = FeatureDeclaration.double(name = "query_jaro_winkler_similarity") { "Jaro-Winkler similarity of syntactic element to file name" }.nullable() val FILENAME_JARO_WINKLER_SIMILARITY: FeatureDeclaration = FeatureDeclaration.double(name = "filename_jaro_winkler_similarity") { "Jaro-Winkler similarity of the files' names" }.nullable() val QUERY_FILE_TYPE: FeatureDeclaration = FeatureDeclaration.string(name = "query_file_type", "file_type") { "Query file's type" }.nullable() @@ -26,12 +26,9 @@ object FindUsagesFileRankerFeatures { val TIME_SINCE_LAST_MODIFIED_MS: FeatureDeclaration = FeatureDeclaration.long(name = "time_since_modified_ms") { "Time since candidate file's modified timestamp in ms at the time of feature calculation" }.nullable() val RECENT_FILES_INDEX: FeatureDeclaration = FeatureDeclaration.int(name = "recent_files_index") { "Index of the candidate file in the list of the most recent files" }.nullable() val DIRECTORY_DISTANCE: FeatureDeclaration = FeatureDeclaration.double(name = "directory_distance") { "Normalized distance between the query file and candidate file" }.nullable() - - fun declarations(): List>> = listOf(extractFeatureDeclarations(FindUsagesFileRankerFeatures)) } -class FindUsagesFileRankerFeatureProvider : FeatureProvider() { - override val featureDeclarations: List> = extractFeatureDeclarations(FindUsagesFileRankerFeatures) +class FindUsagesFileRankerFeatureProvider : FeatureProvider(FindUsagesFileRankerFeatures) { override fun computeFeatures(instance: FindUsagesRankingFileInfo, requiredOutput: FeatureSet): List = buildLazyFeaturesList(requiredOutput) { if (instance.candidateFile != null) { diff --git a/plugins/findUsagesMl/src/com/intellij/findUsagesMl/FileRankerMlServiceImpl.kt b/plugins/findUsagesMl/src/com/intellij/findUsagesMl/FileRankerMlServiceImpl.kt index 5dce961a13b4..501cdad3d50e 100644 --- a/plugins/findUsagesMl/src/com/intellij/findUsagesMl/FileRankerMlServiceImpl.kt +++ b/plugins/findUsagesMl/src/com/intellij/findUsagesMl/FileRankerMlServiceImpl.kt @@ -7,9 +7,8 @@ import com.intellij.openapi.project.Project import com.intellij.openapi.util.registry.Registry import com.intellij.openapi.vfs.VirtualFile import com.intellij.psi.search.FileRankerMlService -import com.jetbrains.ml.api.feature.FeatureSet -import com.jetbrains.ml.tools.logs.MLLogsTree -import com.jetbrains.ml.tools.logs.MLTreeLogger +import com.jetbrains.mlapi.logs.MLLogsTree +import com.jetbrains.mlapi.logs.MLTreeLogger import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.launch import java.util.concurrent.atomic.AtomicLong @@ -115,26 +114,30 @@ class FileRankerMlServiceImpl(private val coroutineScope: CoroutineScope) : File projectPath: String) { val tree = MLLogsTree( analysis = analysisProvider.provideAnalysisTargets( - info = FindUsagesFileRankingAnalysisInfo(isUsage = isUsage, - timestamp = timeStamp, - isSearchValid = isSearchValid, - numberOfUsageFiles = numberOfUsageFiles, - numberOfCandidates = numberOfCandidates, - indexInOriginalOrder = indexInOriginalOrder, - activeSessionId = activeSessionId, - finishSessionId = sessionId)) , - features = featureProvider.provideFeatures( - instance = FindUsagesRankingFileInfo(queryNames = queryNames, - queryFiles = queryFiles, - candidateFile = file, - recentFilesList = recentFilesList, - timeStamp = timeStamp, - projectPath = projectPath), - requiredOutput = FeatureSet.ALL + info = FindUsagesFileRankingAnalysisInfo( + customSessionId = sessionId, + isUsage = isUsage, + timestamp = timeStamp, + isSearchValid = isSearchValid, + numberOfUsageFiles = numberOfUsageFiles, + numberOfCandidates = numberOfCandidates, + indexInOriginalOrder = indexInOriginalOrder, + activeSessionId = activeSessionId, + finishSessionId = sessionId + ) ), + features = featureProvider.provideFeatures( + instance = FindUsagesRankingFileInfo( + queryNames = queryNames, + queryFiles = queryFiles, + candidateFile = file, + recentFilesList = recentFilesList, + timeStamp = timeStamp, + projectPath = projectPath + ) + ) ) - logger.log(tree = tree, - customSessionId = sessionId) + logger.log(tree) } private fun logInvalid(activeSessionId: Long, finishedSessionId: Long) { diff --git a/plugins/findUsagesMl/test/com/intellij/findUsagesMl/FindUsagesFileRankerFeatureProviderTest.kt b/plugins/findUsagesMl/test/com/intellij/findUsagesMl/FindUsagesFileRankerFeatureProviderTest.kt index cb4006606301..c349bcce734c 100644 --- a/plugins/findUsagesMl/test/com/intellij/findUsagesMl/FindUsagesFileRankerFeatureProviderTest.kt +++ b/plugins/findUsagesMl/test/com/intellij/findUsagesMl/FindUsagesFileRankerFeatureProviderTest.kt @@ -2,9 +2,9 @@ package com.intellij.findUsagesMl import com.intellij.openapi.fileTypes.PlainTextFileType import com.intellij.openapi.vfs.VirtualFile -import com.jetbrains.ml.api.feature.Feature -import com.jetbrains.ml.api.feature.FeatureDeclaration -import com.jetbrains.ml.api.feature.FeatureSet +import com.jetbrains.mlapi.feature.Feature +import com.jetbrains.mlapi.feature.FeatureDeclaration +import com.jetbrains.mlapi.feature.FeatureSet import org.apache.commons.text.similarity.JaroWinklerSimilarity import org.junit.jupiter.api.Test import java.lang.reflect.Method diff --git a/plugins/search-everywhere-ml/plugin-content.yaml b/plugins/search-everywhere-ml/plugin-content.yaml index ac8730c7d52b..23ecadd74e60 100644 --- a/plugins/search-everywhere-ml/plugin-content.yaml +++ b/plugins/search-everywhere-ml/plugin-content.yaml @@ -12,20 +12,10 @@ module: intellij.searchEverywhereMl files: - name: $MAVEN_REPOSITORY$/ai/grazie/model/swagger-annotations-jvm/0/swagger-annotations-jvm-0.jar -- name: lib/jetbrains.mlapi.catboost.shadow.need.slf4j.jar - library: jetbrains.mlapi.catboost.shadow.need.slf4j +- name: lib/jetbrains.mlapi.core.jar + library: jetbrains.mlapi.core files: - - name: $MAVEN_REPOSITORY$/com/jetbrains/mlapi/catboost-shadow-need-slf4j/1/catboost-shadow-need-slf4j-1.jar - reason: <- intellij.platform.ml.logs -- name: lib/jetbrains.mlapi.ml.api.jar - library: jetbrains.mlapi.ml.api - files: - - name: $MAVEN_REPOSITORY$/com/jetbrains/mlapi/ml-api/0/ml-api-0.jar - reason: <- intellij.platform.ml.logs -- name: lib/jetbrains.mlapi.ml.tools.jar - library: jetbrains.mlapi.ml.tools - files: - - name: $MAVEN_REPOSITORY$/com/jetbrains/mlapi/ml-tools/0/ml-tools-0.jar + - name: $MAVEN_REPOSITORY$/com/jetbrains/mlapi/mlapi-core/0/mlapi-core-0.jar reason: <- intellij.platform.ml.logs - name: lib/modules/intellij.searchEverywhereMl.ranking.core.jar contentModules: diff --git a/python/intellij.python.ml.features/BUILD.bazel b/python/intellij.python.ml.features/BUILD.bazel index 8fd82f0efe79..6f71257bc70d 100644 --- a/python/intellij.python.ml.features/BUILD.bazel +++ b/python/intellij.python.ml.features/BUILD.bazel @@ -30,6 +30,8 @@ jvm_library( "//platform/core-ui", "//platform/platform-impl/codeinsight-inline:ide-codeinsight-inline", "@lib//:python-ml-features-jetbrains-ml-models-python-imports-ranking-model", + "@lib//:jetbrains-mlapi-catboost", + "@lib//:jetbrains-mlapi-catboost-shadow-need-slf4j", ] ) ### auto-generated section `build intellij.python.ml.features` end \ No newline at end of file diff --git a/python/intellij.python.ml.features/intellij.python.ml.features.iml b/python/intellij.python.ml.features/intellij.python.ml.features.iml index 4f18f73ebb4e..7d7f1f0a470b 100644 --- a/python/intellij.python.ml.features/intellij.python.ml.features.iml +++ b/python/intellij.python.ml.features/intellij.python.ml.features.iml @@ -25,21 +25,23 @@ - + - - cff0591184b48356a622f7635c6de1399fdbeb161f98d68206d09807c83fa98c + + 2dbd0ae71bac24a83db76238f0a0cf2bd6b0fd1963b6827371ae8813a0434827 - + - + + + \ No newline at end of file diff --git a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/BaseProjectFeatures.kt b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/BaseProjectFeatures.kt index b216b651e961..65f53915d3c2 100644 --- a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/BaseProjectFeatures.kt +++ b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/BaseProjectFeatures.kt @@ -5,12 +5,16 @@ package com.intellij.python.ml.features.imports.features import com.intellij.openapi.application.readAction import com.intellij.openapi.fileEditor.FileEditorManager import com.intellij.openapi.fileEditor.TextEditor +import com.intellij.platform.ml.logs.IJFeatureDeclarations import com.intellij.psi.PsiElement import com.intellij.psi.PsiManager import com.intellij.psi.search.FileTypeIndex import com.intellij.psi.search.GlobalSearchScope import com.intellij.psi.util.PsiTreeUtil -import com.jetbrains.ml.api.feature.* +import com.jetbrains.mlapi.feature.Feature +import com.jetbrains.mlapi.feature.FeatureContainer +import com.jetbrains.mlapi.feature.FeatureDeclaration +import com.jetbrains.mlapi.feature.FeatureSet import com.jetbrains.python.PythonFileType import com.jetbrains.python.psi.* @@ -31,19 +35,15 @@ enum class FileExtensionType { PXI // .pxi files } -object BaseProjectFeatures : ImportRankingContextFeatures() { - object Features { +object BaseProjectFeatures : ImportRankingContextFeatures(Features) { + object Features : FeatureContainer { val NUM_PYTHON_FILES_IN_PROJECT: FeatureDeclaration = FeatureDeclaration.int("num_python_files_in_project") { "The estimated amount of files in the project (by a power of 2)" }.nullable() - val PSI_PARENT_OF_ORIG: List?>> = (1..5).map { i -> FeatureDeclaration.aClass("psi_parent_of_orig_$i") { "PSI parent of original element #$i" }.nullable() } - val FILE_EXTENSION_TYPE: FeatureDeclaration?> = FeatureDeclaration.enum("file_extension_type") { "extension of the original python file" }.nullable() + val PSI_PARENT_OF_ORIG: List?>> = (1..5).map { i -> IJFeatureDeclarations.aClass("psi_parent_of_orig_$i") { "PSI parent of original element #$i" }.nullable() } + val FILE_EXTENSION_TYPE: FeatureDeclaration = IJFeatureDeclarations.enum("file_extension_type", lazyDescription = { "extension of the original python file" }).nullable() } - override val featureComputationPolicy: FeatureComputationPolicy = FeatureComputationPolicy(true, true) - - override val namespaceFeatureDeclarations: List> = extractFeatureDeclarations(Features) - override suspend fun computeNamespaceFeatures(instance: ImportRankingContext, filter: FeatureSet): List = buildList { val candidates = instance.candidates if (candidates.isEmpty()) return@buildList diff --git a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/CandidatesListFeatures.kt b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/CandidatesListFeatures.kt index f95f0473f252..b87b1ae525d6 100644 --- a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/CandidatesListFeatures.kt +++ b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/CandidatesListFeatures.kt @@ -1,10 +1,13 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.python.ml.features.imports.features -import com.jetbrains.ml.api.feature.* +import com.jetbrains.mlapi.feature.Feature +import com.jetbrains.mlapi.feature.FeatureContainer +import com.jetbrains.mlapi.feature.FeatureDeclaration +import com.jetbrains.mlapi.feature.FeatureSet -object CandidatesListFeatures : ImportRankingContextFeatures() { - object Features { +object CandidatesListFeatures : ImportRankingContextFeatures(Features) { + object Features : FeatureContainer { val LENGTH: FeatureDeclaration = FeatureDeclaration.int("n_candidates") { "The amount of import candidates" } @@ -13,10 +16,6 @@ object CandidatesListFeatures : ImportRankingContextFeatures() { }.nullable() } - override val namespaceFeatureDeclarations: List> = extractFeatureDeclarations(Features) - - override val featureComputationPolicy: FeatureComputationPolicy = FeatureComputationPolicy(tolerateRedundantFeatures = true, putNullImplicitly = true) - override suspend fun computeNamespaceFeatures(instance: ImportRankingContext, filter: FeatureSet): List = buildList { add(Features.LENGTH with instance.candidates.size) add(Features.HIGHEST_OLD_RELEVANCE with instance.candidates.maxOfOrNull { it.relevance }) diff --git a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/ImportCandidateRelativeFeatures.kt b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/ImportCandidateRelativeFeatures.kt index 3187c6b16514..a4b564abd4b4 100644 --- a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/ImportCandidateRelativeFeatures.kt +++ b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/ImportCandidateRelativeFeatures.kt @@ -1,20 +1,19 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.python.ml.features.imports.features -import com.jetbrains.ml.api.feature.* +import com.jetbrains.mlapi.feature.Feature +import com.jetbrains.mlapi.feature.FeatureContainer +import com.jetbrains.mlapi.feature.FeatureDeclaration +import com.jetbrains.mlapi.feature.FeatureSet -object ImportCandidateRelativeFeatures : ImportCandidateFeatures() { - object Features { +object ImportCandidateRelativeFeatures : ImportCandidateFeatures(Features) { + object Features : FeatureContainer { val RELATIVE_POSITION: FeatureDeclaration = FeatureDeclaration.int("original_position") { "The import's position without ML ranking" } } - override val namespaceFeatureDeclarations: List> = extractFeatureDeclarations(Features) - - override val featureComputationPolicy: FeatureComputationPolicy = FeatureComputationPolicy(tolerateRedundantFeatures = true, putNullImplicitly = true) - - override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureSet): List = buildList { + override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureSet): List = buildList { add(Features.RELATIVE_POSITION with instance.candidates.indexOf(instance.candidate)) } } diff --git a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/ImportsFeatures.kt b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/ImportsFeatures.kt index 9040aebc4683..19b6a9e8f1e4 100644 --- a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/ImportsFeatures.kt +++ b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/ImportsFeatures.kt @@ -6,23 +6,22 @@ import com.intellij.openapi.fileEditor.FileEditorManager import com.intellij.psi.PsiFile import com.intellij.psi.PsiManager import com.intellij.psi.util.QualifiedName -import com.jetbrains.ml.api.feature.* +import com.jetbrains.mlapi.feature.Feature +import com.jetbrains.mlapi.feature.FeatureContainer +import com.jetbrains.mlapi.feature.FeatureDeclaration +import com.jetbrains.mlapi.feature.FeatureSet import com.jetbrains.python.psi.PyFile import com.jetbrains.python.psi.PyFromImportStatement import com.jetbrains.python.psi.PyImportElement -object ImportsFeatures : ImportCandidateFeatures() { - object Features { +object ImportsFeatures : ImportCandidateFeatures(Features) { + object Features : FeatureContainer { val EXISTING_IMPORT_FROM_PREFIX: FeatureDeclaration = FeatureDeclaration.int("existing_import_from_prefix") { "A maximal prefix for which there exists some import from" }.nullable() val EXISTING_IMPORT_PREFIX: FeatureDeclaration = FeatureDeclaration.int("existing_import_prefix") { "A maximal prefix for which there exists some import" }.nullable() val NEEDED_IMPORT_FROM_PREFIX: FeatureDeclaration = FeatureDeclaration.int("needed_import_from_prefix") { "COMPONENT_COUNT - EXISTING_IMPORT_FROM_PREFIX" }.nullable() val NEEDED_IMPORT_PREFIX: FeatureDeclaration = FeatureDeclaration.int("needed_import_prefix") { "COMPONENT_COUNT - EXISTING_IMPORT_PREFIX" }.nullable() } - override val featureComputationPolicy: FeatureComputationPolicy = FeatureComputationPolicy(true, true) - - override val namespaceFeatureDeclarations: List> = extractFeatureDeclarations(Features) - override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureSet): List = buildList { val importCandidate = instance.candidate val project = readAction { importCandidate.importable?.project } ?: return@buildList diff --git a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/NeighborFilesImportsFeatures.kt b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/NeighborFilesImportsFeatures.kt index 78ee32c5f859..251ca2a8a1fd 100644 --- a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/NeighborFilesImportsFeatures.kt +++ b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/NeighborFilesImportsFeatures.kt @@ -5,25 +5,24 @@ import com.intellij.openapi.application.readAction import com.intellij.openapi.fileEditor.FileEditorManager import com.intellij.openapi.vfs.VirtualFile import com.intellij.psi.PsiManager -import com.jetbrains.ml.api.feature.* +import com.jetbrains.mlapi.feature.Feature +import com.jetbrains.mlapi.feature.FeatureContainer +import com.jetbrains.mlapi.feature.FeatureDeclaration +import com.jetbrains.mlapi.feature.FeatureSet import com.jetbrains.python.psi.PyFile import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope private const val FILES_TO_WATCH = 8 -object NeighborFilesImportsFeatures : ImportCandidateFeatures() { - object Features { +object NeighborFilesImportsFeatures : ImportCandidateFeatures(Features) { + object Features : FeatureContainer { val NEIGHBOR_FILES_EXISTING_IMPORT_FROM_PREFIX: FeatureDeclaration = FeatureDeclaration.int("neighbor_files_existing_import_from_prefix") { "A maximal prefix for which there exists some import from" }.nullable() val NEIGHBOR_FILES_EXISTING_IMPORT_PREFIX: FeatureDeclaration = FeatureDeclaration.int("neighbor_files_existing_import_prefix") { "A maximal prefix for which there exists some import" }.nullable() val NEIGHBOR_FILES_NEEDED_IMPORT_FROM_PREFIX: FeatureDeclaration = FeatureDeclaration.int("neighbor_files_needed_import_from_prefix") { "COMPONENT_COUNT - NEIGHBOR_FILES_EXISTING_IMPORT_FROM_PREFIX" }.nullable() val NEIGHBOR_FILES_NEEDED_IMPORT_PREFIX: FeatureDeclaration = FeatureDeclaration.int("neighbor_files_needed_import_prefix") { "COMPONENT_COUNT - NEIGHBOR_FILES_EXISTING_IMPORT_PREFIX" }.nullable() } - override val featureComputationPolicy: FeatureComputationPolicy = FeatureComputationPolicy(true, true) - - override val namespaceFeatureDeclarations: List> = extractFeatureDeclarations(Features) - override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureSet): List = coroutineScope { val importCandidate = instance.candidate if (importCandidate.path == null) { diff --git a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/OpenFilesImportsFeatures.kt b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/OpenFilesImportsFeatures.kt index 44a6beede7d7..e4d4d12c7116 100644 --- a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/OpenFilesImportsFeatures.kt +++ b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/OpenFilesImportsFeatures.kt @@ -4,25 +4,24 @@ package com.intellij.python.ml.features.imports.features import com.intellij.openapi.application.readAction import com.intellij.openapi.fileEditor.FileEditorManager import com.intellij.psi.PsiManager -import com.jetbrains.ml.api.feature.* +import com.jetbrains.mlapi.feature.Feature +import com.jetbrains.mlapi.feature.FeatureContainer +import com.jetbrains.mlapi.feature.FeatureDeclaration +import com.jetbrains.mlapi.feature.FeatureSet import com.jetbrains.python.psi.PyFile import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope private const val FILES_TO_WATCH = 8 -object OpenFilesImportsFeatures : ImportCandidateFeatures() { - object Features { +object OpenFilesImportsFeatures : ImportCandidateFeatures(Features) { + object Features : FeatureContainer { val OPEN_FILES_EXISTING_IMPORT_FROM_PREFIX: FeatureDeclaration = FeatureDeclaration.int("open_files_existing_import_from_prefix") { "A maximal prefix for which there exists some import from" }.nullable() val OPEN_FILES_EXISTING_IMPORT_PREFIX: FeatureDeclaration = FeatureDeclaration.int("open_files_existing_import_prefix") { "A maximal prefix for which there exists some import" }.nullable() val OPEN_FILES_NEEDED_IMPORT_FROM_PREFIX: FeatureDeclaration = FeatureDeclaration.int("open_files_needed_import_from_prefix") { "COMPONENT_COUNT - OPEN_FILES_EXISTING_IMPORT_FROM_PREFIX" }.nullable() val OPEN_FILES_NEEDED_IMPORT_PREFIX: FeatureDeclaration = FeatureDeclaration.int("open_files_needed_import_prefix") { "COMPONENT_COUNT - OPEN_FILES_EXISTING_IMPORT_PREFIX" }.nullable() } - override val featureComputationPolicy: FeatureComputationPolicy = FeatureComputationPolicy(true, true) - - override val namespaceFeatureDeclarations: List> = extractFeatureDeclarations(Features) - override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureSet): List = coroutineScope { val importCandidate = instance.candidate if (importCandidate.path == null) { diff --git a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/PrimitiveImportFeatures.kt b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/PrimitiveImportFeatures.kt index 7895917b108a..74949b4894a9 100644 --- a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/PrimitiveImportFeatures.kt +++ b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/PrimitiveImportFeatures.kt @@ -1,11 +1,14 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.python.ml.features.imports.features -import com.jetbrains.ml.api.feature.* +import com.jetbrains.mlapi.feature.Feature +import com.jetbrains.mlapi.feature.FeatureContainer +import com.jetbrains.mlapi.feature.FeatureDeclaration +import com.jetbrains.mlapi.feature.FeatureSet -object PrimitiveImportFeatures : ImportCandidateFeatures() { - object Features { +object PrimitiveImportFeatures : ImportCandidateFeatures(Features) { + object Features : FeatureContainer { val RELEVANCE: FeatureDeclaration = FeatureDeclaration.int("old_relevance") { """ Heuristic-based relevance computed in com.jetbrains.python.codeInsight.completion.PyCompletionUtilsKt.computeCompletionWeight @@ -16,10 +19,6 @@ object PrimitiveImportFeatures : ImportCandidateFeatures() { }.nullable() } - override val namespaceFeatureDeclarations: List> = extractFeatureDeclarations(Features) - - override val featureComputationPolicy: FeatureComputationPolicy = FeatureComputationPolicy(tolerateRedundantFeatures = true, putNullImplicitly = true) - override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureSet): List = buildList { add(Features.RELEVANCE with instance.candidate.relevance) add(Features.COMPONENT_COUNT with instance.candidate.path?.componentCount) diff --git a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/PsiStructureFeatures.kt b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/PsiStructureFeatures.kt index 982d3b015045..cceea60ecc9c 100644 --- a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/PsiStructureFeatures.kt +++ b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/PsiStructureFeatures.kt @@ -2,20 +2,20 @@ package com.intellij.python.ml.features.imports.features import com.intellij.openapi.application.readAction +import com.intellij.platform.ml.logs.IJFeatureDeclarations import com.intellij.psi.PsiElement -import com.jetbrains.ml.api.feature.* +import com.jetbrains.mlapi.feature.Feature +import com.jetbrains.mlapi.feature.FeatureContainer +import com.jetbrains.mlapi.feature.FeatureDeclaration +import com.jetbrains.mlapi.feature.FeatureSet -object PsiStructureFeatures : ImportCandidateFeatures() { - object Features { - val PSI_CLASS: FeatureDeclaration?> = FeatureDeclaration.aClass("importable_class") { "PSI class of the imported element" }.nullable() - val PSI_PARENT: List?>> = (1..4).map { i -> FeatureDeclaration.aClass("psi_parent_$i") { "PSI parent #$i" }.nullable() } +object PsiStructureFeatures : ImportCandidateFeatures(Features) { + object Features : FeatureContainer { + val PSI_CLASS: FeatureDeclaration?> = IJFeatureDeclarations.aClass("importable_class") { "PSI class of the imported element" }.nullable() + val PSI_PARENT: List?>> = (1..4).map { i -> IJFeatureDeclarations.aClass("psi_parent_$i") { "PSI parent #$i" }.nullable() } } - override val namespaceFeatureDeclarations: List> = extractFeatureDeclarations(Features) - - override val featureComputationPolicy: FeatureComputationPolicy = FeatureComputationPolicy(tolerateRedundantFeatures = true, putNullImplicitly = true) - override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureSet): List = buildList { readAction { add(Features.PSI_CLASS with (instance.candidate.importable?.javaClass)) diff --git a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/RelevanceEvaluationFeatures.kt b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/RelevanceEvaluationFeatures.kt index 8625c39f03cd..c27c9b9562f7 100644 --- a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/RelevanceEvaluationFeatures.kt +++ b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/RelevanceEvaluationFeatures.kt @@ -5,8 +5,12 @@ import com.intellij.openapi.application.readAction import com.intellij.openapi.module.ModuleUtilCore import com.intellij.openapi.projectRoots.Sdk import com.intellij.openapi.vfs.VirtualFile +import com.intellij.platform.ml.logs.IJFeatureDeclarations import com.intellij.psi.util.QualifiedName -import com.jetbrains.ml.api.feature.* +import com.jetbrains.mlapi.feature.Feature +import com.jetbrains.mlapi.feature.FeatureContainer +import com.jetbrains.mlapi.feature.FeatureDeclaration +import com.jetbrains.mlapi.feature.FeatureSet import com.jetbrains.python.sdk.PythonSdkUtil enum class UnderscoresType { @@ -27,17 +31,13 @@ enum class ModuleSourceType { } -object RelevanceEvaluationFeatures : ImportCandidateFeatures() { - object Features { +object RelevanceEvaluationFeatures : ImportCandidateFeatures(Features) { + object Features : FeatureContainer { val UNDERSCORES_IN_PATH: FeatureDeclaration = FeatureDeclaration.int("underscores_in_path") { "number of prefix and suffix underscores in path" }.nullable() - val MODULE_SOURCE_TYPE: FeatureDeclaration?> = FeatureDeclaration.enum("module_source_type") { "info about lib being std, local, or external" }.nullable() - val UNDERSCORES_TYPES_OF_PACKAGES: List?>> = (1..4).map { i -> FeatureDeclaration.enum("underscores_types_of_package_$i") { "underscores types of package #$i" }.nullable() } + val MODULE_SOURCE_TYPE: FeatureDeclaration = IJFeatureDeclarations.enum("module_source_type", lazyDescription = { "info about lib being std, local, or external" }).nullable() + val UNDERSCORES_TYPES_OF_PACKAGES: List> = (1..4).map { i -> IJFeatureDeclarations.enum("underscores_types_of_package_$i", lazyDescription = { "underscores types of package #$i" }).nullable() } } - override val featureComputationPolicy: FeatureComputationPolicy = FeatureComputationPolicy(true, true) - - override val namespaceFeatureDeclarations: List> = extractFeatureDeclarations(Features) - override suspend fun computeNamespaceFeatures(instance: ImportCandidateContext, filter: FeatureSet): List = buildList { val importCandidate = instance.candidate add(Features.UNDERSCORES_IN_PATH with countBoundaryUnderscores(importCandidate.path)) diff --git a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/mlFeatures.kt b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/mlFeatures.kt index d4e15fdeca74..bdf167068f62 100644 --- a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/mlFeatures.kt +++ b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/mlFeatures.kt @@ -1,10 +1,10 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.python.ml.features.imports.features -import com.jetbrains.ml.api.feature.Feature -import com.jetbrains.ml.api.feature.FeatureDeclaration -import com.jetbrains.ml.api.feature.FeatureSet -import com.jetbrains.ml.api.feature.suspendable.FeatureProvider +import com.jetbrains.mlapi.feature.Feature +import com.jetbrains.mlapi.feature.FeatureDeclaration +import com.jetbrains.mlapi.feature.FeatureSet +import com.jetbrains.mlapi.feature.suspendable.AsyncFeatureProvider import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope @@ -43,14 +43,14 @@ internal object FeaturesRegistry { jobs.flatMap { it.await() } } - private suspend fun FeatureProvider.computeFeaturesWithImplicitNull( + private suspend fun AsyncFeatureProvider.computeFeaturesWithImplicitNull( instance: T, filter: FeatureSet ): Collection { val features = provideFeatures(instance, filter) - if (!featureComputationPolicy.putNullImplicitly) { - return features - } + //if (!featureComputationPolicy.putNullImplicitly) { + // return features + //} val implicitNullFeatures = mutableListOf() for (declaration in featureDeclarations) { if (declaration.isNullable && !filter.contains(declaration)) { diff --git a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/mlUnits.kt b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/mlUnits.kt index 19154354dc6d..674f75506e29 100644 --- a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/mlUnits.kt +++ b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/features/mlUnits.kt @@ -1,7 +1,9 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.python.ml.features.imports.features -import com.jetbrains.ml.api.feature.suspendable.FeatureProvider +import com.jetbrains.mlapi.feature.FeatureComputationPolicy +import com.jetbrains.mlapi.feature.FeatureContainer +import com.jetbrains.mlapi.feature.suspendable.AsyncFeatureProvider import com.jetbrains.python.codeInsight.imports.ImportCandidateHolder @@ -15,6 +17,16 @@ data class ImportCandidateContext( ) -abstract class ImportRankingContextFeatures : FeatureProvider.InNamespace("import_candidates_list") +abstract class ImportRankingContextFeatures(featuresContainer: FeatureContainer) : + AsyncFeatureProvider.InNamespace( + "import_candidates_list", '_', + featuresContainer, + FeatureComputationPolicy(true, true) + ) -abstract class ImportCandidateFeatures : FeatureProvider.InNamespace("import_candidate") +abstract class ImportCandidateFeatures(featuresContainer: FeatureContainer) : + AsyncFeatureProvider.InNamespace( + "import_candidate", '_', + featuresContainer, + FeatureComputationPolicy(true, true) + ) diff --git a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlAB.kt b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlAB.kt index ee93f73e9440..d8d2bcc767b6 100644 --- a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlAB.kt +++ b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlAB.kt @@ -6,8 +6,8 @@ import com.intellij.openapi.components.Service import com.intellij.openapi.components.service import com.intellij.openapi.util.registry.Registry import com.intellij.python.ml.features.imports.FinalImportRankingStatusService.RegistryOption -import com.jetbrains.ml.tools.model.MLModel -import com.jetbrains.ml.tools.model.catboost.prediction.CatBoostRegressionResult +import com.jetbrains.mlapi.model.MLModel +import com.jetbrains.mlapi.model.prediction.RegressionResult @Service internal class FinalImportRankingStatusService { @@ -47,6 +47,6 @@ internal sealed class FinalImportRankingStatus( val mlModelUnavailable: Boolean, val registryOption: RegistryOption, ) { - class Enabled(val mlModel: MLModel, registryOption: RegistryOption) : FinalImportRankingStatus(true, false, registryOption) + class Enabled(val mlModel: MLModel, registryOption: RegistryOption) : FinalImportRankingStatus(true, false, registryOption) class Disabled(mlModelUnavailable: Boolean, registryOption: RegistryOption) : FinalImportRankingStatus(false, mlModelUnavailable, registryOption) } diff --git a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlAnalysis.kt b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlAnalysis.kt index f5b34b1a2c22..ce1dace8982a 100644 --- a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlAnalysis.kt +++ b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlAnalysis.kt @@ -1,24 +1,23 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.python.ml.features.imports -import com.jetbrains.ml.api.logs.BooleanEventField -import com.jetbrains.ml.api.logs.EnumEventField -import com.jetbrains.ml.api.logs.IntEventField -import com.jetbrains.ml.api.logs.LongEventField +import com.intellij.platform.ml.logs.IJFeatureDeclarations +import com.jetbrains.mlapi.feature.FeatureContainer +import com.jetbrains.mlapi.feature.FeatureDeclaration -internal object ContextAnalysis { - val CANCELLED = BooleanEventField("selection_cancelled", null) { "No item has been selected" } - val SELECTED_POSITION = IntEventField("selected_position") { "The position of the selected import statement" } - val MODEL_UNAVAILABLE = BooleanEventField("model_unavailable", null) { "ML model was unavailable" } - val SELECTED_POSITION_INITIAL = IntEventField("selected_position_initial") { "The position of the selected import statement, if the final ML ranking would not have happened" } - val TIME_MS_TO_DISPLAY = LongEventField("time_ms_before_displayed") { "Duration from the quickfix start until when the imports were displayed" } - val TIME_MS_BEFORE_CLOSED = LongEventField("time_ms_before_closed") { "Duration from the quickfix start until the pop-up was closed" } - val ML_LOGGING_STATE = EnumEventField("ml_logging_state", null, { "State of the ML session logging" }, LoggingOption::class.java) - val ML_ENABLED = BooleanEventField("ml_enabled") { "Machine Learning ranking is enabled" } - val REGISTRY_OPTION = EnumEventField("registry_option", null, { "Registry option of the experiment status" }, FinalImportRankingStatusService.RegistryOption::class.java) +internal object ContextAnalysis : FeatureContainer { + val CANCELLED = FeatureDeclaration.boolean("selection_cancelled") { "No item has been selected" } + val SELECTED_POSITION = FeatureDeclaration.int("selected_position") { "The position of the selected import statement" } + val MODEL_UNAVAILABLE = FeatureDeclaration.boolean("model_unavailable") { "ML model was unavailable" } + val SELECTED_POSITION_INITIAL = FeatureDeclaration.int("selected_position_initial") { "The position of the selected import statement, if the final ML ranking would not have happened" } + val TIME_MS_TO_DISPLAY = FeatureDeclaration.long("time_ms_before_displayed") { "Duration from the quickfix start until when the imports were displayed" } + val TIME_MS_BEFORE_CLOSED = FeatureDeclaration.long("time_ms_before_closed") { "Duration from the quickfix start until the pop-up was closed" } + val ML_LOGGING_STATE = IJFeatureDeclarations.enum("ml_logging_state", lazyDescription = { "State of the ML session logging" }) + val ML_ENABLED = FeatureDeclaration.boolean("ml_enabled") { "Machine Learning ranking is enabled" } + val REGISTRY_OPTION = IJFeatureDeclarations.enum("registry_option", lazyDescription = { "Registry option of the experiment status" }) } -internal object CandidateAnalysis { - val MLFieldCorrectElement = BooleanEventField("is_correct") { "The candidate was chosen" } +internal object CandidateAnalysis : FeatureContainer { + val MLFieldCorrectElement = FeatureDeclaration.boolean("is_correct") { "The candidate was chosen" } } diff --git a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlImplementation.kt b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlImplementation.kt index ba3dfe058e8f..46000bca8bda 100644 --- a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlImplementation.kt +++ b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlImplementation.kt @@ -10,12 +10,11 @@ import com.intellij.python.ml.features.imports.features.FeaturesRegistry import com.intellij.python.ml.features.imports.features.ImportCandidateContext import com.intellij.python.ml.features.imports.features.ImportRankingContext import com.intellij.util.application -import com.jetbrains.ml.api.feature.Feature -import com.jetbrains.ml.api.feature.FeatureDeclaration -import com.jetbrains.ml.api.logs.EventPair -import com.jetbrains.ml.tools.logs.MLLogsTree -import com.jetbrains.ml.tools.model.MLModel -import com.jetbrains.ml.tools.model.catboost.prediction.CatBoostRegressionResult +import com.jetbrains.mlapi.feature.Feature +import com.jetbrains.mlapi.feature.FeatureDeclaration +import com.jetbrains.mlapi.logs.MLLogsTree +import com.jetbrains.mlapi.model.MLModel +import com.jetbrains.mlapi.model.prediction.RegressionResult import com.jetbrains.python.codeInsight.imports.ImportCandidateHolder import kotlinx.coroutines.* import java.util.concurrent.atomic.AtomicBoolean @@ -29,7 +28,7 @@ internal class MLApiComputations( internal sealed class FinalCandidatesRanker( protected val contextFeatures: MutableList, - protected val contextAnalysis: MutableList>, + protected val contextAnalysis: MutableList, protected val timestampStarted: Long, ) { @@ -40,9 +39,9 @@ internal sealed class FinalCandidatesRanker( private class ExperimentalMLRanker( contextFeatures: MutableList, - contextAnalysis: MutableList>, + contextAnalysis: MutableList, timestampStarted: Long, - private val mlModel: MLModel, + private val mlModel: MLModel, ) : FinalCandidatesRanker(contextFeatures, contextAnalysis, timestampStarted) { override val mlEnabled = true @@ -55,7 +54,10 @@ private class ExperimentalMLRanker( async { FeaturesRegistry.computeCandidateFeatures(ImportCandidateContext(initialCandidatesOrder, candidate), mlModel.inputFeatures) } }.mapValues { it.value.await() } - val scoreByCandidate = mlModel.predictBatch(contextFeatures + fillTypingFeatures(), importCandidatesFeatures) + val resultContextFeatures = contextFeatures + fillTypingFeatures() + val featuresBatch = importCandidatesFeatures.mapValues { (_, fs) -> fs + resultContextFeatures } + + val scoreByCandidate = mlModel.predictBatch(featuresBatch) val relevanceCandidateOrder = scoreByCandidate.toList() .sortedByDescending { it.second.logit } @@ -79,7 +81,7 @@ private class ExperimentalMLRanker( private class InitialOrderKeepingRanker( contextFeatures: MutableList, - contextLogs: MutableList>, + contextLogs: MutableList, timestampStarted: Long, ) : FinalCandidatesRanker(contextFeatures, contextLogs, timestampStarted) { override val mlEnabled = false @@ -100,7 +102,7 @@ internal fun launchMLRanking(initialCandidatesOrder: MutableList() - val mlContextLogs = mutableListOf>() + val mlContextLogs = mutableListOf() val rankingStatus = service().status val ranker = when (rankingStatus) { is FinalImportRankingStatus.Disabled -> { @@ -124,14 +126,14 @@ internal fun launchMLRanking(initialCandidatesOrder: MutableList, - private val contextAnalysis: MutableList>, + private val contextAnalysis: MutableList, private val importCandidates: Map>, val orderInitial: List, val order: List, private val timestampStarted: Long, ) { private var submitted = AtomicBoolean(false) - private val candidateAnalysis: MutableMap> = mutableMapOf() + private val candidateAnalysis: MutableMap = mutableMapOf() fun submitPopUpClosed() { val timestampClosed = System.currentTimeMillis() diff --git a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlLogs.kt b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlLogs.kt index 40f88aef5264..8af2c277257b 100644 --- a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlLogs.kt +++ b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlLogs.kt @@ -6,20 +6,19 @@ import com.intellij.internal.statistic.service.fus.collectors.CounterUsagesColle import com.intellij.platform.ml.logs.IntelliJFusEventRegister import com.intellij.python.ml.features.imports.features.FeaturesRegistry import com.intellij.util.application -import com.jetbrains.ml.tools.logs.MLTreeLoggers.withOneEvent -import com.jetbrains.ml.tools.logs.extractEventFields +import com.jetbrains.mlapi.logs.MLTreeLogger import kotlin.random.Random internal object PyCharmImportsRankingLogs : CounterUsagesCollector() { - private val GROUP = EventLogGroup("pycharm.quickfix.imports", 7, "ML") - val mlLogger = withOneEvent( - fusEventName = "pycharm_import_statements_ranking", - fusEventRegister = IntelliJFusEventRegister(GROUP), + private val GROUP = EventLogGroup("pycharm.quickfix.imports", 8, "ML") + val mlLogger = MLTreeLogger.withOneEvent( + eventName = "pycharm_import_statements_ranking", + logsEventRegister = IntelliJFusEventRegister(GROUP), treeFeatures = FeaturesRegistry.declarations, treeAnalysis = listOf( - extractEventFields(ContextAnalysis), - extractEventFields(CandidateAnalysis) + ContextAnalysis.extractFeatureDeclarations(), + CandidateAnalysis.extractFeatureDeclarations() ) ) diff --git a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlModel.kt b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlModel.kt index 4014b4f72cef..6a96c26abbf0 100644 --- a/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlModel.kt +++ b/python/intellij.python.ml.features/src/com/intellij/python/ml/features/imports/mlModel.kt @@ -8,10 +8,10 @@ import com.intellij.openapi.project.Project import com.intellij.openapi.startup.ProjectActivity import com.intellij.util.concurrency.AppExecutorUtil import com.jetbrains.ml.models.PythonImportsRankingModelHolder -import com.jetbrains.ml.tools.model.MLModel -import com.jetbrains.ml.tools.model.catboost.prediction.CatBoostRegressionResult -import com.jetbrains.ml.tools.model.pipeline.ModelPipeline -import com.jetbrains.ml.tools.model.pipeline.ModelPipelineLoader +import com.jetbrains.mlapi.bundle.ModelPipelineLoader +import com.jetbrains.mlapi.model.pipeline.ModelPipeline +import com.jetbrains.mlapi.model.prediction.RegressionResult +import com.jetbrains.mlapi.model.prediction.predicting import java.util.concurrent.CompletableFuture @@ -24,23 +24,24 @@ internal class ImportsRankingModelService { private var modelFuture: CompletableFuture? = null - var model: ModelPipeline, CatBoostRegressionResult>? = null + var model: ModelPipeline? = null private set private fun loadModel() { modelFuture?.cancel(true) LOG.info("Loading CatBoost Imports Ranking model") - modelFuture = ModelPipelineLoader.load( + modelFuture = ModelPipelineLoader(this::class.java.classLoader).load( PythonImportsRankingModelHolder.getStream(), AppExecutorUtil.getAppExecutorService() ).thenAccept { model -> LOG.info("Successfully loaded imports ranking model") - this.model = model.predicting() + this.model = model.predicting() }.exceptionally { e -> LOG.warn("Failed to load CatBoost imports ranking model", e) null } + } companion object { diff --git a/python/pluginCore/plugin-content.yaml b/python/pluginCore/plugin-content.yaml index 7ffeacdac49a..cb25af4f3efc 100644 --- a/python/pluginCore/plugin-content.yaml +++ b/python/pluginCore/plugin-content.yaml @@ -116,11 +116,20 @@ contentModules: - name: intellij.python.markdown - name: lib/modules/intellij.python.ml.features.jar + projectLibraries: + - name: jetbrains.mlapi.catboost + files: + - name: $MAVEN_REPOSITORY$/com/jetbrains/mlapi/mlapi-catboost/0/mlapi-catboost-0.jar + reason: <- intellij.python.ml.features + - name: jetbrains.mlapi.catboost.shadow.need.slf4j + files: + - name: $MAVEN_REPOSITORY$/com/jetbrains/mlapi/catboost-shadow-need-slf4j/1/catboost-shadow-need-slf4j-1.jar + reason: <- intellij.python.ml.features contentModules: - name: intellij.python.ml.features libraries: jetbrains.ml.models.python.imports.ranking.model: - - name: $MAVEN_REPOSITORY$/com/jetbrains/ml/models/python-imports-ranking-model/coral-panda-republished-4/python-imports-ranking-model-coral-panda-republished-4.jar + - name: $MAVEN_REPOSITORY$/com/jetbrains/ml/models/python-imports-ranking-model/coral-panda-republished-6/python-imports-ranking-model-coral-panda-republished-6.jar - name: lib/modules/intellij.python.parser.jar contentModules: - name: intellij.python.parser