IJ-CR-3383: [ml-completion] updated java model to 0.3.3 - jvm cat boost; updated java experiment model to 0.3.1 - composite model;

GitOrigin-RevId: 000a3805acded0908ec8d36ebcc9ca62828c9134

java/java-impl/intellij.java.impl.iml

@@ -59,13 +59,13 @@
     <orderEntry type="module" module-name="intellij.platform.diff.impl" />
     <orderEntry type="module-library">
       <library name="completion-ranking-java" type="repository">
-        <properties include-transitive-deps="false" maven-id="org.jetbrains.intellij.deps.completion:completion-ranking-java:0.3.1" />
+        <properties include-transitive-deps="false" maven-id="org.jetbrains.intellij.deps.completion:completion-ranking-java:0.3.3" />
         <CLASSES>
-          <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/intellij/deps/completion/completion-ranking-java/0.3.1/completion-ranking-java-0.3.1.jar!/" />
+          <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/intellij/deps/completion/completion-ranking-java/0.3.3/completion-ranking-java-0.3.3.jar!/" />
         </CLASSES>
         <JAVADOC />
         <SOURCES>
-          <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/intellij/deps/completion/completion-ranking-java/0.3.1/completion-ranking-java-0.3.1-sources.jar!/" />
+          <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/intellij/deps/completion/completion-ranking-java/0.3.3/completion-ranking-java-0.3.3-sources.jar!/" />
         </SOURCES>
       </library>
     </orderEntry>

java/java-impl/src/com/intellij/codeInsight/completion/ml/JavaMLRankingProvider.kt

@@ -1,19 +1,13 @@
 // Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
 package com.intellij.codeInsight.completion.ml
 
-import com.intellij.internal.ml.*
-import com.intellij.internal.ml.completion.CompletionRankingModelBase
-import com.intellij.internal.ml.completion.JarCompletionModelProvider
+import com.intellij.internal.ml.catboost.NaiveCatBoostJarCompletionModelProvider
 import com.intellij.java.JavaBundle
 import com.intellij.lang.Language
-import com.jetbrains.completion.ranker.model.java.MLGlassBox
+import com.intellij.lang.java.JavaLanguage
 
-class JavaMLRankingProvider : JarCompletionModelProvider(JavaBundle.message("settings.completion.ml.java.display.name"), "java_features") {
-  override fun createModel(metadata: ModelMetadata): DecisionFunction {
-    return object : CompletionRankingModelBase(metadata) {
-      override fun predict(features: DoubleArray?): Double = MLGlassBox.makePredict(features)
-    }
-  }
+class JavaMLRankingProvider : NaiveCatBoostJarCompletionModelProvider(JavaBundle.message("settings.completion.ml.java.display.name"),
+                                                                      "java_features2", "java_model2") {
 
-  override fun isLanguageSupported(language: Language): Boolean = language.id.compareTo("java", ignoreCase = true) == 0
+  override fun isLanguageSupported(language: Language): Boolean = JavaLanguage.INSTANCE == language
 }

platform/build-scripts/groovy/org/jetbrains/intellij/build/CommunityLibraryLicenses.groovy

@@ -192,8 +192,6 @@ class CommunityLibraryLicenses {
                        url: "https://github.com/raphw/byte-buddy", licenseUrl: "http://www.apache.org/licenses/LICENSE-2.0"),
     new LibraryLicense(name: "caffeine", libraryName: "caffeine", license: "Apache 2.0",
                        url: "https://github.com/ben-manes/caffeine", licenseUrl: "https://github.com/ben-manes/caffeine/blob/master/LICENSE"),
-    new LibraryLicense(name: "CatBoost Model Applier", libraryName: "ai.catboost:catboost-prediction:0.24", license: "Apache 2.0",
-                       url: "https://github.com/catboost/catboost", licenseUrl: "https://github.com/catboost/catboost/blob/master/LICENSE"),
     new LibraryLicense(name: "CGLib", libraryName: "CGLIB", license: "Apache", url: "http://cglib.sourceforge.net/",
                        licenseUrl: "http://www.apache.org/foundation/licence-FAQ.html"),
     new LibraryLicense(name: "classworlds", libraryName: "Maven", transitiveDependency: true, version: "1.1", license: "codehaus",
@@ -663,7 +661,6 @@ class CommunityLibraryLicenses {
     jetbrainsLibrary("completion-log-events"),
     jetbrainsLibrary("completion-ranking-java"),
     jetbrainsLibrary("completion-ranking-java-exp"),
-    jetbrainsLibrary("completion-ranking-java-exp2"),
     jetbrainsLibrary("completion-ranking-kotlin"),
     jetbrainsLibrary("completion-ranking-php-exp"),
     jetbrainsLibrary("completion-ranking-python"),

platform/platform-impl/src/com/intellij/internal/ml/catboost/NaiveCatBoostCompletionModelProvider.kt

@@ -1,5 +1,5 @@
 // Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
-package com.jetbrains.completion.ml.ranker.cb.jvm
+package com.intellij.internal.ml.catboost
 
 import com.intellij.internal.ml.DecisionFunction
 import com.intellij.internal.ml.FeaturesInfo

platform/platform-impl/src/com/intellij/internal/ml/catboost/NaiveCatBoostModel.java

@@ -1,5 +1,5 @@
 // Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
-package com.jetbrains.completion.ml.ranker.cb.jvm;
+package com.intellij.internal.ml.catboost;
 
 import java.io.*;
 import java.nio.ByteBuffer;
@@ -150,7 +150,7 @@ public class NaiveCatBoostModel {
   private static <T> ArrayList<T> readArrayList(DataInputStream stream, Function<DataInputStream, T> readElement) throws IOException {
     int size = stream.readInt();
     assert size > 0;
-    ArrayList<T> list = new ArrayList<T>(size);
+    ArrayList<T> list = new ArrayList<>(size);
     for (int i = 0; i < size; i++) {
       T element = readElement.apply(stream);
       list.add(element);

platform/platform-impl/src/com/intellij/internal/ml/catboost/NaiveCatBoostResourcesModelMetadataReader.kt

@@ -1,4 +1,5 @@
-package com.jetbrains.completion.ml.ranker.cb.jvm
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.ml.catboost
 
 import com.intellij.internal.ml.InconsistentMetadataException
 import com.intellij.internal.ml.ResourcesModelMetadataReader

plugins/completion-ml-ranking-models/intellij.completionMlRankingModels.iml

@@ -12,25 +12,13 @@
     <orderEntry type="module" module-name="intellij.completionMlRanking" />
     <orderEntry type="module-library">
       <library name="completion-ranking-java-exp" type="repository">
-        <properties include-transitive-deps="false" maven-id="org.jetbrains.intellij.deps.completion:completion-ranking-java:0.3.2" />
+        <properties include-transitive-deps="false" maven-id="org.jetbrains.intellij.deps.completion:completion-ranking-java:0.3.1" />
         <CLASSES>
-          <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/intellij/deps/completion/completion-ranking-java/0.3.2/completion-ranking-java-0.3.2.jar!/" />
+          <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/intellij/deps/completion/completion-ranking-java/0.3.1/completion-ranking-java-0.3.1.jar!/" />
         </CLASSES>
         <JAVADOC />
         <SOURCES>
-          <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/intellij/deps/completion/completion-ranking-java/0.3.2/completion-ranking-java-0.3.2-sources.jar!/" />
-        </SOURCES>
-      </library>
-    </orderEntry>
-    <orderEntry type="module-library">
-      <library name="completion-ranking-java-exp2" type="repository">
-        <properties include-transitive-deps="false" maven-id="org.jetbrains.intellij.deps.completion:completion-ranking-java:0.3.3" />
-        <CLASSES>
-          <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/intellij/deps/completion/completion-ranking-java/0.3.3/completion-ranking-java-0.3.3.jar!/" />
-        </CLASSES>
-        <JAVADOC />
-        <SOURCES>
-          <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/intellij/deps/completion/completion-ranking-java/0.3.3/completion-ranking-java-0.3.3-sources.jar!/" />
+          <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/intellij/deps/completion/completion-ranking-java/0.3.1/completion-ranking-java-0.3.1-sources.jar!/" />
         </SOURCES>
       </library>
     </orderEntry>
@@ -79,27 +67,6 @@
         </SOURCES>
       </library>
     </orderEntry>
-    <orderEntry type="module-library">
-      <library name="ai.catboost:catboost-prediction:0.24" type="repository">
-        <properties maven-id="ai.catboost:catboost-prediction:0.24">
-          <exclude>
-            <dependency maven-id="com.google.code.findbugs:jsr305" />
-            <dependency maven-id="org.slf4j:slf4j-api" />
-          </exclude>
-        </properties>
-        <CLASSES>
-          <root url="jar://$MAVEN_REPOSITORY$/ai/catboost/catboost-prediction/0.24/catboost-prediction-0.24.jar!/" />
-          <root url="jar://$MAVEN_REPOSITORY$/javax/validation/validation-api/1.1.0.Final/validation-api-1.1.0.Final.jar!/" />
-          <root url="jar://$MAVEN_REPOSITORY$/ai/catboost/catboost-common/0.24/catboost-common-0.24.jar!/" />
-        </CLASSES>
-        <JAVADOC />
-        <SOURCES>
-          <root url="jar://$MAVEN_REPOSITORY$/ai/catboost/catboost-prediction/0.24/catboost-prediction-0.24-sources.jar!/" />
-          <root url="jar://$MAVEN_REPOSITORY$/javax/validation/validation-api/1.1.0.Final/validation-api-1.1.0.Final-sources.jar!/" />
-          <root url="jar://$MAVEN_REPOSITORY$/ai/catboost/catboost-common/0.24/catboost-common-0.24-sources.jar!/" />
-        </SOURCES>
-      </library>
-    </orderEntry>
     <orderEntry type="module-library">
       <library name="completion-ranking-php-exp" type="repository">
         <properties maven-id="org.jetbrains.intellij.deps.completion:completion-ranking-php:0.2.2" />

plugins/completion-ml-ranking-models/resources/META-INF/plugin.xml

@@ -14,11 +14,9 @@
   <depends>com.intellij.completion.ml.ranking</depends>
 
   <extensions defaultExtensionNs="com.intellij">
-    <preloadingActivity implementation="com.jetbrains.completion.ml.ranker.cb.jvm.PreloadNaiveCatBoostModel" />
     <completion.ml.model implementation="com.jetbrains.completion.ml.ranker.ExperimentKotlinMLRankingProvider"/>
     <completion.ml.model implementation="com.jetbrains.completion.ml.ranker.ExperimentScalaMLRankingProvider"/>
     <completion.ml.model implementation="com.jetbrains.completion.ml.ranker.ExperimentJavaMLRankingProvider"/>
-    <completion.ml.model implementation="com.jetbrains.completion.ml.ranker.ExperimentJavaMLRankingProvider2"/>
     <completion.ml.model implementation="com.jetbrains.completion.ml.ranker.ExperimentPythonMLRankingProvider"/>
     <completion.ml.model implementation="com.jetbrains.completion.ml.ranker.ExperimentRustMLRankingProvider"/>
     <completion.ml.model implementation="com.jetbrains.completion.ml.ranker.ExperimentPhpMLRankingProvider"/>

plugins/completion-ml-ranking-models/src/com/jetbrains/completion/ml/ranker/ExperimentJavaMLRankingProvider.kt

@@ -1,22 +1,22 @@
 package com.jetbrains.completion.ml.ranker
 
 import com.intellij.completion.ml.ranker.ExperimentModelProvider
+import com.intellij.internal.ml.DecisionFunction
+import com.intellij.internal.ml.ModelMetadata
+import com.intellij.internal.ml.completion.CompletionRankingModelBase
+import com.intellij.internal.ml.completion.JarCompletionModelProvider
 import com.intellij.lang.Language
-import com.jetbrains.completion.ml.ranker.cb.JarCatBoostCompletionModelProvider
-import com.jetbrains.completion.ml.ranker.cb.jvm.NaiveCatBoostJarCompletionModelProvider
+import com.jetbrains.completion.ranker.model.java.MLGlassBox
 
-class ExperimentJavaMLRankingProvider: JarCatBoostCompletionModelProvider(
-  CompletionRankingModelsBundle.message("ml.completion.experiment.model.java"), "java_features", "java_model"), ExperimentModelProvider {
+class ExperimentJavaMLRankingProvider: JarCompletionModelProvider(
+  CompletionRankingModelsBundle.message("ml.completion.experiment.model.java"), "java_features"), ExperimentModelProvider {
+  override fun createModel(metadata: ModelMetadata): DecisionFunction {
+    return object : CompletionRankingModelBase(metadata) {
+      override fun predict(features: DoubleArray?): Double = MLGlassBox.makePredict(features)
+    }
+  }
 
-  override fun isLanguageSupported(language: Language): Boolean = language.id.compareTo("Java", ignoreCase = true) == 0
+  override fun isLanguageSupported(language: Language): Boolean = language.id.compareTo("java", ignoreCase = true) == 0
 
   override fun experimentGroupNumber(): Int = 13
-}
-
-class ExperimentJavaMLRankingProvider2: NaiveCatBoostJarCompletionModelProvider(
-  CompletionRankingModelsBundle.message("ml.completion.experiment.model.java"), "java_features2", "java_model2"), ExperimentModelProvider {
-
-  override fun isLanguageSupported(language: Language): Boolean = language.id.compareTo("Java", ignoreCase = true) == 0
-
-  override fun experimentGroupNumber(): Int = 14
 }

plugins/completion-ml-ranking-models/src/com/jetbrains/completion/ml/ranker/cb/CatBoostResourcesModelMetadataReader.kt

@@ -1,18 +0,0 @@
-package com.jetbrains.completion.ml.ranker.cb
-
-import ai.catboost.CatBoostModel
-import com.intellij.internal.ml.InconsistentMetadataException
-import com.intellij.internal.ml.ResourcesModelMetadataReader
-
-class CatBoostResourcesModelMetadataReader(metadataHolder: Class<*>,
-                                           featuresDirectory: String,
-                                           private val modelDirectory: String) : ResourcesModelMetadataReader(metadataHolder, featuresDirectory) {
-
-  fun loadModel(): CatBoostModel {
-    val resource = "$modelDirectory/model.cbm"
-    val fileStream = metadataHolder.classLoader.getResourceAsStream(resource)
-                     ?: throw InconsistentMetadataException(
-                       "Metadata file not found: $resource. Resources holder: ${metadataHolder.name}")
-    return CatBoostModel.loadModel(fileStream)
-  }
-}

plugins/completion-ml-ranking-models/src/com/jetbrains/completion/ml/ranker/cb/JarCatBoostCompletionModelProvider.kt

@@ -1,66 +0,0 @@
-package com.jetbrains.completion.ml.ranker.cb
-
-import com.intellij.internal.ml.DecisionFunction
-import com.intellij.internal.ml.FeaturesInfo
-import com.intellij.internal.ml.InconsistentMetadataException
-import com.intellij.internal.ml.completion.CompletionRankingModelBase
-import com.intellij.internal.ml.completion.RankingModelProvider
-import com.intellij.openapi.diagnostic.logger
-import org.jetbrains.annotations.Nls
-import org.jetbrains.annotations.NonNls
-import org.jetbrains.annotations.TestOnly
-
-abstract class JarCatBoostCompletionModelProvider(@Nls(capitalization = Nls.Capitalization.Title) private val displayName: String,
-                                                  @NonNls private val resourceDirectory: String,
-                                                  @NonNls private val modelDirectory: String) : RankingModelProvider {
-  private val lazyModel: DecisionFunction by lazy {
-    val metadataReader = CatBoostResourcesModelMetadataReader(this::class.java, resourceDirectory, modelDirectory)
-    val metadata = FeaturesInfo.buildInfo(metadataReader)
-    val model = metadataReader.loadModel()
-    return@lazy object : CompletionRankingModelBase(metadata) {
-      override fun predict(features: DoubleArray): Double {
-
-        val floatArray = FloatArray(features.size)
-        for (i in features.indices) {
-          floatArray[i] = features[i].toFloat()
-        }
-
-        try {
-          return model.predict(floatArray, emptyArray<String>()).get(0, 0)
-        } catch (t: Throwable) {
-          LOG.error(t)
-          return 0.0
-        }
-      }
-    }
-  }
-
-  override fun getModel(): DecisionFunction = lazyModel
-
-  override fun getDisplayNameInSettings(): String = displayName
-
-  @TestOnly
-  fun assertModelMetadataConsistent() {
-    try {
-      val decisionFunction = model
-      decisionFunction.version()
-
-      val unknownRequiredFeatures = decisionFunction.getUnknownFeatures(decisionFunction.requiredFeatures)
-      assert(unknownRequiredFeatures.isEmpty()) { "All required features should be known, but $unknownRequiredFeatures unknown" }
-
-      val featuresOrder = decisionFunction.featuresOrder
-      val unknownUsedFeatures = decisionFunction.getUnknownFeatures(featuresOrder.map { it.featureName }.distinct())
-      assert(unknownUsedFeatures.isEmpty()) { "All used features should be known, but $unknownUsedFeatures unknown" }
-
-      val features = DoubleArray(featuresOrder.size)
-      decisionFunction.predict(features)
-    }
-    catch (e: InconsistentMetadataException) {
-      throw AssertionError("Model metadata inconsistent", e)
-    }
-  }
-
-  companion object {
-    private val LOG = logger<JarCatBoostCompletionModelProvider>()
-  }
-}

plugins/completion-ml-ranking-models/test/com/jetbrains/completion/ml/ranker/MetadataConsistencyTest.kt

@@ -15,9 +15,6 @@ class MetadataConsistencyTest {
   @Test
   fun testJavaMetadata() = ExperimentJavaMLRankingProvider().assertModelMetadataConsistent()
 
-  @Test
-  fun testJavaMetadata2() = ExperimentJavaMLRankingProvider2().assertModelMetadataConsistent()
-
   @Test
   fun testRustMetadata() = ExperimentRustMLRankingProvider().assertModelMetadataConsistent()
 }

plugins/completion-ml-ranking/resources/META-INF/plugin.xml

@@ -94,6 +94,8 @@
     <statistics.validation.customValidationRule
       implementation="com.intellij.completion.ml.settings.MLCompletionSettingsCollector$MLRankingSettingsValidationRule"/>
     <statistics.counterUsagesCollector implementationClass="com.intellij.completion.ml.settings.MLCompletionSettingsCollector"/>
+
+    <preloadingActivity implementation="com.intellij.completion.ml.ranker.PreloadNaiveCatBoostModel" />
   </extensions>
 
   <projectListeners>

plugins/completion-ml-ranking/src/com/intellij/completion/ml/ranker/PreloadNaiveCatBoostModel.kt

@@ -1,6 +1,7 @@
-package com.jetbrains.completion.ml.ranker.cb.jvm
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.completion.ml.ranker
 
-import com.intellij.completion.ml.ranker.ExperimentModelProvider
+import com.intellij.internal.ml.catboost.NaiveCatBoostJarCompletionModelProvider
 import com.intellij.openapi.application.PreloadingActivity
 import com.intellij.openapi.progress.ProgressIndicator
 import com.intellij.util.PlatformUtils