From 12dabb63a49bc5fa7f866b3f2bee584f5fc620aa Mon Sep 17 00:00:00 2001 From: Vadim Lomshakov Date: Wed, 7 Apr 2021 19:28:16 +0300 Subject: [PATCH] [ml-completion] don't cast ml features to string representation to reduce memory traffic GitOrigin-RevId: 3aa4aad3edc49d49d7a3c30b76efc41a838a6711 --- .../ml/features/ContextFeaturesStorage.kt | 4 ++-- .../ml/features/MLCompletionWeigher.kt | 12 +++++------- .../completion/ml/features/MLFeaturesUtil.kt | 8 ++++---- .../completion/ml/util/RelevanceUtil.kt | 17 ++++++++++------- 4 files changed, 21 insertions(+), 20 deletions(-) diff --git a/plugins/completion-ml-ranking/src/com/intellij/completion/ml/features/ContextFeaturesStorage.kt b/plugins/completion-ml-ranking/src/com/intellij/completion/ml/features/ContextFeaturesStorage.kt index ea113fc8934d..bd4cc7175d95 100644 --- a/plugins/completion-ml-ranking/src/com/intellij/completion/ml/features/ContextFeaturesStorage.kt +++ b/plugins/completion-ml-ranking/src/com/intellij/completion/ml/features/ContextFeaturesStorage.kt @@ -17,9 +17,9 @@ class ContextFeaturesStorage(private val featuresSnapshot: Map(name)?.value override fun classNameValue(name: String): String? = findValue(name) - ?.let { MLFeaturesUtil.valueAsString(it) } + ?.let { MLFeaturesUtil.getClassNameSafe(it) } - override fun asMap(): Map = featuresSnapshot.mapValues { MLFeaturesUtil.valueAsString(it.value) } + override fun asMap(): Map = featuresSnapshot.mapValues { MLFeaturesUtil.getRawValue(it.value).toString() } private inline fun findValue(name: String): T? { return featuresSnapshot[name] as? T diff --git a/plugins/completion-ml-ranking/src/com/intellij/completion/ml/features/MLCompletionWeigher.kt b/plugins/completion-ml-ranking/src/com/intellij/completion/ml/features/MLCompletionWeigher.kt index 870e1af67556..d2b611851a0b 100644 --- a/plugins/completion-ml-ranking/src/com/intellij/completion/ml/features/MLCompletionWeigher.kt +++ b/plugins/completion-ml-ranking/src/com/intellij/completion/ml/features/MLCompletionWeigher.kt @@ -29,19 +29,17 @@ class MLCompletionWeigher : CompletionWeigher() { return if (result.isEmpty()) DummyComparable.EMPTY else DummyComparable(result) } - private class DummyComparable(values: Map) : Comparable { - val representation = calculateRepresentation(values) + internal class DummyComparable(values: Map) : Comparable { + val mlFeatures = values.mapValues { MLFeaturesUtil.getRawValue(it.value) } override fun compareTo(other: Any): Int = 0 - override fun toString(): String = representation + override fun toString(): String { + return mlFeatures.entries.joinToString(",", "[", "]", transform = { "${it.key}=${it.value}" }) + } companion object { val EMPTY = DummyComparable(emptyMap()) - - private fun calculateRepresentation(values: Map): String { - return values.entries.joinToString(",", "[", "]", transform = { "${it.key}=${MLFeaturesUtil.valueAsString(it.value)}" }) - } } } } \ No newline at end of file diff --git a/plugins/completion-ml-ranking/src/com/intellij/completion/ml/features/MLFeaturesUtil.kt b/plugins/completion-ml-ranking/src/com/intellij/completion/ml/features/MLFeaturesUtil.kt index 489c7c36e9cc..95b34ce9939e 100644 --- a/plugins/completion-ml-ranking/src/com/intellij/completion/ml/features/MLFeaturesUtil.kt +++ b/plugins/completion-ml-ranking/src/com/intellij/completion/ml/features/MLFeaturesUtil.kt @@ -6,10 +6,10 @@ import com.intellij.codeInsight.completion.ml.MLFeatureValue import com.intellij.internal.statistic.utils.getPluginInfo internal object MLFeaturesUtil { - fun valueAsString(featureValue: MLFeatureValue): String { + fun getRawValue(featureValue: MLFeatureValue): Any { return when (featureValue) { - is MLFeatureValue.BinaryValue -> if (featureValue.value) "1" else "0" - is MLFeatureValue.FloatValue -> featureValue.value.toString() + is MLFeatureValue.BinaryValue -> if (featureValue.value) 1 else 0 + is MLFeatureValue.FloatValue -> featureValue.value is MLFeatureValue.CategoricalValue -> featureValue.value is MLFeatureValue.ClassNameValue -> getClassNameSafe(featureValue) } @@ -25,7 +25,7 @@ internal object MLFeaturesUtil { private val CLASS_NAMES_CACHE = Caffeine.newBuilder().maximumSize(100).build() - private fun getClassNameSafe(feature: MLFeatureValue.ClassNameValue): String { + fun getClassNameSafe(feature: MLFeatureValue.ClassNameValue): String { val clazz = feature.value val names = CLASS_NAMES_CACHE.get(clazz.name) { if (getPluginInfo(clazz).isSafeToReport()) clazz.getNames() else THIRD_PARTY_NAME }!! return if (feature.useSimpleName) names.simpleName else names.fullName diff --git a/plugins/completion-ml-ranking/src/com/intellij/completion/ml/util/RelevanceUtil.kt b/plugins/completion-ml-ranking/src/com/intellij/completion/ml/util/RelevanceUtil.kt index cff98cfab144..a137df9e42d4 100644 --- a/plugins/completion-ml-ranking/src/com/intellij/completion/ml/util/RelevanceUtil.kt +++ b/plugins/completion-ml-ranking/src/com/intellij/completion/ml/util/RelevanceUtil.kt @@ -1,5 +1,6 @@ package com.intellij.completion.ml.util +import com.intellij.completion.ml.features.MLCompletionWeigher import com.intellij.completion.ml.sorting.FeatureUtils import com.intellij.internal.statistic.utils.PluginType import com.intellij.internal.statistic.utils.getPluginInfo @@ -45,7 +46,7 @@ object RelevanceUtil { relevanceMap.addProximityValues("swift_prox", value) } "kotlin.callableWeight" -> relevanceMap.addDataClassValues("kotlin.callableWeight", value.toString()) - "ml_weigh" -> additionalMap.addCompoundValues("ml", value.toString()) + "ml_weigh" -> additionalMap.addMlFeatures("ml", value) else -> if (acceptValue(value) || name == FeatureUtils.ML_RANK) relevanceMap[name] = value } } @@ -66,13 +67,15 @@ object RelevanceUtil { return substringBefore('@') } - /** - * Proximity features now came like [samePsiFile=true, openedInEditor=false], need to convert to proper map - */ - private fun MutableMap.addCompoundValues(prefix: String, proximity: String) { - val items = proximity.replace("[", "").replace("]", "").split(",") + private fun MutableMap.addMlFeatures(prefix: String, comparable: Any) { + if (comparable !is MLCompletionWeigher.DummyComparable) { + LOG.error("Unexpected value type of `$prefix`: ${comparable.javaClass.simpleName}") + return + } - this.addProperties(prefix, items) + for ((name, value) in comparable.mlFeatures) { + this["${prefix}_$name"] = value + } } private fun MutableMap.addProximityValues(prefix: String, proximity: Any) {