LAB-29: add new path similarity features

GitOrigin-RevId: 2a3c63f6a4d111b6a712a95570715b878f19b7d0

plugins/filePrediction/src/com/intellij/filePrediction/features/FilePredictionSimilarityFeatures.kt

@@ -12,22 +12,38 @@ import com.intellij.openapi.util.SystemInfo
 import com.intellij.openapi.util.io.FileUtil
 import com.intellij.openapi.util.text.StringUtil
 import com.intellij.openapi.util.text.StringUtil.toLowerCase
+import com.intellij.openapi.util.text.Strings
+import com.intellij.openapi.vfs.VfsUtil
 import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.testFramework.LightVirtualFile
-import com.intellij.util.PathUtil
-import java.io.File
+import com.intellij.util.containers.ContainerUtil
+import com.intellij.util.text.NameUtilCore
 
 class FilePredictionSimilarityFeatures : FilePredictionFeatureProvider {
   companion object {
     private val FEATURES = arrayListOf(
+      "ancestor",
+      "common",
+      "common_norm",
+      "common_words",
+      "common_words_norm",
+      "distance",
+      "distance_norm",
       "excluded",
       "in_library",
       "in_project",
       "in_source",
       "light",
       "name_prefix",
+      "name_prefix_norm",
       "path_prefix",
+      "path_prefix_norm",
+      "relative_common",
+      "relative_common_norm",
+      "relative_distance",
+      "relative_distance_norm",
       "relative_path_prefix",
+      "relative_path_prefix_norm",
       "same_dir",
       "same_module"
     )
@@ -60,28 +76,85 @@ class FilePredictionSimilarityFeatures : FilePredictionFeatureProvider {
 
     if (prevFile != null) {
       val newFileName = unify(newFile.name)
-      val newFilePath = unify(newFile.path)
       val prevFileName = unify(prevFile.name)
-      result["name_prefix"] = numerical(StringUtil.commonPrefixLength(newFileName, prevFileName))
+      addNameSimilarity(newFileName, prevFileName, result)
 
-      val prevFilePath = unify(prevFile.path)
-      result["path_prefix"] = numerical(StringUtil.commonPrefixLength(newFilePath, prevFilePath))
+      addPathSimilarity(newFile, prevFile, null, "", result)
 
       if (!project.isDisposed) {
-        val baseDir = project.guessProjectDir()?.path?.let { unify(it) }
-        if (baseDir != null) {
-          val newRelativePath = FileUtil.getRelativePath(baseDir, newFilePath, File.separatorChar, false)
-          val prevRelativePath = FileUtil.getRelativePath(baseDir, prevFilePath, File.separatorChar, false)
-          if (newRelativePath != null && prevRelativePath != null) {
-            result["relative_path_prefix"] = numerical(StringUtil.commonPrefixLength(newRelativePath, prevRelativePath))
-          }
+        val baseDir = project.guessProjectDir()
+        if (baseDir != null && VfsUtil.isAncestor(baseDir, newFile, false) && VfsUtil.isAncestor(baseDir, prevFile, false)) {
+          addPathSimilarity(newFile, prevFile, baseDir, "relative_", result)
         }
       }
-      result["same_dir"] = binary(PathUtil.getParentPath(newFilePath) == PathUtil.getParentPath(prevFilePath))
+
+      val newDir = newFile.parent
+      val prevDir = prevFile.parent
+      result["same_dir"] = binary(FileUtil.pathsEqual(newDir.path, prevDir.path))
+
+      val isAncestor =
+        FileUtil.isAncestor(newDir.path, prevDir.path, false) ||
+        FileUtil.isAncestor(prevDir.path, newDir.path, false)
+      result["ancestor"] = binary(isAncestor)
     }
     return result
   }
 
+  private fun addPathSimilarity(newFile: VirtualFile,
+                                prevFile: VirtualFile,
+                                root: VirtualFile?,
+                                prefix: String,
+                                result: HashMap<String, FilePredictionFeature>) {
+    val newPath = getRelativePath(newFile, root)
+    val prevPath = getRelativePath(prevFile, root)
+    val commonPrefixLen = StringUtil.commonPrefixLength(newPath, prevPath)
+    result["${prefix}path_prefix"] = numerical(commonPrefixLen)
+    result["${prefix}path_prefix_norm"] = numerical((2 * commonPrefixLen.toDouble()) / (newPath.length + prevPath.length))
+
+    val newDirs = FileUtil.splitPath(newPath, '/')
+    val prevDirs = FileUtil.splitPath(prevPath, '/')
+    val common = findCommonAncestor(newDirs, prevDirs)
+
+    val maxDistance = newDirs.size + prevDirs.size - 2
+    result["${prefix}common"] = numerical(common)
+    result["${prefix}common_norm"] = numerical(if (maxDistance != 0) (2 * common.toDouble()) / maxDistance else 0.0)
+
+    val distance = maxDistance - 2 * common
+    result["${prefix}distance"] = numerical(distance)
+    result["${prefix}distance_norm"] = numerical(if (maxDistance != 0) (distance.toDouble() / maxDistance) else 0.0)
+  }
+
+  private fun findCommonAncestor(newDirs: List<String>, prevDirs: List<String>): Int {
+    var i = 0
+    while (i < newDirs.size - 1 && i < prevDirs.size - 1 && FileUtil.namesEqual(newDirs[i], prevDirs[i])) i++
+    return i
+  }
+
+  private fun getRelativePath(file: VirtualFile, ancestor: VirtualFile?): String {
+    if (ancestor != null) {
+      val relative = VfsUtil.getRelativePath(file, ancestor)
+      if (relative != null) {
+        return unify(relative)
+      }
+    }
+    return unify(file.path)
+  }
+
+  private fun addNameSimilarity(newFileName: String, prevFileName: String, result: HashMap<String, FilePredictionFeature>) {
+    val commonPrefixLen = StringUtil.commonPrefixLength(newFileName, prevFileName)
+    result["name_prefix"] = numerical(commonPrefixLen)
+    result["name_prefix_norm"] = numerical((2 * commonPrefixLen.toDouble()) / (newFileName.length + prevFileName.length))
+
+    var common = 0
+    val newWords = ContainerUtil.map2Set(NameUtilCore.nameToWords(newFileName), Strings::toLowerCase)
+    val prevWords = ContainerUtil.map2Set(NameUtilCore.nameToWords(prevFileName), Strings::toLowerCase)
+    for (prevWord in prevWords) {
+      if (newWords.contains(prevWord)) common++
+    }
+    result["common_words"] = numerical(common)
+    result["common_words_norm"] = numerical((2 * common.toDouble()) / (newWords.size + prevWords.size))
+  }
+
   private fun unify(path: String): String {
     val caseSensitive = SystemInfo.isFileSystemCaseSensitive
     return if (caseSensitive) FileUtil.getNameWithoutExtension(path) else FileUtil.getNameWithoutExtension(toLowerCase(path))

plugins/filePrediction/src/com/intellij/filePrediction/logger/FileNavigationLogger.kt

@@ -10,7 +10,7 @@ import com.intellij.openapi.project.Project
 
 internal class FileNavigationLogger : CounterUsagesCollector() {
   companion object {
-    private val GROUP = EventLogGroup("file.prediction", 8)
+    private val GROUP = EventLogGroup("file.prediction", 9)
 
     private var session: IntEventField = EventFields.Int("session")
     private var performance: LongListEventField = EventFields.LongList("performance")

plugins/filePrediction/test/com/intellij/filePrediction/features/FilePredictionCommonFeaturesTest.kt

@@ -17,6 +17,7 @@ import com.intellij.util.io.URLUtil
 import org.jetbrains.jps.model.java.JavaSourceRootType
 import org.jetbrains.jps.model.java.JpsJavaExtensionService
 import java.io.File
+import kotlin.math.abs
 
 class FilePredictionCommonFeaturesTest : CodeInsightFixtureTestCase<ModuleFixtureBuilder<ModuleFixture>>() {
   private fun doTestSimilarityFeatures(prevPath: String,
@@ -46,7 +47,20 @@ class FilePredictionCommonFeaturesTest : CodeInsightFixtureTestCase<ModuleFixtur
     val expected = expectedFeaturesProvider.produce(myFixture.project)
     for (feature in expected.entries) {
       assertTrue("Cannot find feature '${feature.key}' in $actual", actual.containsKey(feature.key))
-      assertEquals("The value of feature '${feature.key}' is different from expected", feature.value, actual[feature.key])
+
+      if (feature.value.value is Double) {
+        val expectedValue = feature.value.value as Double
+        println(feature.key)
+        val actualValue = actual[feature.key]!!.value as Double
+        val diff = abs(expectedValue - actualValue)
+        assertTrue(
+          "The value of feature '${feature.key}' is different from expected. Expected: $expectedValue, Actual: $actualValue",
+          diff < 0.0001
+        )
+      }
+      else {
+        assertEquals("The value of feature '${feature.key}' is different from expected", feature.value, actual[feature.key])
+      }
     }
   }
 
@@ -70,11 +84,52 @@ class FilePredictionCommonFeaturesTest : CodeInsightFixtureTestCase<ModuleFixtur
     }
   }
 
+  fun `test file name with no common word`() {
+    doTestSimilarityFeatures(
+      "prevSome.txt", "nextFile.txt",
+      ConstFileFeaturesProducer(
+        "common_words" to FilePredictionFeature.numerical(0),
+        "common_words_norm" to FilePredictionFeature.numerical(0.0)
+      )
+    )
+  }
+
+  fun `test file name with common word`() {
+    doTestSimilarityFeatures(
+      "prevFile.txt", "nextFile.txt",
+      ConstFileFeaturesProducer(
+        "common_words" to FilePredictionFeature.numerical(1),
+        "common_words_norm" to FilePredictionFeature.numerical(0.5)
+      )
+    )
+  }
+
+  fun `test file name with multiple common words`() {
+    doTestSimilarityFeatures(
+      "prevFileFoo.txt", "fooNextFileSome.txt",
+      ConstFileFeaturesProducer(
+        "common_words" to FilePredictionFeature.numerical(2),
+        "common_words_norm" to FilePredictionFeature.numerical(0.5714)
+      )
+    )
+  }
+
+  fun `test file name with common words and different length`() {
+    doTestSimilarityFeatures(
+      "foo.txt", "nextFileSomeFoo.txt",
+      ConstFileFeaturesProducer(
+        "common_words" to FilePredictionFeature.numerical(1),
+        "common_words_norm" to FilePredictionFeature.numerical(0.4)
+      )
+    )
+  }
+
   fun `test file name prefix for completely different files`() {
     doTestSimilarityFeatures(
       "prevFile.txt", "nextFile.txt",
       ConstFileFeaturesProducer(
-        "name_prefix" to FilePredictionFeature.numerical(0)
+        "name_prefix" to FilePredictionFeature.numerical(0),
+        "name_prefix_norm" to FilePredictionFeature.numerical(0.0)
       )
     )
   }
@@ -83,7 +138,8 @@ class FilePredictionCommonFeaturesTest : CodeInsightFixtureTestCase<ModuleFixtur
     doTestSimilarityFeatures(
       "myPrevFile.txt", "myNextFile.txt",
       ConstFileFeaturesProducer(
-        "name_prefix" to FilePredictionFeature.numerical(2)
+        "name_prefix" to FilePredictionFeature.numerical(2),
+        "name_prefix_norm" to FilePredictionFeature.numerical(0.2)
       )
     )
   }
@@ -92,7 +148,8 @@ class FilePredictionCommonFeaturesTest : CodeInsightFixtureTestCase<ModuleFixtur
     doTestSimilarityFeatures(
       "file.txt", "src/file.txt",
       ConstFileFeaturesProducer(
-        "name_prefix" to FilePredictionFeature.numerical(4)
+        "name_prefix" to FilePredictionFeature.numerical(4),
+        "name_prefix_norm" to FilePredictionFeature.numerical(1.0)
       )
     )
   }
@@ -101,7 +158,18 @@ class FilePredictionCommonFeaturesTest : CodeInsightFixtureTestCase<ModuleFixtur
     doTestSimilarityFeatures(
       "someFile.txt", "src/file.txt",
       ConstFileFeaturesProducer(
-        "name_prefix" to FilePredictionFeature.numerical(0)
+        "name_prefix" to FilePredictionFeature.numerical(0),
+        "name_prefix_norm" to FilePredictionFeature.numerical(0.0)
+      )
+    )
+  }
+
+  fun `test file name with common prefix but different length`() {
+    doTestSimilarityFeatures(
+      "filePrevious.txt", "src/file.txt",
+      ConstFileFeaturesProducer(
+        "name_prefix" to FilePredictionFeature.numerical(4),
+        "name_prefix_norm" to FilePredictionFeature.numerical(0.5)
       )
     )
   }
@@ -110,7 +178,8 @@ class FilePredictionCommonFeaturesTest : CodeInsightFixtureTestCase<ModuleFixtur
     doTestSimilarityFeatures(
       "src/someFile.txt", "src/file.txt",
       ConstFileFeaturesProducer(
-        "name_prefix" to FilePredictionFeature.numerical(0)
+        "name_prefix" to FilePredictionFeature.numerical(0),
+        "name_prefix_norm" to FilePredictionFeature.numerical(0.0)
       )
     )
   }
@@ -119,7 +188,8 @@ class FilePredictionCommonFeaturesTest : CodeInsightFixtureTestCase<ModuleFixtur
     doTestSimilarityFeatures(
       "src/com/site/ui/someFile.txt", "src/com/site/component/file.txt",
       ConstFileFeaturesProducer(
-        "name_prefix" to FilePredictionFeature.numerical(0)
+        "name_prefix" to FilePredictionFeature.numerical(0),
+        "name_prefix_norm" to FilePredictionFeature.numerical(0.0)
       )
     )
   }
@@ -128,7 +198,9 @@ class FilePredictionCommonFeaturesTest : CodeInsightFixtureTestCase<ModuleFixtur
     doTestSimilarityFeatures(
       "prevFile.txt", "nextFile.txt",
       FileFeaturesByProjectPathProducer(
-        "path_prefix" to FilePredictionFeature.numerical(0)
+        "path_prefix" to FilePredictionFeature.numerical(0),
+        "relative_path_prefix" to FilePredictionFeature.numerical(0),
+        "relative_path_prefix_norm" to FilePredictionFeature.numerical(0.0)
       )
     )
   }
@@ -137,7 +209,9 @@ class FilePredictionCommonFeaturesTest : CodeInsightFixtureTestCase<ModuleFixtur
     doTestSimilarityFeatures(
       "src/prevFile.txt", "src/nextFile.txt",
       FileFeaturesByProjectPathProducer(
-        "path_prefix" to FilePredictionFeature.numerical(4)
+        "path_prefix" to FilePredictionFeature.numerical(4),
+        "relative_path_prefix" to FilePredictionFeature.numerical(4),
+        "relative_path_prefix_norm" to FilePredictionFeature.numerical(0.333333333)
       )
     )
   }
@@ -146,7 +220,9 @@ class FilePredictionCommonFeaturesTest : CodeInsightFixtureTestCase<ModuleFixtur
     doTestSimilarityFeatures(
       "src/ui/prevFile.txt", "src/components/nextFile.txt",
       FileFeaturesByProjectPathProducer(
-        "path_prefix" to FilePredictionFeature.numerical(4)
+        "path_prefix" to FilePredictionFeature.numerical(4),
+        "relative_path_prefix" to FilePredictionFeature.numerical(4),
+        "relative_path_prefix_norm" to FilePredictionFeature.numerical(0.21052)
       )
     )
   }
@@ -155,7 +231,81 @@ class FilePredictionCommonFeaturesTest : CodeInsightFixtureTestCase<ModuleFixtur
     doTestSimilarityFeatures(
       "firstFile.txt", "another/nextFile.txt",
       FileFeaturesByProjectPathProducer(
-        "path_prefix" to FilePredictionFeature.numerical(0)
+        "path_prefix" to FilePredictionFeature.numerical(0),
+        "relative_path_prefix" to FilePredictionFeature.numerical(0),
+        "relative_path_prefix_norm" to FilePredictionFeature.numerical(0.0)
+      )
+    )
+  }
+
+  fun `test common ancestor for files in the same directory`() {
+    doTestSimilarityFeatures(
+      "src/firstFile.txt", "src/nextFile.txt",
+      FileFeaturesByProjectPathProducer(
+        "relative_common" to FilePredictionFeature.numerical(1),
+        "relative_common_norm" to FilePredictionFeature.numerical(1.0),
+        "relative_distance" to FilePredictionFeature.numerical(0),
+        "relative_distance_norm" to FilePredictionFeature.numerical(0.0)
+      )
+    )
+  }
+
+  fun `test common ancestor for files with the same name in the same directory`() {
+    doTestSimilarityFeatures(
+      "src/file.txt", "src/file.java",
+      FileFeaturesByProjectPathProducer(
+        "relative_common" to FilePredictionFeature.numerical(1),
+        "relative_common_norm" to FilePredictionFeature.numerical(1.0),
+        "relative_distance" to FilePredictionFeature.numerical(0),
+        "relative_distance_norm" to FilePredictionFeature.numerical(0.0)
+      )
+    )
+  }
+
+  fun `test common ancestor for files in root directory`() {
+    doTestSimilarityFeatures(
+      "firstFile.txt", "nextFile.txt",
+      FileFeaturesByProjectPathProducer(
+        "relative_common" to FilePredictionFeature.numerical(0),
+        "relative_common_norm" to FilePredictionFeature.numerical(0.0),
+        "relative_distance" to FilePredictionFeature.numerical(0),
+        "relative_distance_norm" to FilePredictionFeature.numerical(0.0)
+      )
+    )
+  }
+
+  fun `test common ancestor for files in neighbor directory`() {
+    doTestSimilarityFeatures(
+      "src/ui/firstFile.txt", "src/components/nextFile.txt",
+      FileFeaturesByProjectPathProducer(
+        "relative_common" to FilePredictionFeature.numerical(1),
+        "relative_common_norm" to FilePredictionFeature.numerical(0.5),
+        "relative_distance" to FilePredictionFeature.numerical(2),
+        "relative_distance_norm" to FilePredictionFeature.numerical(0.5)
+      )
+    )
+  }
+
+  fun `test common ancestor for path with different length`() {
+    doTestSimilarityFeatures(
+      "src/ui/foo/bar/firstFile.txt", "src/components/nextFile.txt",
+      FileFeaturesByProjectPathProducer(
+        "relative_common" to FilePredictionFeature.numerical(1),
+        "relative_common_norm" to FilePredictionFeature.numerical(0.3333),
+        "relative_distance" to FilePredictionFeature.numerical(4),
+        "relative_distance_norm" to FilePredictionFeature.numerical(0.6666)
+      )
+    )
+  }
+
+  fun `test long common ancestor`() {
+    doTestSimilarityFeatures(
+      "src/ui/foo/firstFile.txt", "src/ui/foo/components/nextFile.txt",
+      FileFeaturesByProjectPathProducer(
+        "relative_common" to FilePredictionFeature.numerical(3),
+        "relative_common_norm" to FilePredictionFeature.numerical(0.8571),
+        "relative_distance" to FilePredictionFeature.numerical(1),
+        "relative_distance_norm" to FilePredictionFeature.numerical(0.1428)
       )
     )
   }
@@ -165,7 +315,8 @@ class FilePredictionCommonFeaturesTest : CodeInsightFixtureTestCase<ModuleFixtur
       "prevFile.txt", "nextFile.txt",
       ConstFileFeaturesProducer(
         "same_dir" to FilePredictionFeature.binary(true),
-        "same_module" to FilePredictionFeature.binary(true)
+        "same_module" to FilePredictionFeature.binary(true),
+        "ancestor" to FilePredictionFeature.binary(true)
       )
     )
   }
@@ -175,7 +326,8 @@ class FilePredictionCommonFeaturesTest : CodeInsightFixtureTestCase<ModuleFixtur
       "src/prevFile.txt", "src/nextFile.txt",
       ConstFileFeaturesProducer(
         "same_dir" to FilePredictionFeature.binary(true),
-        "same_module" to FilePredictionFeature.binary(true)
+        "same_module" to FilePredictionFeature.binary(true),
+        "ancestor" to FilePredictionFeature.binary(true)
       )
     )
   }
@@ -185,7 +337,41 @@ class FilePredictionCommonFeaturesTest : CodeInsightFixtureTestCase<ModuleFixtur
       "src/prevFile.txt", "test/nextFile.txt",
       ConstFileFeaturesProducer(
         "same_dir" to FilePredictionFeature.binary(false),
-        "same_module" to FilePredictionFeature.binary(true)
+        "same_module" to FilePredictionFeature.binary(true),
+        "ancestor" to FilePredictionFeature.binary(false)
+      )
+    )
+  }
+
+  fun `test previous files in child directory`() {
+    doTestSimilarityFeatures(
+      "src/sub/prevFile.txt", "src/nextFile.txt",
+      ConstFileFeaturesProducer(
+        "same_dir" to FilePredictionFeature.binary(false),
+        "same_module" to FilePredictionFeature.binary(true),
+        "ancestor" to FilePredictionFeature.binary(true)
+      )
+    )
+  }
+
+  fun `test new files in child directory`() {
+    doTestSimilarityFeatures(
+      "src/prevFile.txt", "src/sub/nextFile.txt",
+      ConstFileFeaturesProducer(
+        "same_dir" to FilePredictionFeature.binary(false),
+        "same_module" to FilePredictionFeature.binary(true),
+        "ancestor" to FilePredictionFeature.binary(true)
+      )
+    )
+  }
+
+  fun `test files in neighbor directories`() {
+    doTestSimilarityFeatures(
+      "src/ui/prevFile.txt", "src/component/nextFile.txt",
+      ConstFileFeaturesProducer(
+        "same_dir" to FilePredictionFeature.binary(false),
+        "same_module" to FilePredictionFeature.binary(true),
+        "ancestor" to FilePredictionFeature.binary(false)
       )
     )
   }

plugins/filePrediction/test/com/intellij/filePrediction/features/FilePredictionFeaturesProducer.kt

@@ -23,6 +23,7 @@ internal class ConstFileFeaturesProducer(vararg included: Pair<String, FilePredi
 }
 
 internal class FileFeaturesByProjectPathProducer(vararg included: Pair<String, FilePredictionFeature>) : FileFeaturesProducer {
+  val featuresToAdjust: Set<String> = hashSetOf("path_prefix")
   val features: Array<out Pair<String, FilePredictionFeature>> = included
 
   override fun produce(project: Project): Map<String, FilePredictionFeature> {
@@ -34,7 +35,7 @@ internal class FileFeaturesByProjectPathProducer(vararg included: Pair<String, F
     val result: MutableMap<String, FilePredictionFeature> = hashMapOf()
     for (feature in features) {
       val value = feature.second.toString().toIntOrNull()
-      if (value != null) {
+      if (featuresToAdjust.contains(feature.first) && value != null) {
         result[feature.first] = FilePredictionFeature.numerical(prefixLength + value)
       }
       else {