LAB-29: Record features for an opened file to use them in predicting the next one

GitOrigin-RevId: ea2a7976c05df51f0b88dd77a619b8012d0df0cb

.idea/modules.xml

@@ -457,6 +457,7 @@
       <module fileurl="file://$PROJECT_DIR$/plugins/eclipse/common-eclipse-util/intellij.eclipse.common.iml" filepath="$PROJECT_DIR$/plugins/eclipse/common-eclipse-util/intellij.eclipse.common.iml" />
       <module fileurl="file://$PROJECT_DIR$/plugins/eclipse/jps-plugin/intellij.eclipse.jps.iml" filepath="$PROJECT_DIR$/plugins/eclipse/jps-plugin/intellij.eclipse.jps.iml" />
       <module fileurl="file://$PROJECT_DIR$/plugins/editorconfig/intellij.editorconfig.iml" filepath="$PROJECT_DIR$/plugins/editorconfig/intellij.editorconfig.iml" />
+      <module fileurl="file://$PROJECT_DIR$/plugins/filePrediction/intellij.filePrediction.iml" filepath="$PROJECT_DIR$/plugins/filePrediction/intellij.filePrediction.iml" />
       <module fileurl="file://$PROJECT_DIR$/plugins/gradle/intellij.gradle.iml" filepath="$PROJECT_DIR$/plugins/gradle/intellij.gradle.iml" />
       <module fileurl="file://$PROJECT_DIR$/plugins/gradle/intellij.gradle.common.iml" filepath="$PROJECT_DIR$/plugins/gradle/intellij.gradle.common.iml" />
       <module fileurl="file://$PROJECT_DIR$/plugins/gradle/java/intellij.gradle.java.iml" filepath="$PROJECT_DIR$/plugins/gradle/java/intellij.gradle.java.iml" />

build/groovy/org/jetbrains/intellij/build/BaseIdeaProperties.groovy

@@ -64,6 +64,7 @@ abstract class BaseIdeaProperties extends JetBrainsProductProperties {
     "intellij.statsCollector",
     "intellij.sh",
     "intellij.vcs.changeReminder",
+    "intellij.filePrediction",
     "intellij.markdown",
     "intellij.laf.macos",
     "intellij.laf.win10"

intellij.idea.community.main.iml

@@ -137,5 +137,6 @@
     <orderEntry type="module" module-name="intellij.gradle.java.maven" scope="RUNTIME" />
     <orderEntry type="module" module-name="intellij.markdown" scope="TEST" />
     <orderEntry type="module" module-name="intellij.statsCollector.tests" scope="TEST" />
+    <orderEntry type="module" module-name="intellij.filePrediction" scope="RUNTIME" />
   </component>
 </module>

platform/statistics/src/com/intellij/internal/statistic/eventLog/FeatureUsageData.kt

@@ -201,6 +201,12 @@ class FeatureUsageData {
   }
 
   @FeatureUsageDataBuilder(additionalDataFields = ["value::0"])
+  fun addAnonymizedValue(@NonNls key: String, @NonNls value: String?): FeatureUsageData {
+    data[key] = value?.let { EventLogConfiguration.anonymize(value) } ?: "undefined"
+    return this
+  }
+
+  @FeatureUsageDataBuilder(additionalDataFields = ["value"])
   fun addValue(value: Any): FeatureUsageData {
     if (value is String || value is Boolean || value is Int || value is Long || value is Float || value is Double) {
       return addDataInternal("value", value)

plugins/filePrediction/intellij.filePrediction.iml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/resources" type="java-resource" />
+      <sourceFolder url="file://$MODULE_DIR$/test" isTestSource="true" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="library" name="kotlin-stdlib-jdk8" level="project" />
+    <orderEntry type="module" module-name="intellij.platform.ide" />
+    <orderEntry type="module" module-name="intellij.platform.ide.impl" />
+    <orderEntry type="module" module-name="intellij.platform.core" />
+    <orderEntry type="module" module-name="intellij.platform.vcs" />
+    <orderEntry type="module" module-name="intellij.platform.vcs.log" />
+    <orderEntry type="module" module-name="intellij.platform.vcs.log.impl" />
+    <orderEntry type="module" module-name="intellij.vcs.changeReminder" />
+    <orderEntry type="module" module-name="intellij.platform.testFramework" scope="TEST" />
+  </component>
+</module>

plugins/filePrediction/resources/META-INF/file-prediction-vcs.xml

@@ -0,0 +1,5 @@
+<idea-plugin>
+  <extensions defaultExtensionNs="com.intellij">
+    <filePrediction.featureProvider implementation="com.intellij.filePrediction.vcs.FilePredictionVcsFeatures"/>
+  </extensions>
+</idea-plugin>

plugins/filePrediction/resources/META-INF/plugin.xml

@@ -0,0 +1,27 @@
+<idea-plugin>
+  <id>com.jetbrains.filePrediction</id>
+  <name>Next File Prediction</name>
+  <vendor>JetBrains</vendor>
+
+  <description><![CDATA[Predicts next file which will be open in IDE to start long running analysis and pre-load caches.]]></description>
+
+  <depends optional="true" config-file="file-prediction-vcs.xml">com.intellij.modules.vcs</depends>
+
+  <extensionPoints>
+    <extensionPoint qualifiedName="com.intellij.filePrediction.featureProvider" interface="com.intellij.filePrediction.FilePredictionFeatureProvider" dynamic="true"/>
+    <extensionPoint qualifiedName="com.intellij.filePrediction.referencesProvider" interface="com.intellij.filePrediction.FileExternalReferencesProvider" dynamic="true"/>
+  </extensionPoints>
+
+  <extensions defaultExtensionNs="com.intellij">
+    <statistics.counterUsagesCollector groupId="file.prediction" version="1"/>
+
+    <filePrediction.featureProvider implementation="com.intellij.filePrediction.FilePredictionGeneralFeatures"/>
+    <filePrediction.featureProvider implementation="com.intellij.filePrediction.history.FilePredictionHistoryFeatures"/>
+
+    <projectService serviceImplementation="com.intellij.filePrediction.history.FilePredictionHistory"/>
+  </extensions>
+
+  <projectListeners>
+    <listener class="com.intellij.filePrediction.FilePredictionEditorManagerListener" topic="com.intellij.openapi.fileEditor.FileEditorManagerListener"/>
+  </projectListeners>
+</idea-plugin>

plugins/filePrediction/src/com/intellij/filePrediction/FilePredictionEditorManagerListener.kt

@@ -0,0 +1,11 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.filePrediction
+
+import com.intellij.openapi.fileEditor.FileEditorManagerEvent
+import com.intellij.openapi.fileEditor.FileEditorManagerListener
+
+class FilePredictionEditorManagerListener : FileEditorManagerListener {
+  override fun selectionChanged(event: FileEditorManagerEvent) {
+    event.newFile?.let { FileUsagePredictor.onFileOpened(event.manager.project, it, event.oldFile) }
+  }
+}

plugins/filePrediction/src/com/intellij/filePrediction/FilePredictionFeature.kt

@@ -0,0 +1,59 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.filePrediction
+
+import com.intellij.internal.statistic.eventLog.FeatureUsageData
+
+sealed class FilePredictionFeature {
+  companion object {
+    @JvmStatic
+    fun binary(value: Boolean): FilePredictionFeature = if (value) BinaryValue.TRUE else BinaryValue.FALSE
+
+    @JvmStatic
+    fun numerical(value: Int): FilePredictionFeature = NumericalValue(value)
+
+    @JvmStatic
+    fun numerical(value: Double): FilePredictionFeature = DoubleValue(value)
+  }
+
+  protected abstract val value: Any
+
+  abstract fun addToEventData(key: String, data: FeatureUsageData)
+
+  override fun equals(other: Any?): Boolean {
+    if (this === other) return true
+    if (javaClass != other?.javaClass) return false
+
+    other as FilePredictionFeature
+
+    if (value != other.value) return false
+
+    return true
+  }
+
+  override fun hashCode(): Int = value.hashCode()
+
+  override fun toString(): String = value.toString()
+
+  private class BinaryValue private constructor(override val value: Boolean) : FilePredictionFeature() {
+    companion object {
+      val TRUE = BinaryValue(true)
+      val FALSE = BinaryValue(false)
+    }
+
+    override fun addToEventData(key: String, data: FeatureUsageData) {
+      data.addData(key, value)
+    }
+  }
+
+  private class NumericalValue(override val value: Int) : FilePredictionFeature() {
+    override fun addToEventData(key: String, data: FeatureUsageData) {
+      data.addData(key, value)
+    }
+  }
+
+  private class DoubleValue(override val value: Double) : FilePredictionFeature() {
+    override fun addToEventData(key: String, data: FeatureUsageData) {
+      data.addData(key, value)
+    }
+  }
+}

plugins/filePrediction/src/com/intellij/filePrediction/FilePredictionFeatureProvider.kt

@@ -0,0 +1,51 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.filePrediction
+
+import com.intellij.openapi.extensions.ExtensionPointName
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.vfs.VirtualFile
+import com.intellij.psi.PsiFile
+import org.jetbrains.annotations.ApiStatus
+
+object FilePredictionFeaturesHelper {
+  private val EP_NAME = ExtensionPointName.create<FilePredictionFeatureProvider>("com.intellij.filePrediction.featureProvider")
+  private val EXTERNAL_REFERENCES_EP_NAME = ExtensionPointName.create<FileExternalReferencesProvider>("com.intellij.filePrediction.referencesProvider")
+
+  fun calculateExternalReferences(file: PsiFile?): Set<PsiFile> {
+    return file?.let { getReferencesProvider(it) } ?: emptySet()
+  }
+
+  fun calculateFileFeatures(project: Project,
+                            newFile: VirtualFile,
+                            prevFile: VirtualFile?): Map<String, FilePredictionFeature> {
+    val result = HashMap<String, FilePredictionFeature>()
+    val providers = getFeatureProviders()
+    providers.forEach { provider ->
+      val prefix = if (provider.getName().isNotEmpty()) provider.getName() + "_" else ""
+      val features = provider.calculateFileFeatures(project, newFile, prevFile).mapKeys { prefix + it.key }
+      result.putAll(features)
+    }
+    return result
+  }
+
+  private fun getReferencesProvider(file: PsiFile): Set<PsiFile> {
+    return EXTERNAL_REFERENCES_EP_NAME.extensions.flatMap { it.externalReferences(file) }.toSet()
+  }
+
+  private fun getFeatureProviders(): List<FilePredictionFeatureProvider> {
+    return EP_NAME.extensionList
+  }
+}
+
+@ApiStatus.Internal
+interface FileExternalReferencesProvider {
+  fun externalReferences(file: PsiFile): Set<PsiFile>
+}
+
+@ApiStatus.Internal
+interface FilePredictionFeatureProvider {
+
+  fun getName(): String
+
+  fun calculateFileFeatures(project: Project, newFile: VirtualFile, prevFile: VirtualFile?): Map<String, FilePredictionFeature>
+}

plugins/filePrediction/src/com/intellij/filePrediction/FilePredictionGeneralFeatures.kt

@@ -0,0 +1,58 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.filePrediction
+
+import com.intellij.filePrediction.FilePredictionFeature.Companion.binary
+import com.intellij.filePrediction.FilePredictionFeature.Companion.numerical
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.project.guessProjectDir
+import com.intellij.openapi.roots.FileIndexFacade
+import com.intellij.openapi.util.SystemInfo
+import com.intellij.openapi.util.io.FileUtil
+import com.intellij.openapi.util.text.StringUtil
+import com.intellij.openapi.util.text.StringUtil.toLowerCase
+import com.intellij.openapi.vfs.VirtualFile
+import com.intellij.util.PathUtil
+import java.io.File
+
+class FilePredictionGeneralFeatures: FilePredictionFeatureProvider {
+  override fun getName(): String = ""
+
+  override fun calculateFileFeatures(project: Project, newFile: VirtualFile, prevFile: VirtualFile?): Map<String, FilePredictionFeature> {
+    val result = HashMap<String, FilePredictionFeature>()
+    val fileIndex = FileIndexFacade.getInstance(project)
+    result["in_project"] = binary(fileIndex.isInProjectScope(newFile))
+    result["in_source"] = binary(fileIndex.isInSource(newFile))
+    result["in_library"] = binary(fileIndex.isInLibraryClasses(newFile) || fileIndex.isInLibrarySource(newFile))
+    result["excluded"] = binary(fileIndex.isExcludedFile(newFile))
+
+    if (prevFile != null) {
+      val newFileName = unify(newFile.name)
+      val newFilePath = unify(newFile.path)
+      val prevFileName = unify(prevFile.name)
+      result["name_prefix"] = numerical(StringUtil.commonPrefixLength(newFileName, prevFileName))
+
+      val prevFilePath = unify(prevFile.path)
+      result["path_prefix"] = numerical(StringUtil.commonPrefixLength(newFilePath, prevFilePath))
+
+      val baseDir = project.guessProjectDir()?.path?.let { unify(it) }
+      if (baseDir != null) {
+        val newRelativePath = FileUtil.getRelativePath(baseDir, newFilePath, File.separatorChar, false)
+        val prevRelativePath = FileUtil.getRelativePath(baseDir, prevFilePath, File.separatorChar, false)
+        if (newRelativePath != null && prevRelativePath != null) {
+          result["relative_path_prefix"] = numerical(StringUtil.commonPrefixLength(newRelativePath, prevRelativePath))
+        }
+      }
+
+      val newModule = fileIndex.getModuleForFile(newFile)
+      val prevModule = fileIndex.getModuleForFile(prevFile)
+      result["same_dir"] = binary(PathUtil.getParentPath(newFilePath) == PathUtil.getParentPath(prevFilePath))
+      result["same_module"] = binary(newModule != null && newModule == prevModule)
+    }
+    return result
+  }
+
+  private fun unify(path: String) : String {
+    val caseSensitive = SystemInfo.isFileSystemCaseSensitive
+    return if (caseSensitive) FileUtil.getNameWithoutExtension(path) else FileUtil.getNameWithoutExtension(toLowerCase(path))
+  }
+}

plugins/filePrediction/src/com/intellij/filePrediction/FileUsagePredictor.kt

@@ -0,0 +1,107 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.filePrediction
+
+import com.intellij.filePrediction.history.FilePredictionHistory
+import com.intellij.internal.statistic.collectors.fus.fileTypes.FileTypeUsagesCollector
+import com.intellij.internal.statistic.eventLog.FeatureUsageData
+import com.intellij.internal.statistic.service.fus.collectors.FUCounterUsageLogger
+import com.intellij.openapi.application.ApplicationManager
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.roots.FileIndexFacade
+import com.intellij.openapi.util.Computable
+import com.intellij.openapi.vfs.VirtualFile
+import com.intellij.psi.PsiManager
+import com.intellij.util.concurrency.NonUrgentExecutor
+
+object FileUsagePredictor {
+  private const val CALCULATE_CANDIDATE_PROBABILITY: Double = 0.2
+  private const val MAX_CANDIDATE: Int = 10
+
+  fun onFileOpened(project: Project, newFile: VirtualFile, prevFile: VirtualFile?) {
+    NonUrgentExecutor.getInstance().execute {
+      val refs = calculateExternalReferences(project, prevFile)
+
+      FileNavigationLogger.logEvent(project, newFile, prevFile, "file.opened", refs.contains(newFile))
+      if (Math.random() < CALCULATE_CANDIDATE_PROBABILITY) {
+        prevFile?.let {
+          calculateCandidates(project, it, newFile, refs)
+        }
+      }
+
+      FilePredictionHistory.getInstance(project).onFileOpened(newFile.url)
+    }
+  }
+
+  private fun calculateExternalReferences(project: Project, prevFile: VirtualFile?): Set<VirtualFile> {
+    return ApplicationManager.getApplication().runReadAction(Computable<Set<VirtualFile>> {
+      val prevPsiFile = prevFile?.let { PsiManager.getInstance(project).findFile(it) }
+      FilePredictionFeaturesHelper.calculateExternalReferences(prevPsiFile).mapNotNull { file -> file.virtualFile }.toSet()
+    })
+  }
+
+  private fun calculateCandidates(project: Project,
+                                  prevFile: VirtualFile,
+                                  openedFile: VirtualFile,
+                                  refs: Set<VirtualFile>) {
+    val candidates = selectFileCandidates(project, prevFile, refs)
+    for (candidate in candidates) {
+      if (candidate != openedFile) {
+        FileNavigationLogger.logEvent(project, candidate, prevFile, "candidate.calculated", refs.contains(candidate))
+      }
+    }
+  }
+
+  private fun selectFileCandidates(project: Project, currentFile: VirtualFile, refs: Set<VirtualFile>): List<VirtualFile> {
+    val result = ArrayList<VirtualFile>()
+    addWithLimit(refs.iterator(), result, currentFile, MAX_CANDIDATE / 2)
+
+    val fileIndex = FileIndexFacade.getInstance(project)
+    var parent = currentFile.parent
+    while (parent != null && parent.isDirectory && result.size < MAX_CANDIDATE && fileIndex.isInProjectScope(parent)) {
+      addWithLimit(parent.children.iterator(), result, currentFile, MAX_CANDIDATE)
+      parent = parent.parent
+    }
+    return result
+  }
+
+  private fun addWithLimit(from: Iterator<VirtualFile>, to: MutableList<VirtualFile>, skip: VirtualFile, limit: Int) {
+    while (to.size < limit && from.hasNext()) {
+      val next = from.next()
+      if (!next.isDirectory && skip != next) {
+        to.add(next)
+      }
+    }
+  }
+}
+
+private object FileNavigationLogger {
+  private const val GROUP_ID = "file.prediction"
+
+  fun logEvent(project: Project, newFile: VirtualFile, prevFile: VirtualFile?, event: String, isInRef: Boolean) {
+    val data = FileTypeUsagesCollector.newFeatureUsageData(newFile.fileType).
+      addNewFileInfo(newFile, isInRef).
+      addPrevFileInfo(prevFile).
+      addFileFeatures(project, newFile, prevFile)
+
+    FUCounterUsageLogger.getInstance().logEvent(project, GROUP_ID, event, data)
+  }
+
+  private fun FeatureUsageData.addNewFileInfo(newFile: VirtualFile, isInRef: Boolean): FeatureUsageData {
+    return addAnonymizedPath(newFile.path).addData("in_ref", isInRef)
+  }
+
+  private fun FeatureUsageData.addPrevFileInfo(prevFile: VirtualFile?): FeatureUsageData {
+    return addData("prev_file_type", prevFile?.fileType?.name ?: "undefined").addAnonymizedValue("prev_file_path", prevFile?.path)
+  }
+
+  private fun FeatureUsageData.addFileFeatures(project: Project,
+                                               newFile: VirtualFile,
+                                               prevFile: VirtualFile?): FeatureUsageData {
+    val features = FilePredictionFeaturesHelper.calculateFileFeatures(project, newFile, prevFile)
+    for (feature in features) {
+      feature.value.addToEventData(feature.key, this)
+    }
+    return this
+  }
+}
+

plugins/filePrediction/src/com/intellij/filePrediction/history/FilePredictionHistory.kt

@@ -0,0 +1,34 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.filePrediction.history
+
+import com.intellij.openapi.components.*
+import com.intellij.openapi.project.Project
+
+@State(name = "FilePredictionHistory", storages = [Storage(StoragePathMacros.CACHE_FILE)])
+class FilePredictionHistory: PersistentStateComponent<FilePredictionHistoryState> {
+  private var state = FilePredictionHistoryState()
+
+  companion object {
+    private const val RECENT_FILES_LIMIT = 50
+
+    fun getInstance(project: Project): FilePredictionHistory {
+      return ServiceManager.getService(project, FilePredictionHistory::class.java)
+    }
+  }
+
+  fun onFileOpened(fileUrl: String) = state.onFileOpened(fileUrl, RECENT_FILES_LIMIT)
+
+  fun position(fileUrl: String): Int = state.position(fileUrl)
+
+  fun size(): Int = state.size()
+
+  fun cleanup() = state.cleanup()
+
+  override fun getState(): FilePredictionHistoryState? {
+    return state
+  }
+
+  override fun loadState(newState: FilePredictionHistoryState) {
+    state = newState
+  }
+}

plugins/filePrediction/src/com/intellij/filePrediction/history/FilePredictionHistoryFeatures.kt

@@ -0,0 +1,20 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.filePrediction.history
+
+import com.intellij.filePrediction.FilePredictionFeature
+import com.intellij.filePrediction.FilePredictionFeature.Companion.numerical
+import com.intellij.filePrediction.FilePredictionFeatureProvider
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.vfs.VirtualFile
+
+class FilePredictionHistoryFeatures: FilePredictionFeatureProvider {
+  override fun getName(): String = "history"
+
+  override fun calculateFileFeatures(project: Project, newFile: VirtualFile, prevFile: VirtualFile?): Map<String, FilePredictionFeature> {
+    val result = HashMap<String, FilePredictionFeature>()
+    val history = FilePredictionHistory.getInstance(project)
+    result["position"] = numerical(history.position(newFile.url))
+    result["size"] = numerical(history.size())
+    return result
+  }
+}

plugins/filePrediction/src/com/intellij/filePrediction/history/FilePredictionHistoryState.kt

@@ -0,0 +1,35 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.filePrediction.history
+
+import com.intellij.openapi.components.*
+import com.intellij.util.xmlb.annotations.OptionTag
+
+class FilePredictionHistoryState: BaseState() {
+  @get:OptionTag
+  val recentFiles by list<String>()
+
+  @Synchronized
+  fun onFileOpened(fileUrl: String, limit: Int) {
+    recentFiles.remove(fileUrl)
+    recentFiles.add(fileUrl)
+
+    while (recentFiles.size > limit) {
+      recentFiles.removeAt(0)
+    }
+  }
+
+  @Synchronized
+  fun position(fileUrl: String): Int {
+    var i = recentFiles.size - 1
+    while (i >= 0 && recentFiles[i] != fileUrl) {
+      i--
+    }
+    return if (i < 0) -1 else recentFiles.size - 1 - i
+  }
+
+  @Synchronized
+  fun size(): Int = recentFiles.size
+
+  @Synchronized
+  fun cleanup() = recentFiles.clear()
+}

plugins/filePrediction/src/com/intellij/filePrediction/vcs/FilePredictionVcsFeatures.kt

@@ -0,0 +1,23 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.filePrediction.vcs
+
+import com.intellij.filePrediction.FilePredictionFeature
+import com.intellij.filePrediction.FilePredictionFeatureProvider
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.vcs.ProjectLevelVcsManager
+import com.intellij.openapi.vcs.changes.ChangeListManager
+import com.intellij.openapi.vfs.VirtualFile
+
+class FilePredictionVcsFeatures : FilePredictionFeatureProvider {
+  override fun getName(): String = "vcs"
+
+  override fun calculateFileFeatures(project: Project, newFile: VirtualFile, prevFile: VirtualFile?): Map<String, FilePredictionFeature> {
+    if (!ProjectLevelVcsManager.getInstance(project).hasAnyMappings()) return emptyMap()
+
+    val result = HashMap<String, FilePredictionFeature>()
+    val changeListManager = ChangeListManager.getInstance(project)
+    result["prev_in_changelist"] = FilePredictionFeature.binary(prevFile != null && changeListManager.isFileAffected(prevFile))
+    result["in_changelist"] = FilePredictionFeature.binary(changeListManager.isFileAffected(newFile))
+    return result
+  }
+}

plugins/filePrediction/test/com/intellij/filePrediction/FilePredictionFeaturesTest.kt

@@ -0,0 +1,396 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.filePrediction
+
+import com.intellij.openapi.application.ApplicationManager
+import com.intellij.openapi.module.Module
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.project.guessProjectDir
+import com.intellij.openapi.roots.ModuleRootManager
+import com.intellij.openapi.roots.OrderRootType
+import com.intellij.openapi.util.io.FileUtil
+import com.intellij.openapi.vfs.VirtualFileManager
+import com.intellij.testFramework.builders.ModuleFixtureBuilder
+import com.intellij.testFramework.fixtures.CodeInsightFixtureTestCase
+import com.intellij.testFramework.fixtures.ModuleFixture
+import com.intellij.util.io.URLUtil
+import org.jetbrains.jps.model.java.JavaSourceRootType
+import org.jetbrains.jps.model.java.JpsJavaExtensionService
+import org.junit.Test
+import java.io.File
+
+class FilePredictionFeaturesTest : CodeInsightFixtureTestCase<ModuleFixtureBuilder<ModuleFixture>>() {
+
+  private fun doTestGeneralFeatures(prevPath: String, newPath: String, featuresProvider: FileFeaturesProducer) {
+    val prevFile = myFixture.addFileToProject(prevPath, "PREVIOUS FILE")
+    val nextFile = myFixture.addFileToProject(newPath, "NEXT FILE")
+
+    val provider = FilePredictionGeneralFeatures()
+    val actual = provider.calculateFileFeatures(myFixture.project, nextFile.virtualFile, prevFile.virtualFile)
+    val expected = featuresProvider.produce(myFixture.project)
+    for (feature in expected.entries) {
+      assertTrue("Cannot find feature '${feature.key}' in $actual", actual.containsKey(feature.key))
+      assertEquals("The value of feature '${feature.key}' is different from expected", feature.value, actual[feature.key])
+    }
+  }
+
+  private fun doTestGeneralFeatures(newPath: String, configurator: ProjectConfigurator, featuresProvider: FileFeaturesProducer) {
+    val nextFile = myFixture.addFileToProject(newPath, "NEXT FILE")
+    configurator.configure(myFixture.project, myModule)
+
+    val provider = FilePredictionGeneralFeatures()
+    val actual = provider.calculateFileFeatures(myFixture.project, nextFile.virtualFile, null)
+    val expected = featuresProvider.produce(myFixture.project)
+    for (feature in expected.entries) {
+      assertTrue("Cannot find feature '${feature.key}' in $actual", actual.containsKey(feature.key))
+      assertEquals("The value of feature '${feature.key}' is different from expected", feature.value, actual[feature.key])
+    }
+  }
+
+  @Test
+  fun `test file name prefix for completely different files`() {
+    doTestGeneralFeatures(
+      "prevFile.txt", "nextFile.txt",
+      ConstFileFeaturesProducer(
+        "name_prefix" to FilePredictionFeature.numerical(0),
+        "relative_path_prefix" to FilePredictionFeature.numerical(0)
+      )
+    )
+  }
+
+  @Test
+  fun `test file name prefix for files with common prefix`() {
+    doTestGeneralFeatures(
+      "myPrevFile.txt", "myNextFile.txt",
+      ConstFileFeaturesProducer(
+        "name_prefix" to FilePredictionFeature.numerical(2),
+        "relative_path_prefix" to FilePredictionFeature.numerical(2)
+      )
+    )
+  }
+
+  @Test
+  fun `test file name prefix for equal files`() {
+    doTestGeneralFeatures(
+      "file.txt", "src/file.txt",
+      ConstFileFeaturesProducer(
+        "name_prefix" to FilePredictionFeature.numerical(4),
+        "relative_path_prefix" to FilePredictionFeature.numerical(0)
+      )
+    )
+  }
+
+  @Test
+  fun `test file name of different length`() {
+    doTestGeneralFeatures(
+      "someFile.txt", "src/file.txt",
+      ConstFileFeaturesProducer(
+        "name_prefix" to FilePredictionFeature.numerical(0),
+        "relative_path_prefix" to FilePredictionFeature.numerical(1)
+      )
+    )
+  }
+
+  @Test
+  fun `test file name in child directory`() {
+    doTestGeneralFeatures(
+      "src/someFile.txt", "src/file.txt",
+      ConstFileFeaturesProducer(
+        "name_prefix" to FilePredictionFeature.numerical(0),
+        "relative_path_prefix" to FilePredictionFeature.numerical(4)
+      )
+    )
+  }
+
+  @Test
+  fun `test file name in neighbour directories`() {
+    doTestGeneralFeatures(
+      "src/com/site/ui/someFile.txt", "src/com/site/component/file.txt",
+      ConstFileFeaturesProducer(
+        "name_prefix" to FilePredictionFeature.numerical(0),
+        "relative_path_prefix" to FilePredictionFeature.numerical(13)
+      )
+    )
+  }
+
+  @Test
+  fun `test files path in project root`() {
+    doTestGeneralFeatures(
+      "prevFile.txt", "nextFile.txt",
+      FileFeaturesByProjectPathProducer(
+        "path_prefix" to FilePredictionFeature.numerical(0)
+      )
+    )
+  }
+
+  @Test
+  fun `test files path in the same directory`() {
+    doTestGeneralFeatures(
+      "src/prevFile.txt", "src/nextFile.txt",
+      FileFeaturesByProjectPathProducer(
+        "path_prefix" to FilePredictionFeature.numerical(4)
+      )
+    )
+  }
+
+  @Test
+  fun `test files path in the neighbour directories`() {
+    doTestGeneralFeatures(
+      "src/ui/prevFile.txt", "src/components/nextFile.txt",
+      FileFeaturesByProjectPathProducer(
+        "path_prefix" to FilePredictionFeature.numerical(4)
+      )
+    )
+  }
+
+  @Test
+  fun `test files path of different length`() {
+    doTestGeneralFeatures(
+      "firstFile.txt", "another/nextFile.txt",
+      FileFeaturesByProjectPathProducer(
+        "path_prefix" to FilePredictionFeature.numerical(0)
+      )
+    )
+  }
+
+  @Test
+  fun `test files in the root directory`() {
+    doTestGeneralFeatures(
+      "prevFile.txt", "nextFile.txt",
+      ConstFileFeaturesProducer(
+        "same_dir" to FilePredictionFeature.binary(true),
+        "same_module" to FilePredictionFeature.binary(true)
+      )
+    )
+  }
+
+  @Test
+  fun `test files in same child directory`() {
+    doTestGeneralFeatures(
+      "src/prevFile.txt", "src/nextFile.txt",
+      ConstFileFeaturesProducer(
+        "same_dir" to FilePredictionFeature.binary(true),
+        "same_module" to FilePredictionFeature.binary(true)
+      )
+    )
+  }
+
+  @Test
+  fun `test files in different directories`() {
+    doTestGeneralFeatures(
+      "src/prevFile.txt", "test/nextFile.txt",
+      ConstFileFeaturesProducer(
+        "same_dir" to FilePredictionFeature.binary(false),
+        "same_module" to FilePredictionFeature.binary(true)
+      )
+    )
+  }
+
+  @Test
+  fun `test file not in a source root`() {
+    doTestGeneralFeatures(
+      "file.txt",
+      object : ProjectConfigurator {
+        override fun configure(project: Project, module: Module) {
+          TestProjectStructureConfigurator.removeSourceRoot(module)
+        }
+      },
+      ConstFileFeaturesProducer(
+        "in_project" to FilePredictionFeature.binary(true),
+        "in_source" to FilePredictionFeature.binary(false),
+        "in_library" to FilePredictionFeature.binary(false),
+        "excluded" to FilePredictionFeature.binary(false)
+      )
+    )
+  }
+
+  @Test
+  fun `test file in source root`() {
+    doTestGeneralFeatures(
+      "nextFile.txt",
+      EmptyProjectConfigurator,
+      ConstFileFeaturesProducer(
+        "in_project" to FilePredictionFeature.binary(true),
+        "in_source" to FilePredictionFeature.binary(true),
+        "in_library" to FilePredictionFeature.binary(false),
+        "excluded" to FilePredictionFeature.binary(false)
+      )
+    )
+  }
+
+  @Test
+  fun `test file not in a custom source root`() {
+    doTestGeneralFeatures(
+      "nextFile.txt",
+      object : ProjectConfigurator {
+        override fun configure(project: Project, module: Module) {
+          TestProjectStructureConfigurator.removeSourceRoot(module)
+          TestProjectStructureConfigurator.addSourceRoot(module, "src", false)
+        }
+      },
+      ConstFileFeaturesProducer(
+        "in_project" to FilePredictionFeature.binary(true),
+        "in_source" to FilePredictionFeature.binary(false),
+        "in_library" to FilePredictionFeature.binary(false),
+        "excluded" to FilePredictionFeature.binary(false)
+      )
+    )
+  }
+
+  @Test
+  fun `test file in a custom source root`() {
+    doTestGeneralFeatures(
+      "src/nextFile.txt",
+      object : ProjectConfigurator {
+        override fun configure(project: Project, module: Module) {
+          TestProjectStructureConfigurator.removeSourceRoot(module)
+          TestProjectStructureConfigurator.addSourceRoot(module, "src", false)
+        }
+      },
+      ConstFileFeaturesProducer(
+        "in_project" to FilePredictionFeature.binary(true),
+        "in_source" to FilePredictionFeature.binary(true),
+        "in_library" to FilePredictionFeature.binary(false),
+        "excluded" to FilePredictionFeature.binary(false)
+      )
+    )
+  }
+
+  @Test
+  fun `test file in a library source root`() {
+    doTestGeneralFeatures(
+      "lib/nextFile.txt",
+      object : ProjectConfigurator {
+        override fun configure(project: Project, module: Module) {
+          TestProjectStructureConfigurator.removeSourceRoot(module)
+          TestProjectStructureConfigurator.addLibrary(module, "lib", OrderRootType.SOURCES)
+        }
+      },
+      ConstFileFeaturesProducer(
+        "in_project" to FilePredictionFeature.binary(true),
+        "in_source" to FilePredictionFeature.binary(true),
+        "in_library" to FilePredictionFeature.binary(true),
+        "excluded" to FilePredictionFeature.binary(false)
+      )
+    )
+  }
+
+  @Test
+  fun `test file in library classes`() {
+    doTestGeneralFeatures(
+      "lib/nextFile.txt",
+      object : ProjectConfigurator {
+        override fun configure(project: Project, module: Module) {
+          TestProjectStructureConfigurator.removeSourceRoot(module)
+          TestProjectStructureConfigurator.addLibrary(module, "lib", OrderRootType.CLASSES)
+        }
+      },
+      ConstFileFeaturesProducer(
+        "in_project" to FilePredictionFeature.binary(false),
+        "in_source" to FilePredictionFeature.binary(false),
+        "in_library" to FilePredictionFeature.binary(true),
+        "excluded" to FilePredictionFeature.binary(false)
+      )
+    )
+  }
+}
+
+private class ConstFileFeaturesProducer(vararg included: Pair<String, FilePredictionFeature>) : FileFeaturesProducer {
+  val features: MutableMap<String, FilePredictionFeature> = hashMapOf()
+
+  init {
+    for (pair in included) {
+      features[pair.first] = pair.second
+    }
+  }
+
+  override fun produce(project: Project): Map<String, FilePredictionFeature> {
+    return features
+  }
+}
+
+private class FileFeaturesByProjectPathProducer(vararg included: Pair<String, FilePredictionFeature>) : FileFeaturesProducer {
+  val features: Array<out Pair<String, FilePredictionFeature>> = included
+
+  override fun produce(project: Project): Map<String, FilePredictionFeature> {
+    val dir = project.guessProjectDir()?.path
+    CodeInsightFixtureTestCase.assertNotNull(dir)
+
+    val prefixLength = dir!!.length + 1
+
+    val result: MutableMap<String, FilePredictionFeature> = hashMapOf()
+    for (feature in features) {
+      val value = feature.second.toString().toIntOrNull()
+      if (value != null) {
+        result[feature.first] = FilePredictionFeature.numerical(prefixLength + value)
+      }
+      else {
+        result[feature.first] = feature.second
+      }
+    }
+    return result
+  }
+}
+
+private interface FileFeaturesProducer {
+  fun produce(project: Project): Map<String, FilePredictionFeature>
+}
+
+private object EmptyProjectConfigurator : ProjectConfigurator {
+  override fun configure(project: Project, module: Module) = Unit
+}
+
+private interface ProjectConfigurator {
+  fun configure(project: Project, module: Module)
+}
+
+private object TestProjectStructureConfigurator {
+  fun addSourceRoot(module: Module, path: String, isTest: Boolean) {
+    val project = module.project
+    val dir = project.guessProjectDir()?.path
+    CodeInsightFixtureTestCase.assertNotNull(dir)
+
+    val fullPath = "${dir}${File.separator}$path"
+
+    ApplicationManager.getApplication().runWriteAction {
+      val model = ModuleRootManager.getInstance(module).modifiableModel
+      val contentEntry = model.contentEntries.find {
+        it.file?.let { file -> FileUtil.isAncestor(file.path, fullPath, false) } ?: false
+      }
+
+      val rootType = if (isTest) JavaSourceRootType.TEST_SOURCE else JavaSourceRootType.SOURCE
+      val properties = JpsJavaExtensionService.getInstance().createSourceRootProperties("")
+      val url = VirtualFileManager.constructUrl(URLUtil.FILE_PROTOCOL, fullPath)
+      contentEntry?.addSourceFolder(url, rootType, properties)
+
+      model.commit()
+    }
+  }
+
+  fun addLibrary(module: Module, path: String, type: OrderRootType) {
+    val project = module.project
+    val dir = project.guessProjectDir()?.path
+    CodeInsightFixtureTestCase.assertNotNull(dir)
+    val fullPath = "${dir}${File.separator}$path"
+
+    ApplicationManager.getApplication().runWriteAction {
+      val model = ModuleRootManager.getInstance(module).modifiableModel
+      val libraryModel = model.moduleLibraryTable.modifiableModel
+
+      val modifiableModel = libraryModel.createLibrary("test_library").modifiableModel
+      val url = VirtualFileManager.constructUrl(URLUtil.FILE_PROTOCOL, fullPath)
+      modifiableModel.addRoot(url, type)
+
+      modifiableModel.commit()
+      model.commit()
+    }
+  }
+
+  fun removeSourceRoot(module: Module) {
+    ApplicationManager.getApplication().runWriteAction {
+      val model = ModuleRootManager.getInstance(module).modifiableModel
+      val contentEntry = model.contentEntries[0]
+      contentEntry.removeSourceFolder(contentEntry.sourceFolders[0])
+      model.commit()
+    }
+  }
+}

plugins/filePrediction/test/com/intellij/filePrediction/FilePredictionHistoryFeaturesTest.kt

@@ -0,0 +1,118 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.filePrediction
+
+import com.intellij.filePrediction.history.FilePredictionHistoryState
+import com.intellij.testFramework.builders.ModuleFixtureBuilder
+import com.intellij.testFramework.fixtures.CodeInsightFixtureTestCase
+import com.intellij.testFramework.fixtures.ModuleFixture
+
+class FilePredictionHistoryFeaturesTest : CodeInsightFixtureTestCase<ModuleFixtureBuilder<ModuleFixture>>() {
+
+  private fun doTest(openedFiles: List<String>, size: Int, vararg expected: Pair<String, Int>) {
+    val history = FilePredictionHistoryState()
+    try {
+      for (file in openedFiles) {
+        history.onFileOpened(file, 5)
+      }
+
+      assertEquals(size, history.size())
+
+      for (pair in expected) {
+        assertEquals(pair.second, history.position(pair.first))
+      }
+    }
+    finally {
+      history.cleanup()
+    }
+  }
+
+  fun `test position of the file without history`() {
+    doTest(listOf(), 0, "file://a" to -1)
+  }
+
+  fun `test position of the new file`() {
+    doTest(listOf("file://a"), 1, "file://b" to -1)
+  }
+
+  fun `test position of the prev file`() {
+    doTest(listOf("file://a"), 1, "file://a" to 0)
+  }
+
+  fun `test position of the first file`() {
+    doTest(listOf("file://a", "file://b", "file://c"), 3, "file://a" to 2)
+  }
+
+  fun `test position of the middle file`() {
+    doTest(listOf("file://a", "file://b", "file://c"), 3, "file://b" to 1)
+  }
+
+  fun `test position of the latest file`() {
+    doTest(listOf("file://a", "file://b", "file://c"), 3, "file://c" to 0)
+  }
+
+  fun `test position of the file opened multiple times`() {
+    doTest(listOf("file://a", "file://b", "file://a", "file://c"), 3, "file://a" to 1)
+  }
+
+  fun `test position of the latest file opened multiple times`() {
+    doTest(listOf("file://a", "file://b", "file://a", "file://c", "file://a"), 3, "file://a" to 0)
+  }
+
+  fun `test position of the middle file opened multiple times`() {
+    doTest(listOf("file://a", "file://b", "file://a", "file://b", "file://a"), 2, "file://b" to 1)
+  }
+
+  fun `test recent files history is full`() {
+    doTest(
+      listOf("file://a", "file://b", "file://c", "file://d", "file://e"), 5,
+      "file://a" to 4,
+      "file://b" to 3,
+      "file://c" to 2,
+      "file://d" to 1,
+      "file://e" to 0
+    )
+  }
+
+  fun `test recent files history more than limit`() {
+    doTest(
+      listOf("file://a", "file://b", "file://c", "file://d", "file://e", "file://f"), 5,
+      "file://b" to 4,
+      "file://c" to 3,
+      "file://d" to 2,
+      "file://e" to 1,
+      "file://f" to 0
+    )
+  }
+
+  fun `test recent files history twice more than limit`() {
+    doTest(
+      listOf("file://a", "file://b", "file://c", "file://d", "file://e", "file://f", "file://g", "file://h", "file://i"), 5,
+      "file://e" to 4,
+      "file://f" to 3,
+      "file://g" to 2,
+      "file://h" to 1,
+      "file://i" to 0
+    )
+  }
+
+  fun `test recent files history with repetitions`() {
+    doTest(
+      listOf("file://a", "file://b", "file://a", "file://d", "file://a", "file://b", "file://c", "file://a", "file://d"), 4,
+      "file://b" to 3,
+      "file://c" to 2,
+      "file://a" to 1,
+      "file://d" to 0
+    )
+  }
+
+  fun `test recent files history with repetitions more than limit`() {
+    doTest(
+      listOf("file://a", "file://b", "file://a", "file://d", "file://a", "file://e", "file://c", "file://a", "file://d", "file://f"), 5,
+      "file://e" to 4,
+      "file://c" to 3,
+      "file://a" to 2,
+      "file://d" to 1,
+      "file://f" to 0
+    )
+  }
+}