WEB-25900 Webstorm Dart "pub serve" proxy is slow

java/compiler/impl/src/org/jetbrains/builtInWebServer/ArtifactWebServerRootsProvider.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2015 JetBrains s.r.o.
+ * Copyright 2000-2017 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,10 +21,14 @@ import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.packaging.artifacts.ArtifactManager
 
 internal class ArtifactWebServerRootsProvider : PrefixlessWebServerRootsProvider() {
-  override fun resolve(path: String, project: Project, resolver: FileResolver): PathInfo? {
+  override fun resolve(path: String, project: Project, resolver: FileResolver, pathQuery: PathQuery): PathInfo? {
+    if (!pathQuery.searchInArtifacts) {
+      return null
+    }
+    
     for (artifact in ArtifactManager.getInstance(project).artifacts) {
       val root = artifact.outputFile ?: continue
-      return resolver.resolve(path, root)
+      return resolver.resolve(path, root, pathQuery = pathQuery)
     }
     return null
   }

platform/built-in-server-api/src/org/jetbrains/builtInWebServer/PathInfo.kt

@@ -3,13 +3,19 @@ package org.jetbrains.builtInWebServer
 import com.intellij.openapi.fileTypes.FileType
 import com.intellij.openapi.fileTypes.FileTypeManager
 import com.intellij.openapi.util.io.FileUtilRt
+import com.intellij.openapi.vfs.LocalFileSystem
 import com.intellij.openapi.vfs.VfsUtilCore
 import com.intellij.openapi.vfs.VirtualFile
+import com.intellij.util.io.isDirectory
+import com.intellij.util.io.systemIndependentPath
 import java.io.File
 import java.nio.file.Files
 import java.nio.file.Path
 
-class PathInfo(val ioFile: Path?, val file: VirtualFile?, val root: VirtualFile, moduleName: String? = null, val isLibrary: Boolean = false, val isRootNameOptionalInPath: Boolean = false) {
+class PathInfo(val ioFile: Path?, file: VirtualFile?, val root: VirtualFile, moduleName: String? = null, val isLibrary: Boolean = false, val isRootNameOptionalInPath: Boolean = false) {
+  var file = file
+    private set
+  
   var moduleName: String? = moduleName
     set
 
@@ -39,25 +45,36 @@ class PathInfo(val ioFile: Path?, val file: VirtualFile?, val root: VirtualFile,
       builder.append(FileUtilRt.getRelativePath(relativeTo.path, ioFile!!.toString().replace(File.separatorChar, '/'), '/'))
     }
     else {
-      builder.append(VfsUtilCore.getRelativePath(file, relativeTo, '/'))
+      builder.append(VfsUtilCore.getRelativePath(file!!, relativeTo, '/'))
     }
     return builder.toString()
   }
+  
+  fun getOrResolveVirtualFile(): VirtualFile? {
+    return if (file == null) {
+      val result = LocalFileSystem.getInstance().findFileByPath(ioFile!!.systemIndependentPath)
+      file = result
+      result
+    }
+    else {
+      file
+    }
+  }
 
   /**
    * System-dependent path to file.
    */
-  val filePath: String by lazy { if (ioFile == null) FileUtilRt.toSystemDependentName(file!!.path) else ioFile.toString() }
+  val filePath: String by lazy { ioFile?.toString() ?: FileUtilRt.toSystemDependentName(file!!.path) }
 
   val isValid: Boolean
     get() = if (ioFile == null) file!!.isValid else Files.exists(ioFile)
 
   val name: String
-    get() = if (ioFile == null) file!!.name else ioFile.fileName.toString()
+    get() = ioFile?.fileName?.toString() ?: file!!.name
 
   val fileType: FileType
     get() = if (ioFile == null) file!!.fileType else FileTypeManager.getInstance().getFileTypeByFileName(ioFile.fileName.toString())
 
 
-  fun isDirectory(): Boolean = if (ioFile == null) file!!.isDirectory else Files.isDirectory(ioFile)
+  fun isDirectory(): Boolean = ioFile?.isDirectory() ?: file!!.isDirectory
 }

platform/built-in-server/src/org/jetbrains/builtInWebServer/BuiltInWebServer.kt

@@ -238,7 +238,7 @@ private fun doProcess(urlDecoder: QueryStringDecoder, request: FullHttpRequest,
   return false
 }
 
-internal fun HttpRequest.isSignedRequest(): Boolean {
+fun HttpRequest.isSignedRequest(): Boolean {
   if (BuiltInServerOptions.getInstance().allowUnsignedRequests) {
     return true
   }
@@ -252,7 +252,7 @@ internal fun HttpRequest.isSignedRequest(): Boolean {
   return token != null && tokens.getIfPresent(token) != null
 }
 
-internal fun validateToken(request: HttpRequest, channel: Channel, isSignedRequest: Boolean): HttpHeaders? {
+fun validateToken(request: HttpRequest, channel: Channel, isSignedRequest: Boolean): HttpHeaders? {
   if (BuiltInServerOptions.getInstance().allowUnsignedRequests) {
     return EmptyHttpHeaders.INSTANCE
   }

platform/built-in-server/src/org/jetbrains/builtInWebServer/DefaultWebServerPathHandler.kt

@@ -55,7 +55,7 @@ private class DefaultWebServerPathHandler : WebServerPathHandler() {
     val pathToFileManager = WebServerPathToFileManager.getInstance(project)
     var pathInfo = pathToFileManager.pathToInfoCache.getIfPresent(path)
     if (pathInfo == null || !pathInfo.isValid) {
-      pathInfo = pathToFileManager.doFindByRelativePath(path)
+      pathInfo = pathToFileManager.doFindByRelativePath(path, defaultPathQuery)
       if (pathInfo == null) {
         HttpResponseStatus.NOT_FOUND.send(channel, request, extraHeaders = extraHeaders)
         return true

platform/built-in-server/src/org/jetbrains/builtInWebServer/DefaultWebServerRootsProvider.kt

@@ -35,8 +35,12 @@ import com.intellij.project.rootManager
 import com.intellij.util.PlatformUtils
 import com.intellij.util.containers.computeOrNull
 
+internal data class SuitableRoot(val file: VirtualFile, val moduleQualifier: String?) 
+
 private class DefaultWebServerRootsProvider : WebServerRootsProvider() {
-  override fun resolve(path: String, project: Project): PathInfo? {
+  override fun resolve(path: String, project: Project, pathQuery: PathQuery): PathInfo? {
+    val pathToFileManager = WebServerPathToFileManager.getInstance(project)
+    
     var effectivePath = path
     if (PlatformUtils.isIntelliJ()) {
       val index = effectivePath.indexOf('/')
@@ -45,9 +49,9 @@ private class DefaultWebServerRootsProvider : WebServerRootsProvider() {
         val module = runReadAction { ModuleManager.getInstance(project).findModuleByName(moduleName) }
         if (module != null && !module.isDisposed) {
           effectivePath = effectivePath.substring(index + 1)
-          val resolver = WebServerPathToFileManager.getInstance(project).getResolver(effectivePath)
-          val result = RootProvider.values().computeOrNull { findByRelativePath(effectivePath, it.getRoots(module.rootManager), resolver, moduleName) }
-            ?: findInModuleLibraries(effectivePath, module, resolver)
+          val resolver = pathToFileManager.getResolver(effectivePath)
+          val result = RootProvider.values().computeOrNull { findByRelativePath(effectivePath, it.getRoots(module.rootManager), resolver, moduleName, pathQuery) }
+            ?: findInModuleLibraries(effectivePath, module, resolver, pathQuery)
           if (result != null) {
             return result
           }
@@ -55,42 +59,76 @@ private class DefaultWebServerRootsProvider : WebServerRootsProvider() {
       }
     }
 
-    val resolver = WebServerPathToFileManager.getInstance(project).getResolver(effectivePath)
+    val resolver = pathToFileManager.getResolver(effectivePath)
     val modules = runReadAction { ModuleManager.getInstance(project).modules }
-    for (rootProvider in RootProvider.values()) {
-      for (module in modules) {
-        if (module.isDisposed) {
-          continue
-        }
-
-        findByRelativePath(path, rootProvider.getRoots(module.rootManager), resolver, null)?.let {
-          it.moduleName = getModuleNameQualifier(project, module)
-          return it
-        }
+    if (pathQuery.useVfs) {
+      var oldestParent = path.indexOf("/").let { if (it > 0) path.substring(0, it) else null }
+      if (oldestParent == null && !path.isEmpty() && !path.contains('.')) {
+        // maybe it is top level directory? (in case of dart projects - web)
+        oldestParent = path
       }
-    }
-
-    // https://youtrack.jetbrains.com/issue/WEB-24283
-    for (rootProvider in RootProvider.values()) {
-      for (module in modules) {
-        if (module.isDisposed) {
-          continue
-        }
-
-        for (root in rootProvider.getRoots(module.rootManager)) {
-          if (resolver.resolve("/config.json", root) != null) {
-            resolver.resolve("/index.html", root)?.let {
-              it.moduleName = getModuleNameQualifier(project, module)
-              return it
-            }
+      
+      if (oldestParent != null) {
+        for (root in pathToFileManager.parentToSuitableRoot.get(oldestParent)) {
+          root.file.findFileByRelativePath(path)?.let { 
+            return PathInfo(null, it, root.file, root.moduleQualifier)
           }
         }
       }
     }
+    else {
+      for (rootProvider in RootProvider.values()) {
+        for (module in modules) {
+          if (module.isDisposed) {
+            continue
+          }
 
-    return findInLibraries(project, effectivePath, resolver)
+          findByRelativePath(path, rootProvider.getRoots(module.rootManager), resolver, null, pathQuery)?.let {
+            it.moduleName = getModuleNameQualifier(project, module)
+            return it
+          }
+        }
+      }
+    }
+    
+    if (!pathQuery.searchInLibs) {
+      // yes, if !searchInLibs, config.json is also not checked
+      return null
+    }
+    
+    fun findByConfigJson(): PathInfo? {
+      // https://youtrack.jetbrains.com/issue/WEB-24283
+      for (rootProvider in RootProvider.values()) {
+        for (module in modules) {
+          if (module.isDisposed) {
+            continue
+          }
+
+          for (root in rootProvider.getRoots(module.rootManager)) {
+            if (resolver.resolve("config.json", root, pathQuery = pathQuery) != null) {
+              resolver.resolve("index.html", root, pathQuery = pathQuery)?.let {
+                it.moduleName = getModuleNameQualifier(project, module)
+                return it
+              }
+            }
+          }
+        }
+      }
+      return null
+    }
+    
+    val exists = pathToFileManager.pathToExistShortTermCache.getIfPresent("config.json")
+    if (exists == null || exists) {
+      val result = findByConfigJson()
+      pathToFileManager.pathToExistShortTermCache.put("config.json", result != null)
+      if (result != null) {
+        return result
+      }
+    }
+
+    return findInLibraries(project, effectivePath, resolver, pathQuery)
   }
-
+  
   override fun getPathInfo(file: VirtualFile, project: Project): PathInfo? {
     return runReadAction {
       val directoryIndex = DirectoryIndex.getInstance(project)
@@ -138,7 +176,7 @@ private class DefaultWebServerRootsProvider : WebServerRootsProvider() {
   }
 }
 
-private enum class RootProvider {
+internal enum class RootProvider {
   SOURCE {
     override fun getRoots(rootManager: ModuleRootManager): Array<VirtualFile> = rootManager.sourceRoots
   },
@@ -169,7 +207,7 @@ private fun getJavadocOrderRootType(): OrderRootType? {
   }
 }
 
-private fun findInModuleLibraries(path: String, module: Module, resolver: FileResolver): PathInfo? {
+private fun findInModuleLibraries(path: String, module: Module, resolver: FileResolver, pathQuery: PathQuery): PathInfo? {
   val index = path.indexOf('/')
   if (index <= 0) {
     return null
@@ -179,12 +217,12 @@ private fun findInModuleLibraries(path: String, module: Module, resolver: FileRe
   val relativePath = path.substring(index + 1)
   return ORDER_ROOT_TYPES.computeOrNull {
     findInModuleLevelLibraries(module, it) { root, module ->
-      if (StringUtil.equalsIgnoreCase(root.nameSequence, libraryFileName)) resolver.resolve(relativePath, root, isLibrary = true) else null
+      if (StringUtil.equalsIgnoreCase(root.nameSequence, libraryFileName)) resolver.resolve(relativePath, root, isLibrary = true, pathQuery = pathQuery) else null
     }
   }
 }
 
-private fun findInLibraries(project: Project, path: String, resolver: FileResolver): PathInfo? {
+private fun findInLibraries(project: Project, path: String, resolver: FileResolver, pathQuery: PathQuery): PathInfo? {
   val index = path.indexOf('/')
   if (index < 0) {
     return null
@@ -193,7 +231,7 @@ private fun findInLibraries(project: Project, path: String, resolver: FileResolv
   val libraryFileName = path.substring(0, index)
   val relativePath = path.substring(index + 1)
   return findInLibrariesAndSdk(project, ORDER_ROOT_TYPES) { root, module ->
-    if (StringUtil.equalsIgnoreCase(root.nameSequence, libraryFileName)) resolver.resolve(relativePath, root, isLibrary = true) else null
+    if (StringUtil.equalsIgnoreCase(root.nameSequence, libraryFileName)) resolver.resolve(relativePath, root, isLibrary = true, pathQuery = pathQuery) else null
   }
 }
 
@@ -204,14 +242,14 @@ private fun getInfoForDocJar(file: VirtualFile, project: Project): PathInfo? {
   }
 }
 
-private fun getModuleNameQualifier(project: Project, module: Module?): String? {
+internal fun getModuleNameQualifier(project: Project, module: Module?): String? {
   if (module != null && PlatformUtils.isIntelliJ() && !(module.name.equals(project.name, ignoreCase = true) || compareNameAndProjectBasePath(module.name, project))) {
     return module.name
   }
   return null
 }
 
-private fun findByRelativePath(path: String, roots: Array<VirtualFile>, resolver: FileResolver, moduleName: String?) = roots.computeOrNull { resolver.resolve(path, it, moduleName) }
+private fun findByRelativePath(path: String, roots: Array<VirtualFile>, resolver: FileResolver, moduleName: String?, pathQuery: PathQuery) = roots.computeOrNull { resolver.resolve(path, it, moduleName, pathQuery = pathQuery) }
 
 private fun findInLibrariesAndSdk(project: Project, rootTypes: Array<OrderRootType>, fileProcessor: (root: VirtualFile, module: Module?) -> PathInfo?): PathInfo? {
   fun findInLibraryTable(table: LibraryTable, rootType: OrderRootType) = table.libraryIterator.computeOrNull { it.getFiles(rootType).computeOrNull { fileProcessor(it, null) } }

platform/built-in-server/src/org/jetbrains/builtInWebServer/PrefixlessWebServerRootsProvider.kt

@@ -3,7 +3,7 @@ package org.jetbrains.builtInWebServer
 import com.intellij.openapi.project.Project
 
 abstract class PrefixlessWebServerRootsProvider : WebServerRootsProvider() {
-  override final fun resolve(path: String, project: Project) = resolve(path, project, WebServerPathToFileManager.getInstance(project).getResolver(path))
+  override final fun resolve(path: String, project: Project, pathQuery: PathQuery) = resolve(path, project, WebServerPathToFileManager.getInstance(project).getResolver(path), pathQuery)
 
-  abstract fun resolve(path: String, project: Project, resolver: FileResolver): PathInfo?
+  abstract fun resolve(path: String, project: Project, resolver: FileResolver, pathQuery: PathQuery): PathInfo?
 }

platform/built-in-server/src/org/jetbrains/builtInWebServer/WebServerPathToFileManager.kt

@@ -1,9 +1,13 @@
 package org.jetbrains.builtInWebServer
 
+import com.google.common.base.Function
 import com.google.common.cache.CacheBuilder
+import com.google.common.cache.CacheLoader
 import com.intellij.ProjectTopics
 import com.intellij.openapi.application.Application
+import com.intellij.openapi.application.runReadAction
 import com.intellij.openapi.components.ServiceManager
+import com.intellij.openapi.module.ModuleManager
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.roots.ModuleRootEvent
 import com.intellij.openapi.roots.ModuleRootListener
@@ -13,20 +17,55 @@ import com.intellij.openapi.vfs.VirtualFileManager
 import com.intellij.openapi.vfs.newvfs.BulkFileListener
 import com.intellij.openapi.vfs.newvfs.events.VFileContentChangeEvent
 import com.intellij.openapi.vfs.newvfs.events.VFileEvent
+import com.intellij.project.rootManager
+import com.intellij.util.SmartList
 import com.intellij.util.containers.computeOrNull
 import com.intellij.util.io.exists
-import com.intellij.util.io.systemIndependentPath
 import java.nio.file.Paths
 import java.util.concurrent.TimeUnit
 
+private val cacheSize: Long = 4096 * 4
+
 /**
  * Implement [WebServerRootsProvider] to add your provider
  */
 class WebServerPathToFileManager(application: Application, private val project: Project) {
-  val pathToInfoCache = CacheBuilder.newBuilder().maximumSize(512).expireAfterAccess(10, TimeUnit.MINUTES).build<String, PathInfo>()!!
+  val pathToInfoCache = CacheBuilder.newBuilder().maximumSize(cacheSize).expireAfterAccess(10, TimeUnit.MINUTES).build<String, PathInfo>()!!
   // time to expire should be greater than pathToFileCache
-  private val virtualFileToPathInfo = CacheBuilder.newBuilder().maximumSize(512).expireAfterAccess(11, TimeUnit.MINUTES).build<VirtualFile, PathInfo>()
+  private val virtualFileToPathInfo = CacheBuilder.newBuilder().maximumSize(cacheSize).expireAfterAccess(11, TimeUnit.MINUTES).build<VirtualFile, PathInfo>()
+  
+  internal val pathToExistShortTermCache = CacheBuilder.newBuilder().maximumSize(cacheSize).expireAfterAccess(5, TimeUnit.SECONDS).build<String, Boolean>()!!
 
+  /**
+   * https://youtrack.jetbrains.com/issue/WEB-25900
+   * 
+   * Compute suitable roots for oldest parent (web/foo/my/file.dart -> oldest is web and we compute all suitable roots for it in advance) to avoid linear search
+   * (i.e. to avoid two queries for root if files web/foo and web/bar requested if root doesn't have web dir)
+   */
+  internal val parentToSuitableRoot = CacheBuilder.newBuilder().maximumSize(cacheSize).expireAfterAccess(10, TimeUnit.MINUTES).build<String, List<SuitableRoot>>(
+    CacheLoader.from(Function { path ->
+      val suitableRoots = SmartList<SuitableRoot>()
+      var moduleQualifier: String? = null
+      val modules = runReadAction { ModuleManager.getInstance(project).modules }
+      for (rootProvider in RootProvider.values()) {
+        for (module in modules) {
+          if (module.isDisposed) {
+            continue
+          }
+
+          for (root in rootProvider.getRoots(module.rootManager)) {
+            if (root.findChild(path!!) != null) {
+              if (moduleQualifier == null) {
+                moduleQualifier = getModuleNameQualifier(project, module)
+              }
+              suitableRoots.add(SuitableRoot(root, moduleQualifier))
+            }
+          }
+        }
+      }
+      suitableRoots
+    }))!!
+  
   init {
     application.messageBus.connect(project).subscribe(VirtualFileManager.VFS_CHANGES, object : BulkFileListener {
       override fun after(events: List<VFileEvent>) {
@@ -61,19 +100,29 @@ class WebServerPathToFileManager(application: Application, private val project:
   private fun clearCache() {
     pathToInfoCache.invalidateAll()
     virtualFileToPathInfo.invalidateAll()
+    pathToExistShortTermCache.invalidateAll()
+    parentToSuitableRoot.invalidateAll()
   }
 
-  @JvmOverloads fun findVirtualFile(path: String, cacheResult: Boolean = true): VirtualFile? {
-    val pathInfo = getPathInfo(path, cacheResult) ?: return null
-    return pathInfo.file ?: LocalFileSystem.getInstance().findFileByPath(pathInfo.ioFile!!.systemIndependentPath)
+  @JvmOverloads fun findVirtualFile(path: String, cacheResult: Boolean = true, pathQuery: PathQuery = defaultPathQuery): VirtualFile? {
+    return getPathInfo(path, cacheResult, pathQuery)?.getOrResolveVirtualFile()
   }
 
-  @JvmOverloads fun getPathInfo(path: String, cacheResult: Boolean = true): PathInfo? {
+  @JvmOverloads fun getPathInfo(path: String, cacheResult: Boolean = true, pathQuery: PathQuery = defaultPathQuery): PathInfo? {
     var pathInfo = pathToInfoCache.getIfPresent(path)
     if (pathInfo == null || !pathInfo.isValid) {
-      pathInfo = doFindByRelativePath(path)
-      if (cacheResult && pathInfo != null && pathInfo.isValid) {
-        pathToInfoCache.put(path, pathInfo)
+      if (pathToExistShortTermCache.getIfPresent(path) == false) {
+        return null
+      }
+      
+      pathInfo = doFindByRelativePath(path, pathQuery)
+      if (cacheResult) { 
+        if (pathInfo != null && pathInfo.isValid) {
+          pathToInfoCache.put(path, pathInfo)
+        }
+        else {
+          pathToExistShortTermCache.put(path, false)
+        }
       }
     }
     return pathInfo
@@ -92,8 +141,8 @@ class WebServerPathToFileManager(application: Application, private val project:
     return result
   }
 
-  internal fun doFindByRelativePath(path: String): PathInfo? {
-    val result = WebServerRootsProvider.EP_NAME.extensions.computeOrNull { it.resolve(path, project) } ?: return null
+  internal fun doFindByRelativePath(path: String, pathQuery: PathQuery): PathInfo? {
+    val result = WebServerRootsProvider.EP_NAME.extensions.computeOrNull { it.resolve(path, project, pathQuery) } ?: return null
     result.file?.let {
       virtualFileToPathInfo.put(it, result)
     }
@@ -104,32 +153,32 @@ class WebServerPathToFileManager(application: Application, private val project:
 }
 
 interface FileResolver {
-  fun resolve(path: String, root: VirtualFile, moduleName: String? = null, isLibrary: Boolean = false): PathInfo?
+  fun resolve(path: String, root: VirtualFile, moduleName: String? = null, isLibrary: Boolean = false, pathQuery: PathQuery): PathInfo?
 }
 
 private val RELATIVE_PATH_RESOLVER = object : FileResolver {
-  override fun resolve(path: String, root: VirtualFile, moduleName: String?, isLibrary: Boolean): PathInfo? {
+  override fun resolve(path: String, root: VirtualFile, moduleName: String?, isLibrary: Boolean, pathQuery: PathQuery): PathInfo? {
     // WEB-17691 built-in server doesn't serve files it doesn't have in the project tree
     // temp:// reports isInLocalFileSystem == true, but it is not true
-    if (root.isInLocalFileSystem && root.fileSystem == LocalFileSystem.getInstance()) {
-      val file = Paths.get(root.path, path)
-      if (file.exists()) {
-        return PathInfo(file, null, root, moduleName, isLibrary)
-      }
-      else {
-        return null
-      }
+    if (pathQuery.useVfs || root.fileSystem != LocalFileSystem.getInstance() || path == ".htaccess" || path == "config.json") {
+      return root.findFileByRelativePath(path)?.let { PathInfo(null, it, root, moduleName, isLibrary) }
+    }
+
+    val file = Paths.get(root.path, path)
+    return if (file.exists()) {
+      PathInfo(file, null, root, moduleName, isLibrary)
     }
     else {
-      val file = root.findFileByRelativePath(path) ?: return null
-      return PathInfo(null, file, root, moduleName, isLibrary)
+      null
     }
   }
 }
 
 private val EMPTY_PATH_RESOLVER = object : FileResolver {
-  override fun resolve(path: String, root: VirtualFile, moduleName: String?, isLibrary: Boolean): PathInfo? {
+  override fun resolve(path: String, root: VirtualFile, moduleName: String?, isLibrary: Boolean, pathQuery: PathQuery): PathInfo? {
     val file = findIndexFile(root) ?: return null
     return PathInfo(null, file, root, moduleName, isLibrary)
   }
-}
+}
+
+internal val defaultPathQuery = PathQuery()