name index to decrease unsuccessful io operations during class search &

do not store strings in class path cache

platform/util/src/com/intellij/util/lang/ClassPath.java

@@ -24,7 +24,6 @@ import com.intellij.util.SmartList;
 import com.intellij.util.containers.HashMap;
 import com.intellij.util.containers.Stack;
 import org.jetbrains.annotations.NonNls;
-import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import sun.misc.Resource;
 
@@ -105,7 +104,7 @@ class ClassPath {
     push(urls);
   }
 
-  // Accessed by reflection from PluginClassLoader
+  // Accessed by reflection from PluginClassLoader // TODO: do we need it?
   void addURL(URL url) {
     push(new URL[]{url});
   }
@@ -117,7 +116,10 @@ class ClassPath {
     try {
       int i;
       if (myCanUseCache) {
-        for (Loader loader : getLoaders(s)) {
+        List<Loader> loaders = myCache.getLoaders(s);
+        for (int j = 0, size = loaders.size(); j < size; ++j ) {
+          final Loader loader = loaders.get(j);
+          if (!myCache.loaderHasName(s, loader)) continue;
           final Resource resource = loader.getResource(s, flag);
           if (resource != null) {
             if (ourDumpOrder) {
@@ -154,10 +156,6 @@ class ClassPath {
     }
   }
 
-  private @NotNull List<Loader> getLoaders(String s) {
-    return myCache.getLoaders(s);
-  }
-
   public Enumeration<URL> getResources(final String name, final boolean check) {
     return new MyEnumeration(name, check);
   }
@@ -175,7 +173,7 @@ class ClassPath {
 
       Loader loader;
       try {
-        loader = getLoader(url);
+        loader = getLoader(url, myLoaders.size());
         if (loader == null) continue;
       }
       catch (IOException ioexception) {
@@ -190,7 +188,7 @@ class ClassPath {
   }
 
   @Nullable
-  private Loader getLoader(final URL url) throws IOException {
+  private Loader getLoader(final URL url, int index) throws IOException {
     String s;
     if (myAcceptUnescapedUrls) {
       s = url.getFile();
@@ -206,11 +204,11 @@ class ClassPath {
     Loader loader = null;
     if (s != null  && new File(s).isDirectory()) {
       if (FILE_PROTOCOL.equals(url.getProtocol())) {
-        loader = new FileLoader(url);
+        loader = new FileLoader(url, index);
       }
     }
     else {
-      JarLoader jarLoader = new JarLoader(url, myCanLockJars);
+      JarLoader jarLoader = new JarLoader(url, myCanLockJars, index);
       jarLoader.preLoadClasses();
       loader = jarLoader;
     }
@@ -249,10 +247,10 @@ class ClassPath {
       if (myCanUseCache) {
         synchronized (myUrls) {
           if (myUrls.isEmpty()) {
-            loaders = getLoaders(name);
+            loaders = myCache.getLoaders(name);
             if (!name.endsWith("/")) {
               loaders = new SmartList<Loader>(loaders);
-              loaders.addAll(getLoaders(name + "/"));
+              loaders.addAll(myCache.getLoaders(name + "/"));
             }
           }
         }
@@ -268,7 +266,12 @@ class ClassPath {
 
       if (myLoaders != null) {
         while (myIndex < myLoaders.size()) {
-          myRes = myLoaders.get(myIndex++).getResource(myName, myCheck);
+          loader = myLoaders.get(myIndex++);
+          if (!myCache.loaderHasName(myName, loader)) {
+            myRes = null;
+            continue;
+          }
+          myRes = loader.getResource(myName, myCheck);
           if (myRes != null) return true;
         }
       } else {

platform/util/src/com/intellij/util/lang/ClasspathCache.java

@@ -20,14 +20,15 @@
 package com.intellij.util.lang;
 
 import com.intellij.util.SmartList;
+import gnu.trove.TIntHashSet;
+import gnu.trove.TIntObjectHashMap;
 
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 
 public class ClasspathCache {
-  private final Map<String, List<Loader>> myClassPackagesCache = new HashMap<String, List<Loader>>();
-  private final Map<String, List<Loader>> myResourcePackagesCache = new HashMap<String, List<Loader>>();
+  private final TIntObjectHashMap<List<Loader>> myClassPackagesCache = new TIntObjectHashMap<List<Loader>>();
+  private final TIntObjectHashMap<List<Loader>> myResourcePackagesCache = new TIntObjectHashMap<List<Loader>>();
+  private final TIntHashSet myResourceIndex = new TIntHashSet();
 
   public void addResourceEntry(String resourcePath, Loader loader) {
     final List<Loader> loaders = getLoaders(resourcePath);
@@ -41,13 +42,64 @@ public class ClasspathCache {
     final int idx = resourcePath.lastIndexOf('/');
     String packageName = idx > 0 ? resourcePath.substring(0, idx) : "";
 
-    Map<String, List<Loader>> map = isClassFile ? myClassPackagesCache : myResourcePackagesCache;
-    List<Loader> list = map.get(packageName);
+    TIntObjectHashMap<List<Loader>> map = isClassFile ? myClassPackagesCache : myResourcePackagesCache;
+    int hash = packageName.hashCode();
+    List<Loader> list = map.get(hash);
     if (list == null) {
       list = new SmartList<Loader>();
-      map.put(packageName, list);
+      map.put(hash, list);
     }
 
     return list;
   }
+
+  public void addNameEntry(String name, Loader loader) {
+    int hash = hashFromNameAndLoader(transformName(name), loader);
+    myResourceIndex.add(hash);
+  }
+
+  public boolean loaderHasName(String name, Loader loader) {
+    int hash = hashFromNameAndLoader(transformName(name), loader);
+
+    boolean result = myResourceIndex.contains(hash);
+    ++requests;
+
+    if (!result) ++hits;
+
+    if (requests % 1000 == 0 && UrlClassLoader.doDebug && false) {
+      UrlClassLoader.debug("Avoided disk hits: "+hits + " from " + requests);
+    }
+    return result;
+  }
+
+  private String transformName(String name) {
+    if (name.endsWith("/")) {
+      name = name.substring(0, name.length() - 1);
+    }
+    name = name.substring(name.lastIndexOf('/') + 1);
+
+    if (name.endsWith(UrlClassLoader.CLASS_EXTENSION)) {
+      String name1 = name;
+      int $ = name1.indexOf('$');
+      if ($ != -1) name1 = name1.substring(0, $);
+      else {
+        int index = name1.lastIndexOf('.');
+        if (index >= 0) name1 = name1.substring(0, index);
+      }
+      name = name1;
+    }
+    return name;
+  }
+
+  private static int hits, requests;
+
+  private int hashFromNameAndLoader(String name, Loader loader) {
+    int hash = name.hashCode();
+    int i = loader.getIndex();
+    while(i > 0) {
+      hash = hash * 31 + ((i % 10) + '0');
+      i /= 10;
+    }
+    return hash;
+  }
 }

platform/util/src/com/intellij/util/lang/FileLoader.java

@@ -27,10 +27,12 @@ import java.net.URL;
 class FileLoader extends Loader {
   private final File myRootDir;
   private final String myRootDirAbsolutePath;
+  private static int misses;
+  private static int hits;
 
   @SuppressWarnings({"HardCodedStringLiteral"})
-  FileLoader(URL url) throws IOException {
-    super(url);
+  FileLoader(URL url, int index) throws IOException {
+    super(url, index);
     if (!"file".equals(url.getProtocol())) {
       throw new IllegalArgumentException("url");
     }
@@ -58,8 +60,10 @@ class FileLoader extends Loader {
           cache.addResourceEntry(getRelativeResourcePath(file), this);
           containsClasses = true;
         }
+        cache.addNameEntry(file.getName(), this);
       }
       else {
+        cache.addNameEntry(file.getName(), this);
         buildPackageCache(file, cache);
       }
     }
@@ -79,7 +83,18 @@ class FileLoader extends Loader {
       if (!url.getFile().startsWith(getBaseURL().getFile())) return null;
 
       final File file = new File(myRootDir, name.replace('/', File.separatorChar));
-      if (file.exists()) return new MyResource(name, url, file);
+      if (file.exists()) {
+        ++hits;
+        if (hits % 1000 == 0 && UrlClassLoader.doDebug) {
+          UrlClassLoader.debug("Exists file loader: misses:" + misses + ", hits:" + hits);
+        }
+        return new MyResource(name, url, file);
+      }
+
+      if (misses % 1000 == 0 && UrlClassLoader.doDebug) {
+        UrlClassLoader.debug("Missed " + name + " from " + myRootDir);
+      }
+      ++misses;
     }
     catch (Exception exception) {
       return null;

platform/util/src/com/intellij/util/lang/JarLoader.java

@@ -36,6 +36,8 @@ class JarLoader extends Loader {
   private SoftReference<JarMemoryLoader> myMemoryLoader;
   private final boolean myCanLockJar;
   private static final boolean myDebugTime = false;
+  private static int misses;
+  private static int hits;
 
   private static final Logger LOG = Logger.getInstance(JarLoader.class);
 
@@ -57,8 +59,8 @@ class JarLoader extends Loader {
   @NonNls private static final String FILE_PROTOCOL = "file";
   private static final long NS_THRESHOLD = 10000000;
 
-  JarLoader(URL url, boolean canLockJar) throws IOException {
-    super(new URL(JAR_PROTOCOL, "", -1, url + "!/"));
+  JarLoader(URL url, boolean canLockJar, int index) throws IOException {
+    super(new URL(JAR_PROTOCOL, "", -1, url + "!/"), index);
     myURL = url;
     myCanLockJar = canLockJar;
   }
@@ -130,7 +132,9 @@ class JarLoader extends Loader {
 
       while (entries.hasMoreElements()) {
         ZipEntry zipEntry = entries.nextElement();
-        cache.addResourceEntry(zipEntry.getName(), this);
+        String name = zipEntry.getName();
+        cache.addResourceEntry(name, this);
+        cache.addNameEntry(name, this);
       }
     }
     finally {
@@ -153,7 +157,18 @@ class JarLoader extends Loader {
       file = acquireZipFile();
       if (file == null) return null;
       ZipEntry entry = file.getEntry(name);
-      if (entry != null) return new MyResource(entry, new URL(getBaseURL(), name));
+      if (entry != null) {
+        ++hits;
+        if (hits % 1000 == 0 && UrlClassLoader.doDebug) {
+          UrlClassLoader.debug("Exists jar loader: misses:" + misses + ", hits:" + hits);
+        }
+        return new MyResource(entry, new URL(getBaseURL(), name));
+      }
+
+      if (misses % 1000 == 0 && UrlClassLoader.doDebug) {
+        UrlClassLoader.debug("Missed " + name + " from jar:" + myURL);
+      }
+      ++misses;
     }
     catch (Exception e) {
       return null;

platform/util/src/com/intellij/util/lang/Loader.java

@@ -24,9 +24,11 @@ import java.net.URL;
 
 abstract class Loader {
   private final URL myURL;
+  private final int myIndex;
 
-  protected Loader(URL url) {
+  protected Loader(URL url, int index) {
     myURL = url;
+    myIndex = index;
   }
 
 
@@ -38,4 +40,8 @@ abstract class Loader {
   abstract Resource getResource(final String name, boolean flag);
 
   abstract void buildCache(ClasspathCache cache) throws IOException;
+
+  public int getIndex() {
+    return myIndex;
+  }
 }

platform/util/src/com/intellij/util/lang/UrlClassLoader.java

@@ -16,6 +16,7 @@
 
 package com.intellij.util.lang;
 
+import com.intellij.openapi.diagnostic.Logger;
 import org.jetbrains.annotations.NonNls;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
@@ -124,7 +125,7 @@ public class UrlClassLoader extends ClassLoader {
     return defineClass(name, b, 0, b.length);
   }
 
-  @Nullable
+  @Nullable  // Accessed from PluginClassLoader via reflection // TODO do we need it?
   public URL findResource(final String name) {
     final long started = myDebugTime ? System.nanoTime():0;
 
@@ -165,7 +166,16 @@ public class UrlClassLoader extends ClassLoader {
     }
   }
 
+  // Accessed from PluginClassLoader via reflection // TODO do we need it?
   protected Enumeration<URL> findResources(String name) throws IOException {
     return myClassPath.getResources(name, true);
   }
+  
+  static final boolean doDebug = System.getProperty("idea.classloading.debug") != null;
+  private static final Logger LOG = Logger.getInstance("idea.UrlClassLoader");
+
+  static void debug(String s) {
+    System.out.println(s); // TODO: remove
+    LOG.debug(s);
+  }
 }