UrlClassLoader caching: a less exposing API

platform/util/src/com/intellij/util/lang/CachePoolImpl.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2000-2014 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.util.lang;
+
+import com.intellij.util.containers.ContainerUtil;
+import org.jetbrains.annotations.NotNull;
+
+import java.net.URL;
+import java.util.Map;
+
+/**
+ * @author peter
+ */
+class CachePoolImpl implements UrlClassLoader.CachePool {
+  private final Map<URL, ClasspathCache.LoaderData> myLoaderIndexCache = ContainerUtil.newConcurrentMap();
+  
+  void cacheData(@NotNull URL url, @NotNull ClasspathCache.LoaderData data) {
+    myLoaderIndexCache.put(url, data);
+  }
+
+  ClasspathCache.LoaderData getCachedData(@NotNull URL url) {
+    return myLoaderIndexCache.get(url);
+  }
+
+}

platform/util/src/com/intellij/util/lang/ClassPath.java

@@ -24,7 +24,6 @@ import com.intellij.util.SmartList;
 import com.intellij.util.containers.HashMap;
 import com.intellij.util.containers.Stack;
 import com.intellij.util.io.URLUtil;
-import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import sun.misc.Resource;
 
@@ -50,14 +49,22 @@ public class ClassPath {
   private final boolean myCanUseCache;
   private final boolean myAcceptUnescapedUrls;
   private final boolean myPreloadJarContents;
-  private final LoaderIndexProvider myIndexProvider;
+  @Nullable private final CachePoolImpl myCachePool;
+  @Nullable private final UrlClassLoader.CachingCondition myCachingCondition;
 
-  public ClassPath(List<URL> urls, boolean canLockJars, boolean canUseCache, boolean acceptUnescapedUrls, boolean preloadJarContents, @NotNull LoaderIndexProvider indexProvider) {
+  public ClassPath(List<URL> urls,
+                   boolean canLockJars,
+                   boolean canUseCache,
+                   boolean acceptUnescapedUrls,
+                   boolean preloadJarContents,
+                   @Nullable CachePoolImpl cachePool,
+                   @Nullable UrlClassLoader.CachingCondition cachingCondition) {
     myCanLockJars = canLockJars;
     myCanUseCache = canUseCache;
     myAcceptUnescapedUrls = acceptUnescapedUrls;
     myPreloadJarContents = preloadJarContents;
-    myIndexProvider = indexProvider;
+    myCachePool = cachePool;
+    myCachingCondition = cachingCondition;
     push(urls);
   }
 
@@ -184,7 +191,14 @@ public class ClassPath {
     }
 
     if (loader != null && myCanUseCache) {
-      myCache.applyLoaderData(myIndexProvider.getLoaderData(loader), loader);
+      ClasspathCache.LoaderData data = myCachePool == null ? null : myCachePool.getCachedData(url);
+      if (data == null) {
+        data = loader.buildData();
+        if (myCachePool != null && myCachingCondition != null && myCachingCondition.shouldCacheData(url)) {
+          myCachePool.cacheData(url, data);
+        }
+      }
+      myCache.applyLoaderData(data, loader);
     }
 
     return loader;

platform/util/src/com/intellij/util/lang/ClasspathCache.java

@@ -28,6 +28,7 @@ import gnu.trove.TIntObjectHashMap;
 import org.jetbrains.annotations.Nullable;
 import sun.misc.Resource;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.locks.ReadWriteLock;
@@ -53,19 +54,31 @@ public class ClasspathCache {
     myDebugInfo = doDebug ? new DebugInfo() : new NullDebugInfo();
   }
 
+  static class LoaderData {
+    private final List<String> myResourcePaths = new ArrayList<String>();
+    private final List<String> myNames = new ArrayList<String>();
+
+    public void addResourceEntry(String resourcePath) {
+      myResourcePaths.add(resourcePath);
+    }
+
+    public void addNameEntry(String name) {
+      myNames.add(transformName(name));
+    }
+  }
+
   private final ReadWriteLock myLock = new ReentrantReadWriteLock();
 
-  void applyLoaderData(ClasspathLoaderIndex loaderData, final Loader loader) {
+  public void applyLoaderData(LoaderData loaderData, Loader loader) {
     myLock.writeLock().lock();
     try {
-      for (String resourceEntry : loaderData.getResourcePaths()) {
+      for(String resourceEntry:loaderData.myResourcePaths) {
         addResourceEntry(resourceEntry, loader);
       }
-      for (String name : loaderData.getNames()) {
+      for(String name:loaderData.myNames) {
         addNameEntry(name, loader);
       }
-    }
-    finally {
+    } finally {
       myLock.writeLock().unlock();
     }
   }
@@ -360,7 +373,7 @@ public class ClasspathCache {
 
     private static int hashFromNameAndLoader(String name, Loader loader, int n) {
       int hash = StringHash.murmur(name, n);
-      int i = loader.getOrderNumber();
+      int i = loader.getIndex();
       while (i > 0) {
         hash = hash * n + ((i % 10) + '0');
         i /= 10;
@@ -403,7 +416,7 @@ public class ClasspathCache {
 
     protected static int hashFromNameAndLoader(String name, Loader loader) {
       int hash = name.hashCode();
-      int i = loader.getOrderNumber();
+      int i = loader.getIndex();
       while(i > 0) {
         hash = hash * 31 + ((i % 10) + '0');
         i /= 10;

platform/util/src/com/intellij/util/lang/ClasspathLoaderIndex.java

@@ -1,49 +0,0 @@
-/*
- * Copyright 2000-2014 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.intellij.util.lang;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Holds resource and class names that are present inside a jar or a directory represented by a particular {@link Loader} instance.
- * This data is then used by {@link UrlClassLoader} to avoid looking inside particular loaders when searching for classes/resources, and thus
- * to speed up the operation.<p/>
- * 
- * @see LoaderIndexProvider
- * 
- * @author peter
-*/
-public class ClasspathLoaderIndex {
-  private final List<String> myResourcePaths = new ArrayList<String>();
-  private final List<String> myNames = new ArrayList<String>();
-
-  public void addResourceEntry(String resourcePath) {
-    myResourcePaths.add(resourcePath);
-  }
-
-  public void addNameEntry(String name) {
-    myNames.add(ClasspathCache.transformName(name));
-  }
-
-  public List<String> getResourcePaths() {
-    return myResourcePaths;
-  }
-
-  public List<String> getNames() {
-    return myNames;
-  }
-}

platform/util/src/com/intellij/util/lang/FileLoader.java

@@ -17,6 +17,7 @@ package com.intellij.util.lang;
 
 import com.intellij.openapi.util.io.FileUtil;
 import org.jetbrains.annotations.NonNls;
+import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import sun.misc.Resource;
 
@@ -35,7 +36,7 @@ class FileLoader extends Loader {
     myRootDirAbsolutePath = myRootDir.getAbsolutePath();
   }
 
-  private void buildPackageCache(final File dir, ClasspathLoaderIndex loaderData) {
+  private void buildPackageCache(final File dir, ClasspathCache.LoaderData loaderData) {
     loaderData.addResourceEntry(getRelativeResourcePath(dir));
 
     final File[] files = dir.listFiles();
@@ -109,9 +110,10 @@ class FileLoader extends Loader {
     return null;
   }
 
+  @NotNull
   @Override
-  public ClasspathLoaderIndex buildIndex() throws IOException {
-    ClasspathLoaderIndex loaderData = new ClasspathLoaderIndex();
+  public ClasspathCache.LoaderData buildData() throws IOException {
+    ClasspathCache.LoaderData loaderData = new ClasspathCache.LoaderData();
     File index = new File(myRootDir, "classpath.index");
     if (index.exists()) {
       BufferedReader reader = new BufferedReader(new FileReader(index));

platform/util/src/com/intellij/util/lang/JarLoader.java

@@ -19,6 +19,7 @@ import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.openapi.util.io.FileUtil;
 import com.intellij.openapi.util.io.ZipFileCache;
 import com.intellij.util.io.URLUtil;
+import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import sun.misc.Resource;
 
@@ -68,9 +69,10 @@ class JarLoader extends Loader {
     }
   }
 
+  @NotNull
   @Override
-  public ClasspathLoaderIndex buildIndex() throws IOException {
-    ClasspathLoaderIndex loaderData = new ClasspathLoaderIndex();
+  public ClasspathCache.LoaderData buildData() throws IOException {
+    ClasspathCache.LoaderData loaderData = new ClasspathCache.LoaderData();
     ZipFile zipFile = acquireZipFile();
     try {
       Enumeration<? extends ZipEntry> entries = zipFile.entries();

platform/util/src/com/intellij/util/lang/Loader.java

@@ -15,6 +15,7 @@
  */
 package com.intellij.util.lang;
 
+import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import sun.misc.Resource;
 
@@ -27,25 +28,25 @@ import java.net.URL;
  * @see JarLoader
  * @see FileLoader
  */
-public abstract class Loader {
+abstract class Loader {
   private final URL myURL;
-  private final int myOrderNumber;
+  private final int myIndex;
 
-  Loader(URL url, int orderNumber) {
+  Loader(URL url, int index) {
     myURL = url;
-    myOrderNumber = orderNumber;
+    myIndex = index;
   }
 
-  public URL getBaseURL() {
+  URL getBaseURL() {
     return myURL;
   }
 
   @Nullable
   abstract Resource getResource(String name, boolean flag);
   
-  public abstract ClasspathLoaderIndex buildIndex() throws IOException;
+  @NotNull abstract ClasspathCache.LoaderData buildData() throws IOException;
 
-  int getOrderNumber() {
-    return myOrderNumber;
+  int getIndex() {
+    return myIndex;
   }
 }

platform/util/src/com/intellij/util/lang/LoaderIndexProvider.java

@@ -1,51 +0,0 @@
-/*
- * Copyright 2000-2014 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.intellij.util.lang;
-
-import org.jetbrains.annotations.NotNull;
-
-import java.io.IOException;
-
-/**
- * A way to plug into {@link UrlClassLoader} indexing policy. Can be used to reuse the index in several classloaders
- * where particular class path elements are the same.
- * 
- * @see UrlClassLoader.Builder#useCache(LoaderIndexProvider) 
- * 
- * @author peter
- */
-public abstract class LoaderIndexProvider {
-  /**
-   * The default implementation which just calls returns {@link Loader#buildIndex()} result.
-   */
-  public static final LoaderIndexProvider DEFAULT_INSTANCE = new LoaderIndexProvider() {
-    @NotNull
-    @Override
-    public ClasspathLoaderIndex getLoaderData(@NotNull Loader loader) throws IOException {
-      return loader.buildIndex();
-    }
-  };
-
-  /**
-   * @param loader the jar or a file loader to build the index for.
-   * @return a (maybe cached) index for this loader, obtained using {@link Loader#buildIndex()} 
-   * @throws IOException
-   */
-  @NotNull 
-  public abstract ClasspathLoaderIndex getLoaderData(@NotNull Loader loader) throws IOException;
-
-}
-

platform/util/src/com/intellij/util/lang/UrlClassLoader.java

@@ -67,7 +67,8 @@ public class UrlClassLoader extends ClassLoader {
     private boolean myAcceptUnescaped = false;
     private boolean myPreload = true;
     private boolean myAllowBootstrapResources = false;
-    @NotNull private LoaderIndexProvider myIndexProvider = LoaderIndexProvider.DEFAULT_INSTANCE;
+    @Nullable private CachePoolImpl myCachePool = null;
+    @Nullable private CachingCondition myCachingCondition = null;
 
     private Builder() { }
 
@@ -80,13 +81,22 @@ public class UrlClassLoader extends ClassLoader {
     public Builder useCache(boolean useCache) { myUseCache = useCache; return this; }
 
     /**
-     * Requests the class loader being built to use cache and to obtain the necessary classpath index from a custom index provider. 
+     * Requests the class loader being built to use cache and, if possible, retrieve and store the cached data from a special cache pool
+     * that can be shared between several loaders.  
+
+     * @param pool cache pool
+     * @param condition a custom policy to provide a possibility to prohibit caching for some URLs.
+     * @return this instance
      * 
-     * @param provider a custom index provider
-     * @return the same instance
-     * @see LoaderIndexProvider
+     * @see #createCachePool() 
      */
-    public Builder useCache(@NotNull LoaderIndexProvider provider) { myUseCache = true; myIndexProvider = provider; return this; }
+    public Builder useCache(@NotNull CachePool pool, @NotNull CachingCondition condition) { 
+      myUseCache = true;
+      myCachePool = (CachePoolImpl)pool;
+      myCachingCondition = condition; 
+      return this; 
+    }
+    
     public Builder allowUnescaped() { myAcceptUnescaped = true; return this; }
     public Builder noPreload() { myPreload = false; return this; }
     public Builder allowBootstrapResources() { myAllowBootstrapResources = true; return this; }
@@ -131,7 +141,7 @@ public class UrlClassLoader extends ClassLoader {
         return internProtocol(url);
       }
     });
-    myClassPath = new ClassPath(myURLs, lockJars, useCache, allowUnescaped, preload, LoaderIndexProvider.DEFAULT_INSTANCE);
+    myClassPath = new ClassPath(myURLs, lockJars, useCache, allowUnescaped, preload, null, null);
     myAllowBootstrapResources = false;
   }
 
@@ -143,7 +153,7 @@ public class UrlClassLoader extends ClassLoader {
         return internProtocol(url);
       }
     });
-    myClassPath = new ClassPath(myURLs, builder.myLockJars, builder.myUseCache, builder.myAcceptUnescaped, builder.myPreload, builder.myIndexProvider);
+    myClassPath = new ClassPath(myURLs, builder.myLockJars, builder.myUseCache, builder.myAcceptUnescaped, builder.myPreload, builder.myCachePool, builder.myCachingCondition);
     myAllowBootstrapResources = builder.myAllowBootstrapResources;
   }
 
@@ -301,4 +311,40 @@ public class UrlClassLoader extends ClassLoader {
     else if (SystemInfo.isLinux) return "linux/";
     else return "";
   }
+
+  /**
+   * An interface for a pool to store internal class loader caches, that can be shared between several different class loaders,
+   * if they contain the same URLs in their classpaths.<p/>
+   * 
+   * The implementation is subject to change so one shouldn't rely on it.
+   * 
+   * @see #createCachePool()
+   * @see Builder#useCache(CachePool, CachingCondition) 
+   */
+  public interface CachePool {}
+
+  /**
+   * A condition to customize the caching policy when using {@link CachePool}. This might be needed when a class loader is used on a directory
+   * that's being written into, to avoid the situation when a resource path is cached as nonexistent but then a file actually appears there,
+   * and other class loaders with the same caching pool should have access to these new resources. This can happen during compilation process
+   * with several module outputs.
+   */
+  public interface CachingCondition {
+
+    /**
+     * @param url
+     * @return whether the internal information should be cached for files in a specific classpath component URL: inside the directory or
+     * a jar.
+     */
+    boolean shouldCacheData(@NotNull URL url);
+  }
+
+  /**
+   * @return a new pool to be able to share internal class loader caches between several different class loaders, if they contain the same URLs
+   * in their classpaths.
+   */
+  @NotNull 
+  public static CachePool createCachePool() {
+    return new CachePoolImpl();
+  }
 }

plugins/groovy/jps-plugin/src/org/jetbrains/jps/incremental/groovy/InProcessGroovyc.java

@@ -35,7 +35,6 @@ import java.net.URL;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
-import java.util.Map;
 import java.util.regex.Pattern;
 
 /**
@@ -44,7 +43,7 @@ import java.util.regex.Pattern;
 class InProcessGroovyc {
   private static final Pattern GROOVY_ALL_JAR_PATTERN = Pattern.compile("groovy-all(-(.*))?\\.jar");
   private static SoftReference<Pair<String, ClassLoader>> ourParentLoaderCache;
-  private static final Map<URL, ClasspathLoaderIndex> ourLoaderIndexCache = ContainerUtil.newConcurrentMap();
+  private static final UrlClassLoader.CachePool ourLoaderCachePool = UrlClassLoader.createCachePool();
 
   @SuppressWarnings("UseOfSystemOutOrSystemErr")
   static void runGroovycInThisProcess(Collection<String> compilationClassPath,
@@ -55,29 +54,20 @@ class InProcessGroovyc {
 
     ClassLoader parent = obtainParentLoader(compilationClassPath);
 
-    UrlClassLoader loader = UrlClassLoader.build().urls(toUrls(compilationClassPath)).parent(parent).useCache(new LoaderIndexProvider() {
-      @NotNull
-      @Override
-      public ClasspathLoaderIndex getLoaderData(@NotNull Loader loader) throws IOException {
-        URL url = loader.getBaseURL();
-        ClasspathLoaderIndex index = ourLoaderIndexCache.get(url);
-        if (index != null) {
-          return index;
-        }
-
-        index = loader.buildIndex();
-        String file = url.getFile();
-        for (String output : outputs) {
-          if (FileUtil.startsWith(output + "/", file)) {
-            return index; // don't cache outputs
+    UrlClassLoader loader = UrlClassLoader.build().
+      urls(toUrls(compilationClassPath)).parent(parent).
+      useCache(ourLoaderCachePool, new UrlClassLoader.CachingCondition() {
+        @Override
+        public boolean shouldCacheData(@NotNull URL url) {
+          String file = url.getFile();
+          for (String output : outputs) {
+            if (FileUtil.startsWith(output + "/", file)) {
+              return false;
+            }
           }
+          return true;
         }
-
-        ourLoaderIndexCache.put(url, index);
-        return index;
-
-      }
-    }).get();
+      }).get();
 
     PrintStream oldOut = System.out;
     PrintStream oldErr = System.err;