From 9554777a33d746ac38d92ded829bb0955c24ef6a Mon Sep 17 00:00:00 2001 From: peter Date: Thu, 11 Dec 2014 16:25:04 +0100 Subject: [PATCH] UrlClassLoader caching: a less exposing API --- .../com/intellij/util/lang/CachePoolImpl.java | 38 ++++++++++++ .../src/com/intellij/util/lang/ClassPath.java | 24 +++++-- .../intellij/util/lang/ClasspathCache.java | 27 +++++--- .../util/lang/ClasspathLoaderIndex.java | 49 --------------- .../com/intellij/util/lang/FileLoader.java | 8 ++- .../src/com/intellij/util/lang/JarLoader.java | 6 +- .../src/com/intellij/util/lang/Loader.java | 17 ++--- .../util/lang/LoaderIndexProvider.java | 51 --------------- .../intellij/util/lang/UrlClassLoader.java | 62 ++++++++++++++++--- .../incremental/groovy/InProcessGroovyc.java | 36 ++++------- 10 files changed, 162 insertions(+), 156 deletions(-) create mode 100644 platform/util/src/com/intellij/util/lang/CachePoolImpl.java delete mode 100644 platform/util/src/com/intellij/util/lang/ClasspathLoaderIndex.java delete mode 100644 platform/util/src/com/intellij/util/lang/LoaderIndexProvider.java diff --git a/platform/util/src/com/intellij/util/lang/CachePoolImpl.java b/platform/util/src/com/intellij/util/lang/CachePoolImpl.java new file mode 100644 index 000000000000..3648b8139d86 --- /dev/null +++ b/platform/util/src/com/intellij/util/lang/CachePoolImpl.java @@ -0,0 +1,38 @@ +/* + * Copyright 2000-2014 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.lang; + +import com.intellij.util.containers.ContainerUtil; +import org.jetbrains.annotations.NotNull; + +import java.net.URL; +import java.util.Map; + +/** + * @author peter + */ +class CachePoolImpl implements UrlClassLoader.CachePool { + private final Map myLoaderIndexCache = ContainerUtil.newConcurrentMap(); + + void cacheData(@NotNull URL url, @NotNull ClasspathCache.LoaderData data) { + myLoaderIndexCache.put(url, data); + } + + ClasspathCache.LoaderData getCachedData(@NotNull URL url) { + return myLoaderIndexCache.get(url); + } + +} diff --git a/platform/util/src/com/intellij/util/lang/ClassPath.java b/platform/util/src/com/intellij/util/lang/ClassPath.java index 971370ffd1cb..a31bf1b149df 100644 --- a/platform/util/src/com/intellij/util/lang/ClassPath.java +++ b/platform/util/src/com/intellij/util/lang/ClassPath.java @@ -24,7 +24,6 @@ import com.intellij.util.SmartList; import com.intellij.util.containers.HashMap; import com.intellij.util.containers.Stack; import com.intellij.util.io.URLUtil; -import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import sun.misc.Resource; @@ -50,14 +49,22 @@ public class ClassPath { private final boolean myCanUseCache; private final boolean myAcceptUnescapedUrls; private final boolean myPreloadJarContents; - private final LoaderIndexProvider myIndexProvider; + @Nullable private final CachePoolImpl myCachePool; + @Nullable private final UrlClassLoader.CachingCondition myCachingCondition; - public ClassPath(List urls, boolean canLockJars, boolean canUseCache, boolean acceptUnescapedUrls, boolean preloadJarContents, @NotNull LoaderIndexProvider indexProvider) { + public ClassPath(List urls, + boolean canLockJars, + boolean canUseCache, + boolean acceptUnescapedUrls, + boolean preloadJarContents, + @Nullable CachePoolImpl cachePool, + @Nullable UrlClassLoader.CachingCondition cachingCondition) { myCanLockJars = canLockJars; myCanUseCache = canUseCache; myAcceptUnescapedUrls = acceptUnescapedUrls; myPreloadJarContents = preloadJarContents; - myIndexProvider = indexProvider; + myCachePool = cachePool; + myCachingCondition = cachingCondition; push(urls); } @@ -184,7 +191,14 @@ public class ClassPath { } if (loader != null && myCanUseCache) { - myCache.applyLoaderData(myIndexProvider.getLoaderData(loader), loader); + ClasspathCache.LoaderData data = myCachePool == null ? null : myCachePool.getCachedData(url); + if (data == null) { + data = loader.buildData(); + if (myCachePool != null && myCachingCondition != null && myCachingCondition.shouldCacheData(url)) { + myCachePool.cacheData(url, data); + } + } + myCache.applyLoaderData(data, loader); } return loader; diff --git a/platform/util/src/com/intellij/util/lang/ClasspathCache.java b/platform/util/src/com/intellij/util/lang/ClasspathCache.java index 43c09e0b6431..7d43900cf6ee 100644 --- a/platform/util/src/com/intellij/util/lang/ClasspathCache.java +++ b/platform/util/src/com/intellij/util/lang/ClasspathCache.java @@ -28,6 +28,7 @@ import gnu.trove.TIntObjectHashMap; import org.jetbrains.annotations.Nullable; import sun.misc.Resource; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.locks.ReadWriteLock; @@ -53,19 +54,31 @@ public class ClasspathCache { myDebugInfo = doDebug ? new DebugInfo() : new NullDebugInfo(); } + static class LoaderData { + private final List myResourcePaths = new ArrayList(); + private final List myNames = new ArrayList(); + + public void addResourceEntry(String resourcePath) { + myResourcePaths.add(resourcePath); + } + + public void addNameEntry(String name) { + myNames.add(transformName(name)); + } + } + private final ReadWriteLock myLock = new ReentrantReadWriteLock(); - void applyLoaderData(ClasspathLoaderIndex loaderData, final Loader loader) { + public void applyLoaderData(LoaderData loaderData, Loader loader) { myLock.writeLock().lock(); try { - for (String resourceEntry : loaderData.getResourcePaths()) { + for(String resourceEntry:loaderData.myResourcePaths) { addResourceEntry(resourceEntry, loader); } - for (String name : loaderData.getNames()) { + for(String name:loaderData.myNames) { addNameEntry(name, loader); } - } - finally { + } finally { myLock.writeLock().unlock(); } } @@ -360,7 +373,7 @@ public class ClasspathCache { private static int hashFromNameAndLoader(String name, Loader loader, int n) { int hash = StringHash.murmur(name, n); - int i = loader.getOrderNumber(); + int i = loader.getIndex(); while (i > 0) { hash = hash * n + ((i % 10) + '0'); i /= 10; @@ -403,7 +416,7 @@ public class ClasspathCache { protected static int hashFromNameAndLoader(String name, Loader loader) { int hash = name.hashCode(); - int i = loader.getOrderNumber(); + int i = loader.getIndex(); while(i > 0) { hash = hash * 31 + ((i % 10) + '0'); i /= 10; diff --git a/platform/util/src/com/intellij/util/lang/ClasspathLoaderIndex.java b/platform/util/src/com/intellij/util/lang/ClasspathLoaderIndex.java deleted file mode 100644 index acf31aa33261..000000000000 --- a/platform/util/src/com/intellij/util/lang/ClasspathLoaderIndex.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2000-2014 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.intellij.util.lang; - -import java.util.ArrayList; -import java.util.List; - -/** - * Holds resource and class names that are present inside a jar or a directory represented by a particular {@link Loader} instance. - * This data is then used by {@link UrlClassLoader} to avoid looking inside particular loaders when searching for classes/resources, and thus - * to speed up the operation.

- * - * @see LoaderIndexProvider - * - * @author peter -*/ -public class ClasspathLoaderIndex { - private final List myResourcePaths = new ArrayList(); - private final List myNames = new ArrayList(); - - public void addResourceEntry(String resourcePath) { - myResourcePaths.add(resourcePath); - } - - public void addNameEntry(String name) { - myNames.add(ClasspathCache.transformName(name)); - } - - public List getResourcePaths() { - return myResourcePaths; - } - - public List getNames() { - return myNames; - } -} diff --git a/platform/util/src/com/intellij/util/lang/FileLoader.java b/platform/util/src/com/intellij/util/lang/FileLoader.java index 151eb57ec391..f072acd480f4 100644 --- a/platform/util/src/com/intellij/util/lang/FileLoader.java +++ b/platform/util/src/com/intellij/util/lang/FileLoader.java @@ -17,6 +17,7 @@ package com.intellij.util.lang; import com.intellij.openapi.util.io.FileUtil; import org.jetbrains.annotations.NonNls; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import sun.misc.Resource; @@ -35,7 +36,7 @@ class FileLoader extends Loader { myRootDirAbsolutePath = myRootDir.getAbsolutePath(); } - private void buildPackageCache(final File dir, ClasspathLoaderIndex loaderData) { + private void buildPackageCache(final File dir, ClasspathCache.LoaderData loaderData) { loaderData.addResourceEntry(getRelativeResourcePath(dir)); final File[] files = dir.listFiles(); @@ -109,9 +110,10 @@ class FileLoader extends Loader { return null; } + @NotNull @Override - public ClasspathLoaderIndex buildIndex() throws IOException { - ClasspathLoaderIndex loaderData = new ClasspathLoaderIndex(); + public ClasspathCache.LoaderData buildData() throws IOException { + ClasspathCache.LoaderData loaderData = new ClasspathCache.LoaderData(); File index = new File(myRootDir, "classpath.index"); if (index.exists()) { BufferedReader reader = new BufferedReader(new FileReader(index)); diff --git a/platform/util/src/com/intellij/util/lang/JarLoader.java b/platform/util/src/com/intellij/util/lang/JarLoader.java index 9c59b3bcb07a..9c1fa395e6af 100644 --- a/platform/util/src/com/intellij/util/lang/JarLoader.java +++ b/platform/util/src/com/intellij/util/lang/JarLoader.java @@ -19,6 +19,7 @@ import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.util.io.FileUtil; import com.intellij.openapi.util.io.ZipFileCache; import com.intellij.util.io.URLUtil; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import sun.misc.Resource; @@ -68,9 +69,10 @@ class JarLoader extends Loader { } } + @NotNull @Override - public ClasspathLoaderIndex buildIndex() throws IOException { - ClasspathLoaderIndex loaderData = new ClasspathLoaderIndex(); + public ClasspathCache.LoaderData buildData() throws IOException { + ClasspathCache.LoaderData loaderData = new ClasspathCache.LoaderData(); ZipFile zipFile = acquireZipFile(); try { Enumeration entries = zipFile.entries(); diff --git a/platform/util/src/com/intellij/util/lang/Loader.java b/platform/util/src/com/intellij/util/lang/Loader.java index e0424ac3b462..5b73c04f429a 100644 --- a/platform/util/src/com/intellij/util/lang/Loader.java +++ b/platform/util/src/com/intellij/util/lang/Loader.java @@ -15,6 +15,7 @@ */ package com.intellij.util.lang; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import sun.misc.Resource; @@ -27,25 +28,25 @@ import java.net.URL; * @see JarLoader * @see FileLoader */ -public abstract class Loader { +abstract class Loader { private final URL myURL; - private final int myOrderNumber; + private final int myIndex; - Loader(URL url, int orderNumber) { + Loader(URL url, int index) { myURL = url; - myOrderNumber = orderNumber; + myIndex = index; } - public URL getBaseURL() { + URL getBaseURL() { return myURL; } @Nullable abstract Resource getResource(String name, boolean flag); - public abstract ClasspathLoaderIndex buildIndex() throws IOException; + @NotNull abstract ClasspathCache.LoaderData buildData() throws IOException; - int getOrderNumber() { - return myOrderNumber; + int getIndex() { + return myIndex; } } diff --git a/platform/util/src/com/intellij/util/lang/LoaderIndexProvider.java b/platform/util/src/com/intellij/util/lang/LoaderIndexProvider.java deleted file mode 100644 index 3d342eb787e0..000000000000 --- a/platform/util/src/com/intellij/util/lang/LoaderIndexProvider.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2000-2014 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.intellij.util.lang; - -import org.jetbrains.annotations.NotNull; - -import java.io.IOException; - -/** - * A way to plug into {@link UrlClassLoader} indexing policy. Can be used to reuse the index in several classloaders - * where particular class path elements are the same. - * - * @see UrlClassLoader.Builder#useCache(LoaderIndexProvider) - * - * @author peter - */ -public abstract class LoaderIndexProvider { - /** - * The default implementation which just calls returns {@link Loader#buildIndex()} result. - */ - public static final LoaderIndexProvider DEFAULT_INSTANCE = new LoaderIndexProvider() { - @NotNull - @Override - public ClasspathLoaderIndex getLoaderData(@NotNull Loader loader) throws IOException { - return loader.buildIndex(); - } - }; - - /** - * @param loader the jar or a file loader to build the index for. - * @return a (maybe cached) index for this loader, obtained using {@link Loader#buildIndex()} - * @throws IOException - */ - @NotNull - public abstract ClasspathLoaderIndex getLoaderData(@NotNull Loader loader) throws IOException; - -} - diff --git a/platform/util/src/com/intellij/util/lang/UrlClassLoader.java b/platform/util/src/com/intellij/util/lang/UrlClassLoader.java index f83d2ddd5663..7661baa44cb5 100644 --- a/platform/util/src/com/intellij/util/lang/UrlClassLoader.java +++ b/platform/util/src/com/intellij/util/lang/UrlClassLoader.java @@ -67,7 +67,8 @@ public class UrlClassLoader extends ClassLoader { private boolean myAcceptUnescaped = false; private boolean myPreload = true; private boolean myAllowBootstrapResources = false; - @NotNull private LoaderIndexProvider myIndexProvider = LoaderIndexProvider.DEFAULT_INSTANCE; + @Nullable private CachePoolImpl myCachePool = null; + @Nullable private CachingCondition myCachingCondition = null; private Builder() { } @@ -80,13 +81,22 @@ public class UrlClassLoader extends ClassLoader { public Builder useCache(boolean useCache) { myUseCache = useCache; return this; } /** - * Requests the class loader being built to use cache and to obtain the necessary classpath index from a custom index provider. + * Requests the class loader being built to use cache and, if possible, retrieve and store the cached data from a special cache pool + * that can be shared between several loaders. + + * @param pool cache pool + * @param condition a custom policy to provide a possibility to prohibit caching for some URLs. + * @return this instance * - * @param provider a custom index provider - * @return the same instance - * @see LoaderIndexProvider + * @see #createCachePool() */ - public Builder useCache(@NotNull LoaderIndexProvider provider) { myUseCache = true; myIndexProvider = provider; return this; } + public Builder useCache(@NotNull CachePool pool, @NotNull CachingCondition condition) { + myUseCache = true; + myCachePool = (CachePoolImpl)pool; + myCachingCondition = condition; + return this; + } + public Builder allowUnescaped() { myAcceptUnescaped = true; return this; } public Builder noPreload() { myPreload = false; return this; } public Builder allowBootstrapResources() { myAllowBootstrapResources = true; return this; } @@ -131,7 +141,7 @@ public class UrlClassLoader extends ClassLoader { return internProtocol(url); } }); - myClassPath = new ClassPath(myURLs, lockJars, useCache, allowUnescaped, preload, LoaderIndexProvider.DEFAULT_INSTANCE); + myClassPath = new ClassPath(myURLs, lockJars, useCache, allowUnescaped, preload, null, null); myAllowBootstrapResources = false; } @@ -143,7 +153,7 @@ public class UrlClassLoader extends ClassLoader { return internProtocol(url); } }); - myClassPath = new ClassPath(myURLs, builder.myLockJars, builder.myUseCache, builder.myAcceptUnescaped, builder.myPreload, builder.myIndexProvider); + myClassPath = new ClassPath(myURLs, builder.myLockJars, builder.myUseCache, builder.myAcceptUnescaped, builder.myPreload, builder.myCachePool, builder.myCachingCondition); myAllowBootstrapResources = builder.myAllowBootstrapResources; } @@ -301,4 +311,40 @@ public class UrlClassLoader extends ClassLoader { else if (SystemInfo.isLinux) return "linux/"; else return ""; } + + /** + * An interface for a pool to store internal class loader caches, that can be shared between several different class loaders, + * if they contain the same URLs in their classpaths.

+ * + * The implementation is subject to change so one shouldn't rely on it. + * + * @see #createCachePool() + * @see Builder#useCache(CachePool, CachingCondition) + */ + public interface CachePool {} + + /** + * A condition to customize the caching policy when using {@link CachePool}. This might be needed when a class loader is used on a directory + * that's being written into, to avoid the situation when a resource path is cached as nonexistent but then a file actually appears there, + * and other class loaders with the same caching pool should have access to these new resources. This can happen during compilation process + * with several module outputs. + */ + public interface CachingCondition { + + /** + * @param url + * @return whether the internal information should be cached for files in a specific classpath component URL: inside the directory or + * a jar. + */ + boolean shouldCacheData(@NotNull URL url); + } + + /** + * @return a new pool to be able to share internal class loader caches between several different class loaders, if they contain the same URLs + * in their classpaths. + */ + @NotNull + public static CachePool createCachePool() { + return new CachePoolImpl(); + } } diff --git a/plugins/groovy/jps-plugin/src/org/jetbrains/jps/incremental/groovy/InProcessGroovyc.java b/plugins/groovy/jps-plugin/src/org/jetbrains/jps/incremental/groovy/InProcessGroovyc.java index ac3a9426bdcb..2070f2638501 100644 --- a/plugins/groovy/jps-plugin/src/org/jetbrains/jps/incremental/groovy/InProcessGroovyc.java +++ b/plugins/groovy/jps-plugin/src/org/jetbrains/jps/incremental/groovy/InProcessGroovyc.java @@ -35,7 +35,6 @@ import java.net.URL; import java.util.Arrays; import java.util.Collection; import java.util.List; -import java.util.Map; import java.util.regex.Pattern; /** @@ -44,7 +43,7 @@ import java.util.regex.Pattern; class InProcessGroovyc { private static final Pattern GROOVY_ALL_JAR_PATTERN = Pattern.compile("groovy-all(-(.*))?\\.jar"); private static SoftReference> ourParentLoaderCache; - private static final Map ourLoaderIndexCache = ContainerUtil.newConcurrentMap(); + private static final UrlClassLoader.CachePool ourLoaderCachePool = UrlClassLoader.createCachePool(); @SuppressWarnings("UseOfSystemOutOrSystemErr") static void runGroovycInThisProcess(Collection compilationClassPath, @@ -55,29 +54,20 @@ class InProcessGroovyc { ClassLoader parent = obtainParentLoader(compilationClassPath); - UrlClassLoader loader = UrlClassLoader.build().urls(toUrls(compilationClassPath)).parent(parent).useCache(new LoaderIndexProvider() { - @NotNull - @Override - public ClasspathLoaderIndex getLoaderData(@NotNull Loader loader) throws IOException { - URL url = loader.getBaseURL(); - ClasspathLoaderIndex index = ourLoaderIndexCache.get(url); - if (index != null) { - return index; - } - - index = loader.buildIndex(); - String file = url.getFile(); - for (String output : outputs) { - if (FileUtil.startsWith(output + "/", file)) { - return index; // don't cache outputs + UrlClassLoader loader = UrlClassLoader.build(). + urls(toUrls(compilationClassPath)).parent(parent). + useCache(ourLoaderCachePool, new UrlClassLoader.CachingCondition() { + @Override + public boolean shouldCacheData(@NotNull URL url) { + String file = url.getFile(); + for (String output : outputs) { + if (FileUtil.startsWith(output + "/", file)) { + return false; + } } + return true; } - - ourLoaderIndexCache.put(url, index); - return index; - - } - }).get(); + }).get(); PrintStream oldOut = System.out; PrintStream oldErr = System.err;