diff --git a/platform/util/resources/misc/registry.properties b/platform/util/resources/misc/registry.properties index da150300c280..91a71a86e023 100644 --- a/platform/util/resources/misc/registry.properties +++ b/platform/util/resources/misc/registry.properties @@ -1721,3 +1721,5 @@ new.search.everywhere.single.thread.search.description=Disable multi-thread sear autodetect.all.jdks=true autodetect.all.jdks.description=Scan disk and show all found JDKs in "Select Project SDK" dialog +idea.lazy.classloading.caches=false +idea.lazy.classloading.caches.description=Flag for UrlClassLoader to use lazy caching of class locations \ No newline at end of file diff --git a/platform/util/src/com/intellij/util/lang/ClassPath.java b/platform/util/src/com/intellij/util/lang/ClassPath.java index e2aa8223edaa..3d73074754de 100644 --- a/platform/util/src/com/intellij/util/lang/ClassPath.java +++ b/platform/util/src/com/intellij/util/lang/ClassPath.java @@ -54,6 +54,7 @@ public class ClassPath { private final boolean myAcceptUnescapedUrls; final boolean myPreloadJarContents; final boolean myCanHavePersistentIndex; + final boolean myLazyClassloadingCaches; @Nullable private final CachePoolImpl myCachePool; @Nullable private final UrlClassLoader.CachingCondition myCachingCondition; final boolean myLogErrorOnMissingJar; @@ -66,9 +67,12 @@ public class ClassPath { boolean canHavePersistentIndex, @Nullable CachePoolImpl cachePool, @Nullable UrlClassLoader.CachingCondition cachingCondition, - boolean logErrorOnMissingJar) { + boolean logErrorOnMissingJar, + boolean lazyClassloadingCaches + ) { + myLazyClassloadingCaches = lazyClassloadingCaches; myCanLockJars = canLockJars; - myCanUseCache = canUseCache; + myCanUseCache = canUseCache && !myLazyClassloadingCaches; myAcceptUnescapedUrls = acceptUnescapedUrls; myPreloadJarContents = preloadJarContents; myCachePool = cachePool; @@ -208,6 +212,7 @@ public class ClassPath { if (processRecursively) { String[] referencedJars = loadManifestClasspath(loader); if (referencedJars != null) { + //long s2 = System.nanoTime(); for (String referencedJar : referencedJars) { try { URI uri = new URI(referencedJar); @@ -222,6 +227,7 @@ public class ClassPath { Logger.getInstance(ClassPath.class).warn("url: " + url + " / " + referencedJar, e); } } + //System.out.println("Loaded all " + referencedJars.length + " urls " + (System.nanoTime() - s2) / 1000000 + "ms"); } } return loader; @@ -429,10 +435,20 @@ public class ClassPath { if (time > 10000000L) { System.out.println(time / 1000000 + " ms for " + msg); } - if (ourTotalRequests % 1000 == 0) { + if (ourTotalRequests % 10000 == 0) { System.out.println(path + ", requests:" + ourTotalRequests + ", time:" + (ourTotalTime / 1000000) + "ms"); } } + static { + if (ourLogTiming) { + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + System.out.println("Classloading requests:" + ourTotalRequests + ", time:" + (ourTotalTime / 1000000) + "ms"); + } + }); + } + } private static String[] loadManifestClasspath(JarLoader loader) { try { diff --git a/platform/util/src/com/intellij/util/lang/FileLoader.java b/platform/util/src/com/intellij/util/lang/FileLoader.java index ef46ffdd35a7..404f21ccb04b 100644 --- a/platform/util/src/com/intellij/util/lang/FileLoader.java +++ b/platform/util/src/com/intellij/util/lang/FileLoader.java @@ -17,6 +17,7 @@ package com.intellij.util.lang; import com.intellij.openapi.util.io.FileUtil; import com.intellij.openapi.util.text.StringUtil; +import com.intellij.util.SmartList; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -74,10 +75,91 @@ class FileLoader extends Loader { return relativePath; } + private static class DirEntry { + static final int[] empty = new int[0]; + volatile int[] childrenNameHashes; + + volatile List childrenDirectories; + final int nameHash; + final String name; + + DirEntry(int _nameHash, String _name) { + nameHash = _nameHash; + name = _name; + } + } + + private final DirEntry root = new DirEntry(0, null); + @Override @Nullable Resource getResource(final String name) { try { + if (myConfiguration.myLazyClassloadingCaches) { + DirEntry lastEntry = root; + int prevIndex = 0; + int nextIndex = name.indexOf('/', prevIndex); + + while (true) { + int nameEnd = nextIndex == -1 ? name.length() : nextIndex; + int nameHash = StringUtil.stringHashCodeInsensitive(name, prevIndex, nameEnd); + int[] childrenNameHashes = lastEntry.childrenNameHashes; // volatile read + + if (childrenNameHashes == null) { + String[] list = (prevIndex != 0 ? new File(myRootDir, name.substring(0, prevIndex)) : myRootDir).list(); + + if (list != null) { + childrenNameHashes = new int[list.length]; + for (int i = 0; i < list.length; ++i) { + childrenNameHashes[i] = StringUtil.stringHashCodeInsensitive(list[i]); + } + } + else { + childrenNameHashes = DirEntry.empty; + } + lastEntry.childrenNameHashes = childrenNameHashes; // volatile write + } + + boolean found = false; + for (int childNameHash : childrenNameHashes) { + if (childNameHash == nameHash) { + found = true; + break; + } + } + + if (!found) { + return null; + } + if (nextIndex == -1 || nextIndex == name.length() - 1) { + break; + } + + DirEntry nextEntry = null; + List directories = lastEntry.childrenDirectories; // volatile read + + if (directories != null) { + for (DirEntry previouslyScannedDir : directories) { + if (previouslyScannedDir.nameHash == nameHash && previouslyScannedDir.name.regionMatches(0, name, prevIndex, nameEnd - prevIndex)) { + nextEntry = previouslyScannedDir; + break; + } + } + } + + if (nextEntry == null) { + nextEntry = new DirEntry(nameHash, name.substring(prevIndex, nameEnd)); + List newChildrenDirectories = directories != null ? new SmartList(directories) : new SmartList(); + newChildrenDirectories.add(nextEntry); + lastEntry.childrenDirectories = newChildrenDirectories; // volatile write with new copy of data + } + + lastEntry = nextEntry; + prevIndex = nextIndex + 1; + nextIndex = name.indexOf('/', prevIndex); + } + } + URL url = new URL(getBaseURL(), name); if (!url.getFile().startsWith(getBaseURL().getFile())) return null; File file = new File(myRootDir, name.replace('/', File.separatorChar)); diff --git a/platform/util/src/com/intellij/util/lang/JarLoader.java b/platform/util/src/com/intellij/util/lang/JarLoader.java index b599624bc9f2..753ae1707712 100644 --- a/platform/util/src/com/intellij/util/lang/JarLoader.java +++ b/platform/util/src/com/intellij/util/lang/JarLoader.java @@ -59,16 +59,19 @@ class JarLoader extends Loader { myConfiguration = configuration; myUrl = url; - ZipFile zipFile = getZipFile(); // IOException from opening is propagated to caller if zip file isn't valid, - try { - if (configuration.myPreloadJarContents) { - JarMemoryLoader loader = JarMemoryLoader.load(zipFile, getBaseURL(), this); - if (loader != null) { - myMemoryLoader = new SoftReference(loader); + if (!configuration.myLazyClassloadingCaches) { + ZipFile zipFile = getZipFile(); // IOException from opening is propagated to caller if zip file isn't valid, + try { + if (configuration.myPreloadJarContents) { + JarMemoryLoader loader = JarMemoryLoader.load(zipFile, getBaseURL(), this); + if (loader != null) { + myMemoryLoader = new SoftReference(loader); + } } } - } finally { - releaseZipFile(zipFile); + finally { + releaseZipFile(zipFile); + } } } diff --git a/platform/util/src/com/intellij/util/lang/UrlClassLoader.java b/platform/util/src/com/intellij/util/lang/UrlClassLoader.java index add17ca154d6..745e32bf3c5c 100644 --- a/platform/util/src/com/intellij/util/lang/UrlClassLoader.java +++ b/platform/util/src/com/intellij/util/lang/UrlClassLoader.java @@ -93,6 +93,7 @@ public class UrlClassLoader extends ClassLoader { private boolean myPreload = true; private boolean myAllowBootstrapResources; private boolean myErrorOnMissingJar = true; + private boolean myLazyClassloadingCaches = SystemProperties.getBooleanProperty("idea.lazy.classloading.caches", false); @Nullable private CachePoolImpl myCachePool; @Nullable private CachingCondition myCachingCondition; @@ -101,8 +102,14 @@ public class UrlClassLoader extends ClassLoader { public Builder urls(List urls) { myURLs = urls; return this; } public Builder urls(URL... urls) { myURLs = Arrays.asList(urls); return this; } public Builder parent(ClassLoader parent) { myParent = parent; return this; } + + // Keep ZipFile handle opened in SoftReference. Depending on OS, the option significantly speeds up classloading from libraries. + // Caveat: for Windows opened handle will lock the file preventing its modification + // Thus, the option is recommended when jars are not modified or process that uses this option is transient public Builder allowLock() { myLockJars = true; return this; } public Builder allowLock(boolean lockJars) { myLockJars = lockJars; return this; } + + // Build index of packages / class or resource names that allows to avoid IO during classloading public Builder useCache() { myUseCache = true; return this; } public Builder useCache(boolean useCache) { myUseCache = useCache; return this; } @@ -139,6 +146,10 @@ public class UrlClassLoader extends ClassLoader { public Builder noPreload() { myPreload = false; return this; } public Builder allowBootstrapResources() { myAllowBootstrapResources = true; return this; } public Builder setLogErrorOnMissingJar(boolean log) {myErrorOnMissingJar = log; return this; } + + // Cache package contents information in FileLoader lazily upon classloading. + // This is less efficient than useCache / usePersistentClasspathIndexForLocalClassDirectories but the option has much smaller startup overhead. + public Builder useLazyClassloadingCaches(boolean pleaseBeLazy) { myLazyClassloadingCaches = pleaseBeLazy; return this; } public UrlClassLoader get() { return new UrlClassLoader(this); } } @@ -176,7 +187,7 @@ public class UrlClassLoader extends ClassLoader { protected final ClassPath createClassPath(@NotNull Builder builder) { return new ClassPath(myURLs, builder.myLockJars, builder.myUseCache, builder.myAcceptUnescaped, builder.myPreload, builder.myUsePersistentClasspathIndex, builder.myCachePool, builder.myCachingCondition, - builder.myErrorOnMissingJar); + builder.myErrorOnMissingJar, builder.myLazyClassloadingCaches); } public static URL internProtocol(@NotNull URL url) { diff --git a/platform/util/testSrc/com/intellij/util/lang/UrlClassLoaderTest.java b/platform/util/testSrc/com/intellij/util/lang/UrlClassLoaderTest.java index 48e7e561e366..eeb7618082eb 100644 --- a/platform/util/testSrc/com/intellij/util/lang/UrlClassLoaderTest.java +++ b/platform/util/testSrc/com/intellij/util/lang/UrlClassLoaderTest.java @@ -148,14 +148,14 @@ public class UrlClassLoaderTest { File theGood = createTestJar(createTestFile(sadHill, "1_normal.jar"), entryName, "-"); File theBad = createTestFile(sadHill, "2_broken.jar", new String(new char[1024])); - UrlClassLoader flat = UrlClassLoader.build().urls(theBad.toURI().toURL(), theGood.toURI().toURL()).get(); + UrlClassLoader flat = UrlClassLoader.build().urls(theBad.toURI().toURL(), theGood.toURI().toURL()).useLazyClassloadingCaches(false).get(); assertNotNull(findResource(flat, entryName, false)); String content = Attributes.Name.MANIFEST_VERSION + ": 1.0\n" + Attributes.Name.CLASS_PATH + ": " + theBad.toURI().toURL() + " " + theGood.toURI().toURL() + "\n\n"; File theUgly = createTestJar(createTestFile(sadHill, "3_classpath.jar"), JarFile.MANIFEST_NAME, content); - UrlClassLoader recursive = UrlClassLoader.build().urls(theUgly.toURI().toURL()).get(); + UrlClassLoader recursive = UrlClassLoader.build().urls(theUgly.toURI().toURL()).useLazyClassloadingCaches(false).get(); assertNotNull(findResource(recursive, entryName, false)); } finally { diff --git a/plugins/devkit/devkit-core/src/run/JUnitDevKitPatcher.java b/plugins/devkit/devkit-core/src/run/JUnitDevKitPatcher.java index 277692d13fc0..a7e043841ec5 100644 --- a/plugins/devkit/devkit-core/src/run/JUnitDevKitPatcher.java +++ b/plugins/devkit/devkit-core/src/run/JUnitDevKitPatcher.java @@ -25,6 +25,7 @@ import com.intellij.openapi.project.DumbService; import com.intellij.openapi.project.Project; import com.intellij.openapi.projectRoots.*; import com.intellij.openapi.util.io.FileUtil; +import com.intellij.openapi.util.registry.Registry; import com.intellij.openapi.util.text.StringUtil; import com.intellij.psi.JavaPsiFacade; import com.intellij.psi.PsiClass; @@ -65,6 +66,12 @@ public class JUnitDevKitPatcher extends JUnitPatcher { vm.addProperty(SYSTEM_CL_PROPERTY, qualifiedName); } } + + if (Registry.is("idea.lazy.classloading.caches") && + vm.hasProperty(SYSTEM_CL_PROPERTY) && + UrlClassLoader.class.getName().equals(vm.getPropertyValue(SYSTEM_CL_PROPERTY))) { + vm.addProperty("idea.lazy.classloading.caches", "true"); + } jdk = IdeaJdk.findIdeaJdk(jdk); if (jdk == null) return;