From e1df0c050bb1b01b31fcf8ec2dd481e8be23365c Mon Sep 17 00:00:00 2001 From: "Maxim.Mossienko" Date: Mon, 17 Sep 2018 16:42:54 +0200 Subject: [PATCH] test launching performance improvement within IDE project For unit tests' run avoid eager building index of package / class or resource names. Instead lazily cache package locations that were tried during class loading (for FileLoader) and do not cache anything for JarLoader. Upon opening java.util.ZipFile contains hashed file name index in native code so .getEntry() works good for uncontended minimal setup scenario. --- .../util/resources/misc/registry.properties | 2 + .../src/com/intellij/util/lang/ClassPath.java | 22 ++++- .../com/intellij/util/lang/FileLoader.java | 82 +++++++++++++++++++ .../src/com/intellij/util/lang/JarLoader.java | 19 +++-- .../intellij/util/lang/UrlClassLoader.java | 13 ++- .../util/lang/UrlClassLoaderTest.java | 4 +- .../src/run/JUnitDevKitPatcher.java | 7 ++ 7 files changed, 135 insertions(+), 14 deletions(-) diff --git a/platform/util/resources/misc/registry.properties b/platform/util/resources/misc/registry.properties index da150300c280..91a71a86e023 100644 --- a/platform/util/resources/misc/registry.properties +++ b/platform/util/resources/misc/registry.properties @@ -1721,3 +1721,5 @@ new.search.everywhere.single.thread.search.description=Disable multi-thread sear autodetect.all.jdks=true autodetect.all.jdks.description=Scan disk and show all found JDKs in "Select Project SDK" dialog +idea.lazy.classloading.caches=false +idea.lazy.classloading.caches.description=Flag for UrlClassLoader to use lazy caching of class locations \ No newline at end of file diff --git a/platform/util/src/com/intellij/util/lang/ClassPath.java b/platform/util/src/com/intellij/util/lang/ClassPath.java index e2aa8223edaa..3d73074754de 100644 --- a/platform/util/src/com/intellij/util/lang/ClassPath.java +++ b/platform/util/src/com/intellij/util/lang/ClassPath.java @@ -54,6 +54,7 @@ public class ClassPath { private final boolean myAcceptUnescapedUrls; final boolean myPreloadJarContents; final boolean myCanHavePersistentIndex; + final boolean myLazyClassloadingCaches; @Nullable private final CachePoolImpl myCachePool; @Nullable private final UrlClassLoader.CachingCondition myCachingCondition; final boolean myLogErrorOnMissingJar; @@ -66,9 +67,12 @@ public class ClassPath { boolean canHavePersistentIndex, @Nullable CachePoolImpl cachePool, @Nullable UrlClassLoader.CachingCondition cachingCondition, - boolean logErrorOnMissingJar) { + boolean logErrorOnMissingJar, + boolean lazyClassloadingCaches + ) { + myLazyClassloadingCaches = lazyClassloadingCaches; myCanLockJars = canLockJars; - myCanUseCache = canUseCache; + myCanUseCache = canUseCache && !myLazyClassloadingCaches; myAcceptUnescapedUrls = acceptUnescapedUrls; myPreloadJarContents = preloadJarContents; myCachePool = cachePool; @@ -208,6 +212,7 @@ public class ClassPath { if (processRecursively) { String[] referencedJars = loadManifestClasspath(loader); if (referencedJars != null) { + //long s2 = System.nanoTime(); for (String referencedJar : referencedJars) { try { URI uri = new URI(referencedJar); @@ -222,6 +227,7 @@ public class ClassPath { Logger.getInstance(ClassPath.class).warn("url: " + url + " / " + referencedJar, e); } } + //System.out.println("Loaded all " + referencedJars.length + " urls " + (System.nanoTime() - s2) / 1000000 + "ms"); } } return loader; @@ -429,10 +435,20 @@ public class ClassPath { if (time > 10000000L) { System.out.println(time / 1000000 + " ms for " + msg); } - if (ourTotalRequests % 1000 == 0) { + if (ourTotalRequests % 10000 == 0) { System.out.println(path + ", requests:" + ourTotalRequests + ", time:" + (ourTotalTime / 1000000) + "ms"); } } + static { + if (ourLogTiming) { + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + System.out.println("Classloading requests:" + ourTotalRequests + ", time:" + (ourTotalTime / 1000000) + "ms"); + } + }); + } + } private static String[] loadManifestClasspath(JarLoader loader) { try { diff --git a/platform/util/src/com/intellij/util/lang/FileLoader.java b/platform/util/src/com/intellij/util/lang/FileLoader.java index ef46ffdd35a7..404f21ccb04b 100644 --- a/platform/util/src/com/intellij/util/lang/FileLoader.java +++ b/platform/util/src/com/intellij/util/lang/FileLoader.java @@ -17,6 +17,7 @@ package com.intellij.util.lang; import com.intellij.openapi.util.io.FileUtil; import com.intellij.openapi.util.text.StringUtil; +import com.intellij.util.SmartList; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -74,10 +75,91 @@ class FileLoader extends Loader { return relativePath; } + private static class DirEntry { + static final int[] empty = new int[0]; + volatile int[] childrenNameHashes; + + volatile List childrenDirectories; + final int nameHash; + final String name; + + DirEntry(int _nameHash, String _name) { + nameHash = _nameHash; + name = _name; + } + } + + private final DirEntry root = new DirEntry(0, null); + @Override @Nullable Resource getResource(final String name) { try { + if (myConfiguration.myLazyClassloadingCaches) { + DirEntry lastEntry = root; + int prevIndex = 0; + int nextIndex = name.indexOf('/', prevIndex); + + while (true) { + int nameEnd = nextIndex == -1 ? name.length() : nextIndex; + int nameHash = StringUtil.stringHashCodeInsensitive(name, prevIndex, nameEnd); + int[] childrenNameHashes = lastEntry.childrenNameHashes; // volatile read + + if (childrenNameHashes == null) { + String[] list = (prevIndex != 0 ? new File(myRootDir, name.substring(0, prevIndex)) : myRootDir).list(); + + if (list != null) { + childrenNameHashes = new int[list.length]; + for (int i = 0; i < list.length; ++i) { + childrenNameHashes[i] = StringUtil.stringHashCodeInsensitive(list[i]); + } + } + else { + childrenNameHashes = DirEntry.empty; + } + lastEntry.childrenNameHashes = childrenNameHashes; // volatile write + } + + boolean found = false; + for (int childNameHash : childrenNameHashes) { + if (childNameHash == nameHash) { + found = true; + break; + } + } + + if (!found) { + return null; + } + if (nextIndex == -1 || nextIndex == name.length() - 1) { + break; + } + + DirEntry nextEntry = null; + List directories = lastEntry.childrenDirectories; // volatile read + + if (directories != null) { + for (DirEntry previouslyScannedDir : directories) { + if (previouslyScannedDir.nameHash == nameHash && previouslyScannedDir.name.regionMatches(0, name, prevIndex, nameEnd - prevIndex)) { + nextEntry = previouslyScannedDir; + break; + } + } + } + + if (nextEntry == null) { + nextEntry = new DirEntry(nameHash, name.substring(prevIndex, nameEnd)); + List newChildrenDirectories = directories != null ? new SmartList(directories) : new SmartList(); + newChildrenDirectories.add(nextEntry); + lastEntry.childrenDirectories = newChildrenDirectories; // volatile write with new copy of data + } + + lastEntry = nextEntry; + prevIndex = nextIndex + 1; + nextIndex = name.indexOf('/', prevIndex); + } + } + URL url = new URL(getBaseURL(), name); if (!url.getFile().startsWith(getBaseURL().getFile())) return null; File file = new File(myRootDir, name.replace('/', File.separatorChar)); diff --git a/platform/util/src/com/intellij/util/lang/JarLoader.java b/platform/util/src/com/intellij/util/lang/JarLoader.java index b599624bc9f2..753ae1707712 100644 --- a/platform/util/src/com/intellij/util/lang/JarLoader.java +++ b/platform/util/src/com/intellij/util/lang/JarLoader.java @@ -59,16 +59,19 @@ class JarLoader extends Loader { myConfiguration = configuration; myUrl = url; - ZipFile zipFile = getZipFile(); // IOException from opening is propagated to caller if zip file isn't valid, - try { - if (configuration.myPreloadJarContents) { - JarMemoryLoader loader = JarMemoryLoader.load(zipFile, getBaseURL(), this); - if (loader != null) { - myMemoryLoader = new SoftReference(loader); + if (!configuration.myLazyClassloadingCaches) { + ZipFile zipFile = getZipFile(); // IOException from opening is propagated to caller if zip file isn't valid, + try { + if (configuration.myPreloadJarContents) { + JarMemoryLoader loader = JarMemoryLoader.load(zipFile, getBaseURL(), this); + if (loader != null) { + myMemoryLoader = new SoftReference(loader); + } } } - } finally { - releaseZipFile(zipFile); + finally { + releaseZipFile(zipFile); + } } } diff --git a/platform/util/src/com/intellij/util/lang/UrlClassLoader.java b/platform/util/src/com/intellij/util/lang/UrlClassLoader.java index add17ca154d6..745e32bf3c5c 100644 --- a/platform/util/src/com/intellij/util/lang/UrlClassLoader.java +++ b/platform/util/src/com/intellij/util/lang/UrlClassLoader.java @@ -93,6 +93,7 @@ public class UrlClassLoader extends ClassLoader { private boolean myPreload = true; private boolean myAllowBootstrapResources; private boolean myErrorOnMissingJar = true; + private boolean myLazyClassloadingCaches = SystemProperties.getBooleanProperty("idea.lazy.classloading.caches", false); @Nullable private CachePoolImpl myCachePool; @Nullable private CachingCondition myCachingCondition; @@ -101,8 +102,14 @@ public class UrlClassLoader extends ClassLoader { public Builder urls(List urls) { myURLs = urls; return this; } public Builder urls(URL... urls) { myURLs = Arrays.asList(urls); return this; } public Builder parent(ClassLoader parent) { myParent = parent; return this; } + + // Keep ZipFile handle opened in SoftReference. Depending on OS, the option significantly speeds up classloading from libraries. + // Caveat: for Windows opened handle will lock the file preventing its modification + // Thus, the option is recommended when jars are not modified or process that uses this option is transient public Builder allowLock() { myLockJars = true; return this; } public Builder allowLock(boolean lockJars) { myLockJars = lockJars; return this; } + + // Build index of packages / class or resource names that allows to avoid IO during classloading public Builder useCache() { myUseCache = true; return this; } public Builder useCache(boolean useCache) { myUseCache = useCache; return this; } @@ -139,6 +146,10 @@ public class UrlClassLoader extends ClassLoader { public Builder noPreload() { myPreload = false; return this; } public Builder allowBootstrapResources() { myAllowBootstrapResources = true; return this; } public Builder setLogErrorOnMissingJar(boolean log) {myErrorOnMissingJar = log; return this; } + + // Cache package contents information in FileLoader lazily upon classloading. + // This is less efficient than useCache / usePersistentClasspathIndexForLocalClassDirectories but the option has much smaller startup overhead. + public Builder useLazyClassloadingCaches(boolean pleaseBeLazy) { myLazyClassloadingCaches = pleaseBeLazy; return this; } public UrlClassLoader get() { return new UrlClassLoader(this); } } @@ -176,7 +187,7 @@ public class UrlClassLoader extends ClassLoader { protected final ClassPath createClassPath(@NotNull Builder builder) { return new ClassPath(myURLs, builder.myLockJars, builder.myUseCache, builder.myAcceptUnescaped, builder.myPreload, builder.myUsePersistentClasspathIndex, builder.myCachePool, builder.myCachingCondition, - builder.myErrorOnMissingJar); + builder.myErrorOnMissingJar, builder.myLazyClassloadingCaches); } public static URL internProtocol(@NotNull URL url) { diff --git a/platform/util/testSrc/com/intellij/util/lang/UrlClassLoaderTest.java b/platform/util/testSrc/com/intellij/util/lang/UrlClassLoaderTest.java index 48e7e561e366..eeb7618082eb 100644 --- a/platform/util/testSrc/com/intellij/util/lang/UrlClassLoaderTest.java +++ b/platform/util/testSrc/com/intellij/util/lang/UrlClassLoaderTest.java @@ -148,14 +148,14 @@ public class UrlClassLoaderTest { File theGood = createTestJar(createTestFile(sadHill, "1_normal.jar"), entryName, "-"); File theBad = createTestFile(sadHill, "2_broken.jar", new String(new char[1024])); - UrlClassLoader flat = UrlClassLoader.build().urls(theBad.toURI().toURL(), theGood.toURI().toURL()).get(); + UrlClassLoader flat = UrlClassLoader.build().urls(theBad.toURI().toURL(), theGood.toURI().toURL()).useLazyClassloadingCaches(false).get(); assertNotNull(findResource(flat, entryName, false)); String content = Attributes.Name.MANIFEST_VERSION + ": 1.0\n" + Attributes.Name.CLASS_PATH + ": " + theBad.toURI().toURL() + " " + theGood.toURI().toURL() + "\n\n"; File theUgly = createTestJar(createTestFile(sadHill, "3_classpath.jar"), JarFile.MANIFEST_NAME, content); - UrlClassLoader recursive = UrlClassLoader.build().urls(theUgly.toURI().toURL()).get(); + UrlClassLoader recursive = UrlClassLoader.build().urls(theUgly.toURI().toURL()).useLazyClassloadingCaches(false).get(); assertNotNull(findResource(recursive, entryName, false)); } finally { diff --git a/plugins/devkit/devkit-core/src/run/JUnitDevKitPatcher.java b/plugins/devkit/devkit-core/src/run/JUnitDevKitPatcher.java index 277692d13fc0..a7e043841ec5 100644 --- a/plugins/devkit/devkit-core/src/run/JUnitDevKitPatcher.java +++ b/plugins/devkit/devkit-core/src/run/JUnitDevKitPatcher.java @@ -25,6 +25,7 @@ import com.intellij.openapi.project.DumbService; import com.intellij.openapi.project.Project; import com.intellij.openapi.projectRoots.*; import com.intellij.openapi.util.io.FileUtil; +import com.intellij.openapi.util.registry.Registry; import com.intellij.openapi.util.text.StringUtil; import com.intellij.psi.JavaPsiFacade; import com.intellij.psi.PsiClass; @@ -65,6 +66,12 @@ public class JUnitDevKitPatcher extends JUnitPatcher { vm.addProperty(SYSTEM_CL_PROPERTY, qualifiedName); } } + + if (Registry.is("idea.lazy.classloading.caches") && + vm.hasProperty(SYSTEM_CL_PROPERTY) && + UrlClassLoader.class.getName().equals(vm.getPropertyValue(SYSTEM_CL_PROPERTY))) { + vm.addProperty("idea.lazy.classloading.caches", "true"); + } jdk = IdeaJdk.findIdeaJdk(jdk); if (jdk == null) return;