MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

test launching performance improvement within IDE project

Browse Source 
For unit tests' run avoid eager building index of package / class or resource names.
Instead lazily cache package locations that were tried during class loading (for FileLoader) and do not cache anything
for JarLoader. Upon opening java.util.ZipFile contains hashed file name index in native code so .getEntry() works good
for uncontended minimal setup scenario.
This commit is contained in:
Maxim.Mossienko
2018-09-17 16:45:19 +02:00
parent 44e18fe2ca
commit e1df0c050b
7 changed files with 135 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/util/resources/misc/registry.properties
+2
View File
@@ -1721,3 +1721,5 @@ new.search.everywhere.single.thread.search.description=Disable multi-thread sear
autodetect.all.jdks=true
autodetect.all.jdks.description=Scan disk and show all found JDKs in "Select Project SDK" dialog
idea.lazy.classloading.caches=false
idea.lazy.classloading.caches.description=Flag for UrlClassLoader to use lazy caching of class locations
platform/util/src/com/intellij/util/lang/ClassPath.java
+19 -3
View File
@@ -54,6 +54,7 @@ public class ClassPath {
private final boolean myAcceptUnescapedUrls;
final boolean myPreloadJarContents;
final boolean myCanHavePersistentIndex;
final boolean myLazyClassloadingCaches;
@Nullable private final CachePoolImpl myCachePool;
@Nullable private final UrlClassLoader.CachingCondition myCachingCondition;
final boolean myLogErrorOnMissingJar;
@@ -66,9 +67,12 @@ public class ClassPath {
boolean canHavePersistentIndex,
@Nullable CachePoolImpl cachePool,
@Nullable UrlClassLoader.CachingCondition cachingCondition,
boolean logErrorOnMissingJar) {
boolean logErrorOnMissingJar,
boolean lazyClassloadingCaches
) {
myLazyClassloadingCaches = lazyClassloadingCaches;
myCanLockJars = canLockJars;
myCanUseCache = canUseCache;
myCanUseCache = canUseCache && !myLazyClassloadingCaches;
myAcceptUnescapedUrls = acceptUnescapedUrls;
myPreloadJarContents = preloadJarContents;
myCachePool = cachePool;
@@ -208,6 +212,7 @@ public class ClassPath {
if (processRecursively) {
String[] referencedJars = loadManifestClasspath(loader);
if (referencedJars != null) {
//long s2 = System.nanoTime();
for (String referencedJar : referencedJars) {
try {
URI uri = new URI(referencedJar);
@@ -222,6 +227,7 @@ public class ClassPath {
Logger.getInstance(ClassPath.class).warn("url: " + url + " / " + referencedJar, e);
}
}
//System.out.println("Loaded all " + referencedJars.length + " urls " + (System.nanoTime() - s2) / 1000000 + "ms");
}
}
return loader;
@@ -429,10 +435,20 @@ public class ClassPath {
if (time > 10000000L) {
System.out.println(time / 1000000 + " ms for " + msg);
}
if (ourTotalRequests % 1000 == 0) {
if (ourTotalRequests % 10000 == 0) {
System.out.println(path + ", requests:" + ourTotalRequests + ", time:" + (ourTotalTime / 1000000) + "ms");
}
}
static {
if (ourLogTiming) {
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
public void run() {
System.out.println("Classloading requests:" + ourTotalRequests + ", time:" + (ourTotalTime / 1000000) + "ms");
}
});
}
}
private static String[] loadManifestClasspath(JarLoader loader) {
try {
platform/util/src/com/intellij/util/lang/FileLoader.java
+82
View File
@@ -17,6 +17,7 @@ package com.intellij.util.lang;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.util.SmartList;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -74,10 +75,91 @@ class FileLoader extends Loader {
return relativePath;
}
private static class DirEntry {
static final int[] empty = new int[0];
volatile int[] childrenNameHashes;
volatile List<DirEntry> childrenDirectories;
final int nameHash;
final String name;
DirEntry(int _nameHash, String _name) {
nameHash = _nameHash;
name = _name;
}
}
private final DirEntry root = new DirEntry(0, null);
@Override
@Nullable
Resource getResource(final String name) {
try {
if (myConfiguration.myLazyClassloadingCaches) {
DirEntry lastEntry = root;
int prevIndex = 0;
int nextIndex = name.indexOf('/', prevIndex);
while (true) {
int nameEnd = nextIndex == -1 ? name.length() : nextIndex;
int nameHash = StringUtil.stringHashCodeInsensitive(name, prevIndex, nameEnd);
int[] childrenNameHashes = lastEntry.childrenNameHashes; // volatile read
if (childrenNameHashes == null) {
String[] list = (prevIndex != 0 ? new File(myRootDir, name.substring(0, prevIndex)) : myRootDir).list();
if (list != null) {
childrenNameHashes = new int[list.length];
for (int i = 0; i < list.length; ++i) {
childrenNameHashes[i] = StringUtil.stringHashCodeInsensitive(list[i]);
}
}
else {
childrenNameHashes = DirEntry.empty;
}
lastEntry.childrenNameHashes = childrenNameHashes; // volatile write
}
boolean found = false;
for (int childNameHash : childrenNameHashes) {
if (childNameHash == nameHash) {
found = true;
break;
}
}
if (!found) {
return null;
}
if (nextIndex == -1 || nextIndex == name.length() - 1) {
break;
}
DirEntry nextEntry = null;
List<DirEntry> directories = lastEntry.childrenDirectories; // volatile read
if (directories != null) {
for (DirEntry previouslyScannedDir : directories) {
if (previouslyScannedDir.nameHash == nameHash && previouslyScannedDir.name.regionMatches(0, name, prevIndex, nameEnd - prevIndex)) {
nextEntry = previouslyScannedDir;
break;
}
}
}
if (nextEntry == null) {
nextEntry = new DirEntry(nameHash, name.substring(prevIndex, nameEnd));
List<DirEntry> newChildrenDirectories = directories != null ? new SmartList<DirEntry>(directories) : new SmartList<DirEntry>();
newChildrenDirectories.add(nextEntry);
lastEntry.childrenDirectories = newChildrenDirectories; // volatile write with new copy of data
}
lastEntry = nextEntry;
prevIndex = nextIndex + 1;
nextIndex = name.indexOf('/', prevIndex);
}
}
URL url = new URL(getBaseURL(), name);
if (!url.getFile().startsWith(getBaseURL().getFile())) return null;
File file = new File(myRootDir, name.replace('/', File.separatorChar));
platform/util/src/com/intellij/util/lang/JarLoader.java
+11 -8
View File
@@ -59,16 +59,19 @@ class JarLoader extends Loader {
myConfiguration = configuration;
myUrl = url;
ZipFile zipFile = getZipFile(); // IOException from opening is propagated to caller if zip file isn't valid,
try {
if (configuration.myPreloadJarContents) {
JarMemoryLoader loader = JarMemoryLoader.load(zipFile, getBaseURL(), this);
if (loader != null) {
myMemoryLoader = new SoftReference<JarMemoryLoader>(loader);
if (!configuration.myLazyClassloadingCaches) {
ZipFile zipFile = getZipFile(); // IOException from opening is propagated to caller if zip file isn't valid,
try {
if (configuration.myPreloadJarContents) {
JarMemoryLoader loader = JarMemoryLoader.load(zipFile, getBaseURL(), this);
if (loader != null) {
myMemoryLoader = new SoftReference<JarMemoryLoader>(loader);
}
}
}
} finally {
releaseZipFile(zipFile);
finally {
releaseZipFile(zipFile);
}
}
}
platform/util/src/com/intellij/util/lang/UrlClassLoader.java
+12 -1
View File
@@ -93,6 +93,7 @@ public class UrlClassLoader extends ClassLoader {
private boolean myPreload = true;
private boolean myAllowBootstrapResources;
private boolean myErrorOnMissingJar = true;
private boolean myLazyClassloadingCaches = SystemProperties.getBooleanProperty("idea.lazy.classloading.caches", false);
@Nullable private CachePoolImpl myCachePool;
@Nullable private CachingCondition myCachingCondition;
@@ -101,8 +102,14 @@ public class UrlClassLoader extends ClassLoader {
public Builder urls(List<URL> urls) { myURLs = urls; return this; }
public Builder urls(URL... urls) { myURLs = Arrays.asList(urls); return this; }
public Builder parent(ClassLoader parent) { myParent = parent; return this; }
// Keep ZipFile handle opened in SoftReference. Depending on OS, the option significantly speeds up classloading from libraries.
// Caveat: for Windows opened handle will lock the file preventing its modification
// Thus, the option is recommended when jars are not modified or process that uses this option is transient
public Builder allowLock() { myLockJars = true; return this; }
public Builder allowLock(boolean lockJars) { myLockJars = lockJars; return this; }
// Build index of packages / class or resource names that allows to avoid IO during classloading
public Builder useCache() { myUseCache = true; return this; }
public Builder useCache(boolean useCache) { myUseCache = useCache; return this; }
@@ -139,6 +146,10 @@ public class UrlClassLoader extends ClassLoader {
public Builder noPreload() { myPreload = false; return this; }
public Builder allowBootstrapResources() { myAllowBootstrapResources = true; return this; }
public Builder setLogErrorOnMissingJar(boolean log) {myErrorOnMissingJar = log; return this; }
// Cache package contents information in FileLoader lazily upon classloading.
// This is less efficient than useCache / usePersistentClasspathIndexForLocalClassDirectories but the option has much smaller startup overhead.
public Builder useLazyClassloadingCaches(boolean pleaseBeLazy) { myLazyClassloadingCaches = pleaseBeLazy; return this; }
public UrlClassLoader get() { return new UrlClassLoader(this); }
}
@@ -176,7 +187,7 @@ public class UrlClassLoader extends ClassLoader {
protected final ClassPath createClassPath(@NotNull Builder builder) {
return new ClassPath(myURLs, builder.myLockJars, builder.myUseCache, builder.myAcceptUnescaped, builder.myPreload,
builder.myUsePersistentClasspathIndex, builder.myCachePool, builder.myCachingCondition,
builder.myErrorOnMissingJar);
builder.myErrorOnMissingJar, builder.myLazyClassloadingCaches);
}
public static URL internProtocol(@NotNull URL url) {
platform/util/testSrc/com/intellij/util/lang/UrlClassLoaderTest.java
+2 -2
View File
@@ -148,14 +148,14 @@ public class UrlClassLoaderTest {
File theGood = createTestJar(createTestFile(sadHill, "1_normal.jar"), entryName, "-");
File theBad = createTestFile(sadHill, "2_broken.jar", new String(new char[1024]));
UrlClassLoader flat = UrlClassLoader.build().urls(theBad.toURI().toURL(), theGood.toURI().toURL()).get();
UrlClassLoader flat = UrlClassLoader.build().urls(theBad.toURI().toURL(), theGood.toURI().toURL()).useLazyClassloadingCaches(false).get();
assertNotNull(findResource(flat, entryName, false));
String content = Attributes.Name.MANIFEST_VERSION + ": 1.0\n" +
Attributes.Name.CLASS_PATH + ": " + theBad.toURI().toURL() + " " + theGood.toURI().toURL() + "\n\n";
File theUgly = createTestJar(createTestFile(sadHill, "3_classpath.jar"), JarFile.MANIFEST_NAME, content);
UrlClassLoader recursive = UrlClassLoader.build().urls(theUgly.toURI().toURL()).get();
UrlClassLoader recursive = UrlClassLoader.build().urls(theUgly.toURI().toURL()).useLazyClassloadingCaches(false).get();
assertNotNull(findResource(recursive, entryName, false));
}
finally {
plugins/devkit/devkit-core/src/run/JUnitDevKitPatcher.java
+7
View File
@@ -25,6 +25,7 @@ import com.intellij.openapi.project.DumbService;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.projectRoots.*;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.registry.Registry;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.psi.JavaPsiFacade;
import com.intellij.psi.PsiClass;
@@ -65,6 +66,12 @@ public class JUnitDevKitPatcher extends JUnitPatcher {
vm.addProperty(SYSTEM_CL_PROPERTY, qualifiedName);
}
}
if (Registry.is("idea.lazy.classloading.caches") &&
vm.hasProperty(SYSTEM_CL_PROPERTY) &&
UrlClassLoader.class.getName().equals(vm.getPropertyValue(SYSTEM_CL_PROPERTY))) {
vm.addProperty("idea.lazy.classloading.caches", "true");
}
jdk = IdeaJdk.findIdeaJdk(jdk);
if (jdk == null) return;