From 95b3d482d95c27dbb065a5d47c1de2133a85e39b Mon Sep 17 00:00:00 2001 From: peter Date: Fri, 24 Jun 2016 14:57:17 +0200 Subject: [PATCH] more efficient use of indices in stub hierarchy --- .../impl/HierarchyServiceImpl.java | 71 ++++++++++++------- .../stubsHierarchy/impl/JavaStubIndexer.java | 10 +-- .../impl/StubHierarchyIndex.java | 35 ++++----- .../java/stubs/index/JavaUnitDescriptor.java | 4 +- .../stubsHierarchy/StubHierarchyIndexer.java | 5 +- .../impl/java/stubs/hierarchy/IndexTree.java | 18 ++++- .../psi/stubs/hierarchy/GrStubIndexer.java | 12 ++-- 7 files changed, 92 insertions(+), 63 deletions(-) diff --git a/java/java-impl/src/com/intellij/psi/stubsHierarchy/impl/HierarchyServiceImpl.java b/java/java-impl/src/com/intellij/psi/stubsHierarchy/impl/HierarchyServiceImpl.java index 31fa42faa53a..654729f0d1ba 100644 --- a/java/java-impl/src/com/intellij/psi/stubsHierarchy/impl/HierarchyServiceImpl.java +++ b/java/java-impl/src/com/intellij/psi/stubsHierarchy/impl/HierarchyServiceImpl.java @@ -19,13 +19,11 @@ import com.intellij.openapi.progress.ProgressIndicator; import com.intellij.openapi.progress.ProgressIndicatorProvider; import com.intellij.openapi.project.Project; import com.intellij.openapi.roots.ProjectFileIndex; +import com.intellij.openapi.roots.ProjectRootManager; import com.intellij.openapi.util.text.StringUtil; -import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.openapi.vfs.VirtualFileManager; import com.intellij.openapi.vfs.VirtualFileWithId; import com.intellij.psi.impl.java.stubs.hierarchy.IndexTree; -import com.intellij.psi.search.DelegatingGlobalSearchScope; -import com.intellij.psi.search.EverythingGlobalScope; -import com.intellij.psi.search.GlobalSearchScope; import com.intellij.psi.stubsHierarchy.HierarchyService; import com.intellij.psi.util.CachedValue; import com.intellij.psi.util.CachedValueProvider; @@ -33,17 +31,18 @@ import com.intellij.psi.util.CachedValuesManager; import com.intellij.psi.util.PsiModificationTracker; import com.intellij.util.CachedValueBase; import com.intellij.util.indexing.FileBasedIndex; +import com.intellij.util.indexing.FileBasedIndexImpl; import org.jetbrains.annotations.NotNull; +import java.util.BitSet; + public class HierarchyServiceImpl extends HierarchyService { private static final SingleClassHierarchy EMPTY_HIERARCHY = new SingleClassHierarchy(Symbol.ClassSymbol.EMPTY_ARRAY); private final Project myProject; - private final ProjectFileIndex myFileIndex; private final CachedValue myHierarchy; public HierarchyServiceImpl(Project project) { myProject = project; - myFileIndex = ProjectFileIndex.SERVICE.getInstance(project); myHierarchy = CachedValuesManager.getManager(project).createCachedValue( () -> CachedValueProvider.Result.create(buildHierarchy(), PsiModificationTracker.MODIFICATION_COUNT), false); @@ -69,38 +68,56 @@ public class HierarchyServiceImpl extends HierarchyService { private SingleClassHierarchy buildHierarchy() { Symbols symbols = new Symbols(); StubEnter stubEnter = new StubEnter(symbols); + IdSets idSets = IdSets.getIdSets(myProject); - loadUnits(false, symbols.myNameEnvironment, stubEnter); + loadUnits(idSets.libraryFiles, StubHierarchyIndex.BINARY_FILES, symbols.myNameEnvironment, stubEnter); stubEnter.connect1(); - loadUnits(true, symbols.myNameEnvironment, stubEnter); + loadUnits(idSets.sourceFiles, StubHierarchyIndex.SOURCE_FILES, symbols.myNameEnvironment, stubEnter); stubEnter.connect2(); return symbols.createHierarchy(); } - private void loadUnits(boolean sourceMode, NameEnvironment names, StubEnter stubEnter) { - GlobalSearchScope scope = new DelegatingGlobalSearchScope(new EverythingGlobalScope(myProject)) { - @Override - public boolean contains(@NotNull VirtualFile file) { - return sourceMode ? myFileIndex.isInSourceContent(file) : myFileIndex.isInLibraryClasses(file); - } - }; + private void loadUnits(BitSet files, int indexKey, NameEnvironment names, StubEnter stubEnter) { ProgressIndicator indicator = ProgressIndicatorProvider.getGlobalProgressIndicator(); - FileBasedIndex index = FileBasedIndex.getInstance(); - for (String packageName : index.getAllKeys(StubHierarchyIndex.INDEX_ID, myProject)) { - QualifiedName pkg = StringUtil.isEmpty(packageName) ? null : names.fromString(packageName, true); - index.processValues(StubHierarchyIndex.INDEX_ID, packageName, null, new FileBasedIndex.ValueProcessor() { - int count = 0; - @Override - public boolean process(VirtualFile file, IndexTree.Unit unit) { - if (indicator != null && ++count % 128 == 0) indicator.checkCanceled(); - stubEnter.unitEnter(Translator.internNames(names, unit, ((VirtualFileWithId)file).getId(), pkg)); - return true; + FileBasedIndexImpl index = (FileBasedIndexImpl)FileBasedIndex.getInstance(); + index.processAllValues(StubHierarchyIndex.INDEX_ID, indexKey, myProject, new FileBasedIndexImpl.IdValueProcessor() { + int count = 0; + @Override + public boolean process(int fileId, IndexTree.Unit unit) { + if (indicator != null && ++count % 128 == 0) indicator.checkCanceled(); + if (files.get(fileId)) { + QualifiedName pkg = StringUtil.isEmpty(unit.myPackageId) ? null : names.fromString(unit.myPackageId, true); + stubEnter.unitEnter(Translator.internNames(names, unit, fileId, pkg)); } - }, scope); - } + return true; + } + }); } + private static class IdSets { + final BitSet sourceFiles = new BitSet(); + final BitSet libraryFiles = new BitSet(); + + static IdSets getIdSets(@NotNull Project project) { + return CachedValuesManager.getManager(project).getCachedValue(project, () -> { + IdSets answer = new IdSets(); + ProjectFileIndex index = ProjectFileIndex.SERVICE.getInstance(project); + FileBasedIndex.getInstance().iterateIndexableFiles(file -> { + if (!file.isDirectory() && file instanceof VirtualFileWithId) { + if (index.isInSourceContent(file)) { + answer.sourceFiles.set(((VirtualFileWithId) file).getId()); + } + else if (index.isInLibraryClasses(file)) { + answer.libraryFiles.set(((VirtualFileWithId) file).getId()); + } + } + return true; + }, project, ProgressIndicatorProvider.getGlobalProgressIndicator()); + return CachedValueProvider.Result.create(answer, ProjectRootManager.getInstance(project), VirtualFileManager.VFS_STRUCTURE_MODIFICATIONS); + }); + } + } } diff --git a/java/java-impl/src/com/intellij/psi/stubsHierarchy/impl/JavaStubIndexer.java b/java/java-impl/src/com/intellij/psi/stubsHierarchy/impl/JavaStubIndexer.java index 550c33b9c74e..616ea7213f3c 100644 --- a/java/java-impl/src/com/intellij/psi/stubsHierarchy/impl/JavaStubIndexer.java +++ b/java/java-impl/src/com/intellij/psi/stubsHierarchy/impl/JavaStubIndexer.java @@ -18,7 +18,6 @@ package com.intellij.psi.stubsHierarchy.impl; import com.intellij.ide.highlighter.JavaClassFileType; import com.intellij.ide.highlighter.JavaFileType; import com.intellij.openapi.fileTypes.FileType; -import com.intellij.openapi.util.Pair; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.psi.PsiModifier; import com.intellij.psi.PsiNameHelper; @@ -39,7 +38,10 @@ import com.intellij.util.indexing.FileContent; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import java.util.*; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; public class JavaStubIndexer extends StubHierarchyIndexer { @@ -61,7 +63,7 @@ public class JavaStubIndexer extends StubHierarchyIndexer { @Nullable @Override - public List> indexFile(@NotNull FileContent content) { + public Unit indexFile(@NotNull FileContent content) { Stub stubTree = StubTreeBuilder.buildStubTree(content); if (!(stubTree instanceof PsiJavaFileStub)) return null; @@ -87,7 +89,7 @@ public class JavaStubIndexer extends StubHierarchyIndexer { ClassDecl[] classes = classList.isEmpty() ? ClassDecl.EMPTY_ARRAY : classList.toArray(new ClassDecl[classList.size()]); Import[] imports = importList.isEmpty() ? Import.EMPTY_ARRAY : importList.toArray(new Import[importList.size()]); byte type = javaFileStub.isCompiled() ? IndexTree.BYTECODE : IndexTree.JAVA; - return Collections.singletonList(Pair.create(javaFileStub.getPackageName(), new Unit(type, imports, classes))); + return new Unit(javaFileStub.getPackageName(), type, imports, classes); } @Nullable diff --git a/java/java-impl/src/com/intellij/psi/stubsHierarchy/impl/StubHierarchyIndex.java b/java/java-impl/src/com/intellij/psi/stubsHierarchy/impl/StubHierarchyIndex.java index 2550e2fb528a..269bf9fa1f1a 100644 --- a/java/java-impl/src/com/intellij/psi/stubsHierarchy/impl/StubHierarchyIndex.java +++ b/java/java-impl/src/com/intellij/psi/stubsHierarchy/impl/StubHierarchyIndex.java @@ -18,47 +18,42 @@ package com.intellij.psi.stubsHierarchy.impl; import com.intellij.openapi.project.Project; import com.intellij.openapi.project.ProjectManager; import com.intellij.openapi.roots.ProjectFileIndex; -import com.intellij.openapi.util.Pair; -import com.intellij.openapi.util.text.StringUtil; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.psi.impl.java.stubs.hierarchy.IndexTree; import com.intellij.psi.impl.java.stubs.index.JavaUnitDescriptor; import com.intellij.psi.stubsHierarchy.StubHierarchyIndexer; import com.intellij.util.indexing.*; import com.intellij.util.io.DataExternalizer; -import com.intellij.util.io.EnumeratorStringDescriptor; +import com.intellij.util.io.EnumeratorIntegerDescriptor; import com.intellij.util.io.KeyDescriptor; import org.jetbrains.annotations.NotNull; -import java.util.*; +import java.util.Arrays; +import java.util.Collections; /** * @author peter */ -public class StubHierarchyIndex extends FileBasedIndexExtension implements PsiDependentIndex { - static final ID INDEX_ID = ID.create("jvm.hierarchy"); +public class StubHierarchyIndex extends FileBasedIndexExtension implements PsiDependentIndex { + public static final int BINARY_FILES = 0; + public static final int SOURCE_FILES = 1; + static final ID INDEX_ID = ID.create("jvm.hierarchy"); private static final StubHierarchyIndexer[] ourIndexers = StubHierarchyIndexer.EP_NAME.getExtensions(); @NotNull @Override - public ID getName() { + public ID getName() { return INDEX_ID; } @NotNull @Override - public DataIndexer getIndexer() { + public DataIndexer getIndexer() { return inputData -> { for (StubHierarchyIndexer indexer : ourIndexers) { - List> pairs = indexer.handlesFile(inputData.getFile()) ? indexer.indexFile(inputData) : null; - if (pairs != null && !pairs.isEmpty()) { - Map answer = new HashMap<>(); - for (Pair entry : pairs) { - if (entry.second.myDecls.length > 0) { - answer.put(StringUtil.notNullize(entry.first), entry.second); - } - } - return answer; + IndexTree.Unit unit = indexer.handlesFile(inputData.getFile()) ? indexer.indexFile(inputData) : null; + if (unit != null && unit.myDecls.length > 0) { + return Collections.singletonMap(inputData.getFile().getFileType().isBinary() ? BINARY_FILES : SOURCE_FILES, unit); } } return Collections.emptyMap(); @@ -67,8 +62,8 @@ public class StubHierarchyIndex extends FileBasedIndexExtension getKeyDescriptor() { - return EnumeratorStringDescriptor.INSTANCE; + public KeyDescriptor getKeyDescriptor() { + return EnumeratorIntegerDescriptor.INSTANCE; } @NotNull @@ -79,7 +74,7 @@ public class StubHierarchyIndex extends FileBasedIndexExtension { @Override public void save(@NotNull DataOutput out, IndexTree.Unit value) throws IOException { + out.writeUTF(value.myPackageId); out.writeByte(value.myUnitType); if (value.myUnitType != IndexTree.BYTECODE) { DataInputOutputUtil.writeINT(out, value.imports.length); @@ -75,6 +76,7 @@ public class JavaUnitDescriptor implements DataExternalizer { @Override public IndexTree.Unit read(@NotNull DataInput in) throws IOException { + String pid = in.readUTF(); byte type = in.readByte(); IndexTree.Import[] imports = IndexTree.Import.EMPTY_ARRAY; if (type != IndexTree.BYTECODE) { @@ -87,7 +89,7 @@ public class JavaUnitDescriptor implements DataExternalizer { for (int i = 0; i < classes.length; i++) { classes[i] = readClassDecl(in); } - return new IndexTree.Unit(type, imports, classes); + return new IndexTree.Unit(pid, type, imports, classes); } private IndexTree.ClassDecl readClassDecl(DataInput in) throws IOException { diff --git a/java/java-indexing-impl/src/com/intellij/psi/stubsHierarchy/StubHierarchyIndexer.java b/java/java-indexing-impl/src/com/intellij/psi/stubsHierarchy/StubHierarchyIndexer.java index ce9d16602224..4e0e8ac260fc 100644 --- a/java/java-indexing-impl/src/com/intellij/psi/stubsHierarchy/StubHierarchyIndexer.java +++ b/java/java-indexing-impl/src/com/intellij/psi/stubsHierarchy/StubHierarchyIndexer.java @@ -16,15 +16,12 @@ package com.intellij.psi.stubsHierarchy; import com.intellij.openapi.extensions.ExtensionPointName; -import com.intellij.openapi.util.Pair; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.psi.impl.java.stubs.hierarchy.IndexTree; import com.intellij.util.indexing.FileContent; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import java.util.List; - /** * @author peter */ @@ -37,7 +34,7 @@ public abstract class StubHierarchyIndexer { * @return a list of pairs for a specified file content */ @Nullable - public abstract List> indexFile(@NotNull FileContent content); + public abstract IndexTree.Unit indexFile(@NotNull FileContent content); public abstract boolean handlesFile(@NotNull VirtualFile file); diff --git a/java/java-psi-impl/src/com/intellij/psi/impl/java/stubs/hierarchy/IndexTree.java b/java/java-psi-impl/src/com/intellij/psi/impl/java/stubs/hierarchy/IndexTree.java index faf27c3a2d7b..fdf452e9af91 100644 --- a/java/java-psi-impl/src/com/intellij/psi/impl/java/stubs/hierarchy/IndexTree.java +++ b/java/java-psi-impl/src/com/intellij/psi/impl/java/stubs/hierarchy/IndexTree.java @@ -17,6 +17,9 @@ package com.intellij.psi.impl.java.stubs.hierarchy; import com.intellij.openapi.util.registry.Registry; +import com.intellij.openapi.util.text.StringUtil; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import java.util.Arrays; @@ -37,11 +40,13 @@ public class IndexTree { public static final byte GROOVY = 2; public static class Unit { + @NotNull public final String myPackageId; public final byte myUnitType; public final Import[] imports; public final ClassDecl[] myDecls; - public Unit(byte unitType, Import[] imports, ClassDecl[] decls) { + public Unit(@Nullable String packageId, byte unitType, Import[] imports, ClassDecl[] decls) { + this.myPackageId = StringUtil.notNullize(packageId); this.myUnitType = unitType; this.imports = imports; this.myDecls = decls; @@ -54,6 +59,8 @@ public class IndexTree { Unit unit = (Unit)o; + if (myUnitType != unit.myUnitType) return false; + if (!myPackageId.equals(unit.myPackageId)) return false; if (!Arrays.equals(imports, unit.imports)) return false; if (!Arrays.equals(myDecls, unit.myDecls)) return false; @@ -62,7 +69,14 @@ public class IndexTree { @Override public int hashCode() { - return Arrays.hashCode(myDecls); + int hash = myUnitType * 31 + myPackageId.hashCode(); + for (ClassDecl decl : myDecls) { + String name = decl.myName; + if (name != null) { + return hash * 31 + name.hashCode(); + } + } + return hash; } } diff --git a/plugins/groovy/groovy-psi/src/org/jetbrains/plugins/groovy/lang/psi/stubs/hierarchy/GrStubIndexer.java b/plugins/groovy/groovy-psi/src/org/jetbrains/plugins/groovy/lang/psi/stubs/hierarchy/GrStubIndexer.java index 2a9303c09d62..c124aae6cc20 100644 --- a/plugins/groovy/groovy-psi/src/org/jetbrains/plugins/groovy/lang/psi/stubs/hierarchy/GrStubIndexer.java +++ b/plugins/groovy/groovy-psi/src/org/jetbrains/plugins/groovy/lang/psi/stubs/hierarchy/GrStubIndexer.java @@ -15,7 +15,6 @@ */ package org.jetbrains.plugins.groovy.lang.psi.stubs.hierarchy; -import com.intellij.openapi.util.Pair; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.psi.PsiNameHelper; import com.intellij.psi.impl.java.stubs.hierarchy.IndexTree; @@ -33,7 +32,10 @@ import org.jetbrains.plugins.groovy.GroovyFileType; import org.jetbrains.plugins.groovy.lang.parser.GroovyElementTypes; import org.jetbrains.plugins.groovy.lang.psi.stubs.*; -import java.util.*; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; public class GrStubIndexer extends StubHierarchyIndexer { @Override @@ -48,14 +50,14 @@ public class GrStubIndexer extends StubHierarchyIndexer { @Nullable @Override - public List> indexFile(@NotNull FileContent content) { + public Unit indexFile(@NotNull FileContent content) { Stub stubTree = StubTreeBuilder.buildStubTree(content); if (!(stubTree instanceof GrFileStub)) return null; GrFileStub grFileStub = (GrFileStub)stubTree; new StubTree(grFileStub, false); - String pid = null; + String pid = ""; ArrayList classList = new ArrayList(); Set usedNames = new HashSet(); for (StubElement el : grFileStub.getChildrenStubs()) { @@ -82,7 +84,7 @@ public class GrStubIndexer extends StubHierarchyIndexer { } ClassDecl[] classes = classList.isEmpty() ? ClassDecl.EMPTY_ARRAY : classList.toArray(new ClassDecl[classList.size()]); Import[] imports = importList.isEmpty() ? Import.EMPTY_ARRAY : importList.toArray(new Import[importList.size()]); - return Collections.singletonList(Pair.create(pid, new Unit(IndexTree.GROOVY, imports, classes))); + return new Unit(pid, IndexTree.GROOVY, imports, classes); } @Nullable