stub hierarchy: use less memory when reading from index

This commit is contained in:
peter
2016-07-05 08:58:07 +02:00
parent c4e031dfcd
commit da07daf5ce
8 changed files with 346 additions and 260 deletions
@@ -79,14 +79,14 @@ public class HierarchyServiceImpl extends HierarchyService {
}
private void loadUnits(BitSet files, int[] indexKeys, StubEnter stubEnter) {
FileBasedIndexImpl.IdValueProcessor<IndexTree.Unit> processor = new FileBasedIndexImpl.IdValueProcessor<IndexTree.Unit>() {
FileBasedIndexImpl.IdValueProcessor<SerializedUnit> processor = new FileBasedIndexImpl.IdValueProcessor<SerializedUnit>() {
final ProgressIndicator indicator = ProgressIndicatorProvider.getGlobalProgressIndicator();
int count = 0;
@Override
public boolean process(int fileId, IndexTree.Unit unit) {
public boolean process(int fileId, SerializedUnit unit) {
if (indicator != null && ++count % 128 == 0) indicator.checkCanceled();
if (files.get(fileId)) {
stubEnter.unitEnter(unit, fileId);
unit.readUnit(stubEnter, fileId);
}
return true;
}
@@ -15,10 +15,16 @@
*/
package com.intellij.psi.stubsHierarchy.impl;
import com.intellij.psi.impl.java.stubs.hierarchy.IndexTree;
import com.intellij.util.ArrayUtil;
import com.intellij.util.BitUtil;
import com.intellij.util.io.DataInputOutputUtil;
import org.jetbrains.annotations.NotNull;
public class Imports {
import java.io.DataOutput;
import java.io.IOException;
class Imports {
public final static long[] EMPTY_ARRAY = ArrayUtil.EMPTY_LONG_ARRAY;
public static final int onDemandMask = 1 << 29;
@@ -50,4 +56,46 @@ public class Imports {
if (onDemand) lower |= onDemandMask;
return (((long)alias) << 32) | lower;
}
private final static int IS_STATIC = 1;
private final static int IS_ON_DEMAND = 2;
private final static int HAS_ALIAS = 4;
static void writeImports(@NotNull DataOutput out, IndexTree.Unit value) throws IOException {
DataInputOutputUtil.writeINT(out, value.imports.length);
for (IndexTree.Import anImport : value.imports) {
writeImport(out, anImport);
}
}
static long[] readImports(UnitInputStream in) throws IOException {
int importCount = DataInputOutputUtil.readINT(in);
long[] imports = importCount == 0 ? EMPTY_ARRAY : new long[importCount];
for (int i = 0; i < importCount; i++) {
imports[i] = readImport(in);
}
return imports;
}
private static void writeImport(@NotNull DataOutput out, IndexTree.Import anImport) throws IOException {
SerializedUnit.writeQualifiedName(out, anImport.myFullname);
boolean hasAlias = anImport.myAlias != 0;
int flags = 0;
flags = BitUtil.set(flags, IS_STATIC, anImport.myStaticImport);
flags = BitUtil.set(flags, IS_ON_DEMAND, anImport.myOnDemand);
flags = BitUtil.set(flags, HAS_ALIAS, hasAlias);
out.writeByte(flags);
if (hasAlias) {
out.writeInt(anImport.myAlias);
}
}
private static long readImport(UnitInputStream in) throws IOException {
int fullname = in.names.readQualifiedName(in);
int flags = in.readByte();
return mkImport(fullname,
BitUtil.isSet(flags, IS_STATIC), BitUtil.isSet(flags, IS_ON_DEMAND),
BitUtil.isSet(flags, HAS_ALIAS) ? in.readInt() : 0);
}
}
@@ -18,9 +18,14 @@ package com.intellij.psi.stubsHierarchy.impl;
import com.intellij.openapi.util.UserDataHolderBase;
import com.intellij.psi.CommonClassNames;
import com.intellij.psi.impl.java.stubs.hierarchy.IndexTree;
import com.intellij.util.io.DataInputOutputUtil;
import gnu.trove.TIntArrayList;
import gnu.trove.TLongIntHashMap;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
class NameEnvironment extends UserDataHolderBase {
public static final int OBJECT_NAME = IndexTree.hashIdentifier("Object");
public static final int NO_NAME = 0;
@@ -67,6 +72,22 @@ class NameEnvironment extends UserDataHolderBase {
return id;
}
/**
* @see SerializedUnit#writeQualifiedName(DataOutput, int[])
*/
@QNameId int readQualifiedName(DataInput in) throws IOException {
int id = 0;
int len = DataInputOutputUtil.readINT(in);
for (int i = 0; i < len; i++) {
id = qualifiedName(id, in.readInt());
}
return id;
}
int memberQualifiedName(@QNameId int ownerName, @ShortName int name) {
return name == NO_NAME || ownerName < 0 ? -1 : qualifiedName(ownerName, name);
}
@QNameId int qualifiedName(@QNameId int prefix, @ShortName int shortName) {
int existing = findExistingName(prefix, shortName);
return existing >= 0 ? existing : addName(prefix, shortName);
@@ -0,0 +1,217 @@
/*
* Copyright 2000-2016 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.psi.stubsHierarchy.impl;
import com.intellij.psi.impl.java.stubs.hierarchy.IndexTree;
import com.intellij.psi.stubsHierarchy.impl.Symbol.ClassSymbol;
import com.intellij.psi.stubsHierarchy.impl.Symbol.MemberSymbol;
import com.intellij.psi.stubsHierarchy.impl.Symbol.PackageSymbol;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.io.DataInputOutputUtil;
import com.intellij.util.io.DataOutputStream;
import com.intellij.util.io.UnsyncByteArrayInputStream;
import com.intellij.util.io.UnsyncByteArrayOutputStream;
import org.jetbrains.annotations.NotNull;
import java.io.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
* @author peter
*/
class SerializedUnit {
private final byte[] myBytes;
SerializedUnit(byte[] bytes) {
myBytes = bytes;
}
SerializedUnit(IndexTree.Unit unit) {
try {
//noinspection IOResourceOpenedButNotSafelyClosed
UnsyncByteArrayOutputStream stream = new UnsyncByteArrayOutputStream();
writeUnit(new DataOutputStream(stream), unit);
myBytes = stream.toByteArray();
}
catch (IOException impossible) {
throw new RuntimeException(impossible);
}
}
byte[] getSerializedBytes() {
return myBytes;
}
void readUnit(StubEnter stubEnter, int fileId) {
try {
enterUnit(new UnitInputStream(new UnsyncByteArrayInputStream(myBytes), fileId, stubEnter));
}
catch (IOException impossible) {
throw new RuntimeException(impossible);
}
}
/**
* @see NameEnvironment#readQualifiedName(DataInput)
*/
static void writeQualifiedName(DataOutput out, @QNameId int[] array) throws IOException {
DataInputOutputUtil.writeINT(out, array.length);
for (int i : array) {
out.writeInt(i);
}
}
// unit
private static void writeUnit(@NotNull DataOutput out, IndexTree.Unit value) throws IOException {
writeQualifiedName(out, value.myPackageName);
out.writeByte(value.myUnitType);
if (value.myUnitType != IndexTree.BYTECODE) {
Imports.writeImports(out, value);
}
// class Declaration
DataInputOutputUtil.writeINT(out, value.myDecls.length);
for (IndexTree.ClassDecl def : value.myDecls) {
saveClassDecl(out, def);
}
}
private static void enterUnit(UnitInputStream in) throws IOException {
PackageSymbol pkg = in.stubEnter.enterPackage(in);
byte type = in.readByte();
long[] imports = type == IndexTree.BYTECODE ? Imports.EMPTY_ARRAY : Imports.readImports(in);
UnitInfo unitInfo = UnitInfo.mkUnitInfo(type, imports);
int classCount = DataInputOutputUtil.readINT(in);
for (int i = 0; i < classCount; i++) {
readClassDecl(in, unitInfo, pkg, pkg.myQualifiedName);
}
}
// class
private static void saveClassDecl(@NotNull DataOutput out, IndexTree.ClassDecl value) throws IOException {
DataInputOutputUtil.writeINT(out, value.myStubId);
DataInputOutputUtil.writeINT(out, value.myMods);
out.writeInt(value.myName);
writeSupers(out, value);
writeMembers(out, value.myDecls);
}
private static ClassSymbol readClassDecl(UnitInputStream in, UnitInfo info, Symbol owner, @QNameId int ownerName) throws IOException {
int stubId = DataInputOutputUtil.readINT(in);
int mods = DataInputOutputUtil.readINT(in);
@ShortName int name = in.readInt();
@QNameId int[] superNames = readSupers(in);
@QNameId int qname = in.names.memberQualifiedName(ownerName, name);
ClassSymbol symbol = in.stubEnter.classEnter(info, owner, stubId, mods, name, superNames, qname, in.fileId);
readMembers(in, info, qname, symbol);
return symbol;
}
// supers
private static void writeSupers(@NotNull DataOutput out, IndexTree.ClassDecl value) throws IOException {
DataInputOutputUtil.writeINT(out, value.mySupers.length);
for (int[] aSuper : value.mySupers) {
writeQualifiedName(out, aSuper);
}
}
private static @QNameId int[] readSupers(UnitInputStream in) throws IOException {
@QNameId int[] superNames = new int[DataInputOutputUtil.readINT(in)];
for (int i = 0; i < superNames.length; i++) {
superNames[i] = in.names.readQualifiedName(in);
}
return superNames;
}
// members
private static void writeMembers(@NotNull DataOutput out, IndexTree.Decl[] decls) throws IOException {
DataInputOutputUtil.writeINT(out, decls.length);
for (IndexTree.Decl def : decls) {
saveDecl(out, def);
}
}
private static void readMembers(UnitInputStream in,
UnitInfo info,
@QNameId int ownerName,
MemberSymbol symbol) throws IOException {
int memberCount = DataInputOutputUtil.readINT(in);
if (memberCount == 0) return;
List<ClassSymbol> members = new ArrayList<>();
for (int i = 0; i < memberCount; i++) {
ContainerUtil.addIfNotNull(members, readDecl(in, info, symbol, ownerName));
}
symbol.setMembers(members);
}
// decl: class or member
private static void saveDecl(@NotNull DataOutput out, IndexTree.Decl value) throws IOException {
if (value instanceof IndexTree.ClassDecl) {
out.writeBoolean(true);
saveClassDecl(out, (IndexTree.ClassDecl)value);
} else if (value instanceof IndexTree.MemberDecl) {
out.writeBoolean(false);
writeMembers(out, ((IndexTree.MemberDecl)value).myDecls);
}
}
private static ClassSymbol readDecl(UnitInputStream in, UnitInfo info, Symbol owner, @QNameId int ownerName) throws IOException {
if (in.readBoolean()) {
return readClassDecl(in, info, owner, ownerName);
}
readMembers(in, info, ownerName, new MemberSymbol(owner));
return null;
}
@Override
public boolean equals(Object o) {
return this == o || o instanceof SerializedUnit && Arrays.equals(myBytes, ((SerializedUnit)o).myBytes);
}
@Override
public int hashCode() {
int result = myBytes.length;
int length = Math.min(30, myBytes.length);
for (int i = 0; i < length; i++) {
result = 31 * result + myBytes[i];
}
return result;
}
}
class UnitInputStream extends DataInputStream {
final int fileId;
final StubEnter stubEnter;
final NameEnvironment names;
UnitInputStream(InputStream in, int fileId, StubEnter stubEnter) {
super(in);
this.fileId = fileId;
this.stubEnter = stubEnter;
this.names = stubEnter.myNameEnvironment;
}
}
@@ -19,13 +19,15 @@ import com.intellij.psi.impl.java.stubs.hierarchy.IndexTree;
import com.intellij.util.BitUtil;
import org.jetbrains.annotations.Nullable;
import java.io.DataInput;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import static com.intellij.psi.stubsHierarchy.impl.Symbol.*;
import static com.intellij.psi.stubsHierarchy.impl.Symbol.ClassSymbol;
import static com.intellij.psi.stubsHierarchy.impl.Symbol.PackageSymbol;
public class StubEnter {
private final NameEnvironment myNameEnvironment;
final NameEnvironment myNameEnvironment;
private final Symbols mySymbols;
private final StubHierarchyConnector myStubHierarchyConnector;
@@ -37,85 +39,30 @@ public class StubEnter {
myStubHierarchyConnector = new StubHierarchyConnector(myNameEnvironment, symbols);
}
void unitEnter(IndexTree.Unit unit, int fileId) {
@QNameId int pkgName = unit.myPackageName.length > 0 ? myNameEnvironment.internQualifiedName(unit.myPackageName) : 0;
enter(unit.myDecls, UnitInfo.mkUnitInfo(unit.myUnitType, internImports(unit)), mySymbols.enterPackage(pkgName), pkgName, fileId);
PackageSymbol enterPackage(DataInput in) throws IOException {
return mySymbols.enterPackage(myNameEnvironment.readQualifiedName(in));
}
private long[] internImports(IndexTree.Unit unit) {
long[] imports = unit.imports.length == 0 ? Imports.EMPTY_ARRAY : new long[unit.imports.length];
for (int i = 0; i < unit.imports.length; i++) {
imports[i] = processImport(unit.imports[i]);
}
return imports;
}
private long processImport(IndexTree.Import anImport) {
int fullname = myNameEnvironment.internQualifiedName(anImport.myFullname);
return Imports.mkImport(fullname, anImport.myStaticImport, anImport.myOnDemand, anImport.myAlias);
}
private void enter(IndexTree.ClassDecl[] trees, UnitInfo info, Symbol owner, @QNameId int ownerName, int fileId) {
for (IndexTree.ClassDecl tree : trees) {
enter(tree, info, owner, ownerName, fileId);
}
}
private ClassSymbol[] enter(IndexTree.Decl[] trees, UnitInfo info, Symbol owner, @QNameId int ownerName, int fileId) {
ClassSymbol[] members = new ClassSymbol[trees.length];
int i = 0;
for (IndexTree.Decl tree : trees) {
ClassSymbol member = enter(tree, info, owner, ownerName, fileId);
if (member != null && member.myShortName != 0) {
members[i++] = member;
}
}
if (i == 0) return ClassSymbol.EMPTY_ARRAY;
if (i < members.length) {
members = Arrays.copyOf(members, i);
}
Arrays.sort(members, CLASS_SYMBOL_BY_NAME_COMPARATOR);
return members;
}
private ClassSymbol enter(IndexTree.Decl tree, UnitInfo info, Symbol owner, @QNameId int ownerName, int fileId) {
if (tree instanceof IndexTree.ClassDecl) {
return classEnter((IndexTree.ClassDecl)tree, info, owner, ownerName, fileId);
}
if (tree instanceof IndexTree.MemberDecl) {
memberEnter((IndexTree.MemberDecl)tree, info, owner, ownerName, fileId);
return null;
}
return null;
}
private void memberEnter(IndexTree.MemberDecl tree, UnitInfo info, Symbol owner, @QNameId int ownerName, int fileId) {
MemberSymbol mc = new MemberSymbol(owner);
mc.setMembers(enter(tree.myDecls, info, mc, ownerName, fileId));
}
private ClassSymbol classEnter(IndexTree.ClassDecl tree, UnitInfo info, Symbol owner, @QNameId int ownerName, int fileId) {
int flags = checkFlags(tree.myMods, owner, info.getType() == IndexTree.BYTECODE);
int name = tree.myName;
@QNameId int qname = name == NameEnvironment.NO_NAME || ownerName < 0 ? -1
: myNameEnvironment.qualifiedName(ownerName, name);
@CompactArray(QualifiedName.class) Object supers = internSupers(tree.myMods, tree.mySupers);
ClassSymbol classSymbol = mySymbols.enterClass(fileId, tree.myStubId, flags, name, owner, info, supers, qname);
ClassSymbol classEnter(UnitInfo info,
Symbol owner,
int stubId,
int mods,
@ShortName int name,
@QNameId int[] superNames,
@QNameId int qname, int fileId) throws IOException {
int flags = checkFlags(mods, owner, info.getType() == IndexTree.BYTECODE);
@CompactArray(QualifiedName.class) Object supers = internSupers(mods, superNames);
ClassSymbol classSymbol = mySymbols.enterClass(fileId, stubId, flags, name, owner, info, supers, qname);
if (uncompleted != null) {
uncompleted.add(classSymbol);
}
if (tree.myDecls.length > 0) {
classSymbol.setMembers(enter(tree.myDecls, info, classSymbol, qname, fileId));
}
return classSymbol;
}
@Nullable
@CompactArray(QualifiedName.class)
private Object internSupers(int flags, int[][] superNames) {
Object internSupers(int flags, int[] superNames) {
if (BitUtil.isSet(flags, IndexTree.ANNOTATION)) {
return myNameEnvironment.java_lang_annotation_Annotation;
}
@@ -125,12 +72,12 @@ public class StubEnter {
return isEnum ? myNameEnvironment.java_lang_Enum : null;
}
if (superNames.length == 1 && !isEnum) {
return new QualifiedName(myNameEnvironment.internQualifiedName(superNames[0]));
return new QualifiedName(superNames[0]);
}
QualifiedName[] array = new QualifiedName[superNames.length + (isEnum ? 1 : 0)];
for (int i = 0; i < superNames.length; i++) {
array[i] = new QualifiedName(myNameEnvironment.internQualifiedName(superNames[i]));
array[i] = new QualifiedName(superNames[i]);
}
if (isEnum) {
array[array.length - 1] = myNameEnvironment.java_lang_Enum;
@@ -21,14 +21,17 @@ import com.intellij.openapi.roots.ProjectFileIndex;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.openapi.vfs.VirtualFileWithId;
import com.intellij.psi.impl.java.stubs.hierarchy.IndexTree;
import com.intellij.psi.impl.java.stubs.index.JavaUnitDescriptor;
import com.intellij.psi.stubsHierarchy.StubHierarchyIndexer;
import com.intellij.util.indexing.*;
import com.intellij.util.io.DataExternalizer;
import com.intellij.util.io.DataInputOutputUtil;
import com.intellij.util.io.EnumeratorIntegerDescriptor;
import com.intellij.util.io.KeyDescriptor;
import org.jetbrains.annotations.NotNull;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.stream.IntStream;
@@ -36,29 +39,29 @@ import java.util.stream.IntStream;
/**
* @author peter
*/
public class StubHierarchyIndex extends FileBasedIndexExtension<Integer, IndexTree.Unit> implements PsiDependentIndex {
public class StubHierarchyIndex extends FileBasedIndexExtension<Integer, SerializedUnit> implements PsiDependentIndex {
private static final int KEY_COUNT = 20;
static final int[] BINARY_KEYS = IntStream.rangeClosed(1, KEY_COUNT).toArray();
static final int[] SOURCE_KEYS = IntStream.rangeClosed(-KEY_COUNT, -1).toArray();
static final ID<Integer, IndexTree.Unit> INDEX_ID = ID.create("jvm.hierarchy");
static final ID<Integer, SerializedUnit> INDEX_ID = ID.create("jvm.hierarchy");
private static final StubHierarchyIndexer[] ourIndexers = StubHierarchyIndexer.EP_NAME.getExtensions();
@NotNull
@Override
public ID<Integer, IndexTree.Unit> getName() {
public ID<Integer, SerializedUnit> getName() {
return INDEX_ID;
}
@NotNull
@Override
public DataIndexer<Integer, IndexTree.Unit, FileContent> getIndexer() {
public DataIndexer<Integer, SerializedUnit, FileContent> getIndexer() {
return inputData -> {
for (StubHierarchyIndexer indexer : ourIndexers) {
VirtualFile file = inputData.getFile();
IndexTree.Unit unit = indexer.handlesFile(file) ? indexer.indexFile(inputData) : null;
if (unit != null && unit.myDecls.length > 0) {
int[] keys = file.getFileType().isBinary() ? BINARY_KEYS : SOURCE_KEYS;
return Collections.singletonMap(keys[((VirtualFileWithId) file).getId() % keys.length], unit);
return Collections.singletonMap(keys[((VirtualFileWithId) file).getId() % keys.length], new SerializedUnit(unit));
}
}
return Collections.emptyMap();
@@ -73,13 +76,27 @@ public class StubHierarchyIndex extends FileBasedIndexExtension<Integer, IndexTr
@NotNull
@Override
public DataExternalizer<IndexTree.Unit> getValueExternalizer() {
return JavaUnitDescriptor.INSTANCE;
public DataExternalizer<SerializedUnit> getValueExternalizer() {
return new DataExternalizer<SerializedUnit>() {
@Override
public void save(@NotNull DataOutput out, SerializedUnit value) throws IOException {
byte[] bytes = value.getSerializedBytes();
DataInputOutputUtil.writeINT(out, bytes.length);
out.write(bytes);
}
@Override
public SerializedUnit read(@NotNull DataInput in) throws IOException {
byte[] bytes = new byte[DataInputOutputUtil.readINT(in)];
in.readFully(bytes);
return new SerializedUnit(bytes);
}
};
}
@Override
public int getVersion() {
return IndexTree.STUB_HIERARCHY_ENABLED ? 5 + Arrays.stream(ourIndexers).mapToInt(StubHierarchyIndexer::getVersion).sum() : 0;
return IndexTree.STUB_HIERARCHY_ENABLED ? 6 + Arrays.stream(ourIndexers).mapToInt(StubHierarchyIndexer::getVersion).sum() : 0;
}
@NotNull
@@ -21,9 +21,7 @@ import gnu.trove.TIntHashSet;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.util.Collections;
import java.util.Comparator;
import java.util.Set;
import java.util.*;
/**
* Java symbols needed for hierarchy building. Mostly classes ({@link ClassSymbol}) or packages ({@link PackageSymbol}),
@@ -200,12 +198,18 @@ public abstract class Symbol {
(ClassSymbol[])myMembers;
}
void setMembers(ClassSymbol[] members) {
myMembers = members.length == 0 ? null : members.length == 1 ? members[0] : members;
void setMembers(List<ClassSymbol> members) {
myMembers = members.isEmpty() ? null : members.size() == 1 ? members.get(0) : toSortedArray(members);
}
private static ClassSymbol[] toSortedArray(List<ClassSymbol> members) {
ClassSymbol[] array = members.toArray(new ClassSymbol[members.size()]);
Arrays.sort(array, CLASS_SYMBOL_BY_NAME_COMPARATOR);
return array;
}
}
public static final Comparator<ClassSymbol> CLASS_SYMBOL_BY_NAME_COMPARATOR = (s1, s2) -> {
private static final Comparator<ClassSymbol> CLASS_SYMBOL_BY_NAME_COMPARATOR = (s1, s2) -> {
int name1 = s1.myShortName;
int name2 = s2.myShortName;
return (name1 < name2) ? -1 : ((name1 == name2) ? 0 : 1);