more compact class anchor storage in stub hierarchy

java/java-impl/src/com/intellij/psi/stubsHierarchy/impl/AnchorRepository.java

@@ -0,0 +1,138 @@
+/*
+ * Copyright 2000-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.psi.stubsHierarchy.impl;
+
+import com.intellij.lang.ASTNode;
+import com.intellij.openapi.project.Project;
+import com.intellij.openapi.vfs.VirtualFile;
+import com.intellij.openapi.vfs.newvfs.persistent.PersistentFS;
+import com.intellij.psi.PsiClass;
+import com.intellij.psi.PsiElement;
+import com.intellij.psi.PsiFile;
+import com.intellij.psi.PsiManager;
+import com.intellij.psi.impl.source.PsiFileImpl;
+import com.intellij.psi.impl.source.PsiFileWithStubSupport;
+import com.intellij.psi.stubs.StubBase;
+import com.intellij.psi.stubs.StubElement;
+import com.intellij.psi.stubs.StubTree;
+import com.intellij.util.ObjectUtils;
+import com.intellij.util.containers.IntIntHashMap;
+import gnu.trove.TByteArrayList;
+import gnu.trove.TIntArrayList;
+import gnu.trove.TIntIntHashMap;
+import org.jetbrains.annotations.NotNull;
+
+import java.util.List;
+
+/**
+ * @author peter
+ */
+class AnchorRepository {
+  private static final int MAX_BYTE_VALUE = 255;
+  private final TIntArrayList myFileIds = new TIntArrayList(80000);
+  private final TByteArrayList myShortStubIds = new TByteArrayList(80000);
+  private final TIntIntHashMap myLongStubIds = new IntIntHashMap();
+
+  int registerClass(int fileId, int stubId) {
+    int anchorId = myFileIds.size();
+    myFileIds.add(fileId);
+    if (stubId < MAX_BYTE_VALUE) {
+      myShortStubIds.add((byte)stubId);
+    } else {
+      myShortStubIds.add((byte)MAX_BYTE_VALUE);
+      myLongStubIds.put(anchorId, stubId);
+    }
+    return anchorId;
+  }
+
+  StubClassAnchor getAnchor(int anchorId) {
+    return new StubClassAnchor(anchorId, getFileId(anchorId), getStubId(anchorId));
+  }
+
+  int getFileId(int anchorId) {
+    return myFileIds.get(anchorId);
+  }
+
+  int getStubId(int anchorId) {
+    int stubId = Byte.toUnsignedInt(myShortStubIds.get(anchorId));
+    return stubId < MAX_BYTE_VALUE ? stubId : myLongStubIds.get(anchorId);
+  }
+
+  int size() {
+    return myFileIds.size();
+  }
+
+  void trimToSize() {
+    myFileIds.trimToSize();
+    myShortStubIds.trimToSize();
+    myLongStubIds.trimToSize();
+  }
+
+  @NotNull
+  static VirtualFile retrieveFile(int fileId) {
+    return ObjectUtils.assertNotNull(PersistentFS.getInstance().findFileById(fileId));
+  }
+
+  static String anchorToString(int stubId, int fileId) {
+    return stubId + " in " + retrieveFile(fileId).getPath();
+  }
+
+  @NotNull
+  PsiClass retrieveClass(@NotNull Project project, int anchorId) {
+    return retrieveClass(project, getFileId(anchorId), getStubId(anchorId));
+  }
+
+  @NotNull
+  static PsiClass retrieveClass(@NotNull Project project, int fileId, int stubId) {
+    PsiFile psiFile = PsiManager.getInstance(project).findFile(retrieveFile(fileId));
+    assert psiFile != null : anchorToString(stubId, fileId);
+    PsiElement element = restoreFromStubIndex((PsiFileWithStubSupport)psiFile, stubId);
+    if (!(element instanceof PsiClass)) {
+      throw new AssertionError(anchorToString(stubId, fileId) + "; " + psiFile);
+    }
+    return (PsiClass)element;
+  }
+
+  private static PsiElement restoreFromStubIndex(PsiFileWithStubSupport fileImpl, int index) {
+    StubTree tree = fileImpl.getStubTree();
+
+    boolean foreign = tree == null;
+    if (foreign) {
+      if (fileImpl instanceof PsiFileImpl) {
+        tree = ((PsiFileImpl)fileImpl).calcStubTree();
+      }
+      else {
+        return null;
+      }
+    }
+
+    List<StubElement<?>> list = tree.getPlainList();
+    if (index >= list.size()) {
+      return null;
+    }
+    StubElement stub = list.get(index);
+
+    if (foreign) {
+      final PsiElement cachedPsi = ((StubBase)stub).getCachedPsi();
+      if (cachedPsi != null) return cachedPsi;
+
+      final ASTNode ast = fileImpl.findTreeForStub(tree, stub);
+      return ast != null ? ast.getPsi() : null;
+    }
+    return stub.getPsi();
+  }
+
+}

java/java-impl/src/com/intellij/psi/stubsHierarchy/impl/HierarchyServiceImpl.java

@@ -36,7 +36,7 @@ import org.jetbrains.annotations.NotNull;
 import java.util.BitSet;
 
 public class HierarchyServiceImpl extends HierarchyService {
-  private static final SingleClassHierarchy EMPTY_HIERARCHY = new SingleClassHierarchy(Symbol.ClassSymbol.EMPTY_ARRAY);
+  private static final SingleClassHierarchy EMPTY_HIERARCHY = new SingleClassHierarchy(Symbol.ClassSymbol.EMPTY_ARRAY, new AnchorRepository());
   private final Project myProject;
   private final CachedValue<SingleClassHierarchy> myHierarchy;
 
@@ -69,16 +69,16 @@ public class HierarchyServiceImpl extends HierarchyService {
     StubEnter stubEnter = new StubEnter(symbols);
     IdSets idSets = IdSets.getIdSets(myProject);
 
-    loadUnits(idSets.libraryFiles, StubHierarchyIndex.BINARY_FILES, symbols.myNameEnvironment, stubEnter);
+    loadUnits(idSets.libraryFiles, StubHierarchyIndex.BINARY_FILES, stubEnter);
     stubEnter.connect1();
 
-    loadUnits(idSets.sourceFiles, StubHierarchyIndex.SOURCE_FILES, symbols.myNameEnvironment, stubEnter);
+    loadUnits(idSets.sourceFiles, StubHierarchyIndex.SOURCE_FILES, stubEnter);
     stubEnter.connect2();
 
     return symbols.createHierarchy();
   }
 
-  private void loadUnits(BitSet files, int indexKey, NameEnvironment names, StubEnter stubEnter) {
+  private void loadUnits(BitSet files, int indexKey, StubEnter stubEnter) {
     ProgressIndicator indicator = ProgressIndicatorProvider.getGlobalProgressIndicator();
 
     FileBasedIndexImpl index = (FileBasedIndexImpl)FileBasedIndex.getInstance();

java/java-impl/src/com/intellij/psi/stubsHierarchy/impl/SingleClassHierarchy.java

@@ -25,9 +25,14 @@ import com.intellij.psi.stubsHierarchy.ClassHierarchy;
 import com.intellij.psi.stubsHierarchy.SmartClassAnchor;
 import com.intellij.psi.stubsHierarchy.impl.Symbol.ClassSymbol;
 import com.intellij.util.containers.ContainerUtil;
+import org.apache.commons.lang.ArrayUtils;
 import org.jetbrains.annotations.NotNull;
 
-import java.util.*;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 /**
  * Compact representation of total hierarchy of JVM classes.
@@ -36,15 +41,16 @@ public class SingleClassHierarchy extends ClassHierarchy {
   private final BitSet myCoveredFiles;
   private final BitSet myAmbiguousSupers;
   private final BitSet myAnonymous;
-  private final StubClassAnchor[] myClassAnchors;
-  private final StubClassAnchor[] myClassAnchorsByFileIds;
+  private final AnchorRepository myClassAnchors;
+  private final int[] myClassAnchorsByFileIds;
   private int[] mySubtypes;
   private int[] mySubtypeStarts;
 
-  public SingleClassHierarchy(ClassSymbol[] classSymbols) {
+  public SingleClassHierarchy(ClassSymbol[] classSymbols, AnchorRepository classAnchors) {
+    classAnchors.trimToSize();
+    myClassAnchors = classAnchors;
     myCoveredFiles = calcCoveredFiles(classSymbols);
-    myClassAnchors = ContainerUtil.map2Array(classSymbols, StubClassAnchor.class, symbol -> symbol.myClassAnchor);
-    myClassAnchorsByFileIds = mkByFileId(myClassAnchors);
+    myClassAnchorsByFileIds = mkByFileId();
     excludeUncoveredFiles(classSymbols);
     connectSubTypes(classSymbols);
     myAmbiguousSupers = calcAmbiguousSupers(classSymbols);
@@ -56,7 +62,7 @@ public class SingleClassHierarchy extends ClassHierarchy {
     BitSet ambiguousSupers = new BitSet();
     for (ClassSymbol symbol : classSymbols) {
       if (!symbol.isHierarchyIncomplete() && symbol.hasAmbiguousSupers()) {
-        ambiguousSupers.set(symbol.myClassAnchor.myId);
+        ambiguousSupers.set(symbol.myAnchorId);
       }
     }
     return ambiguousSupers;
@@ -67,20 +73,21 @@ public class SingleClassHierarchy extends ClassHierarchy {
     BitSet answer = new BitSet();
     for (ClassSymbol symbol : classSymbols) {
       if (!symbol.isHierarchyIncomplete() && symbol.myShortName == NamesEnumerator.NO_NAME) {
-        answer.set(symbol.myClassAnchor.myId);
+        answer.set(symbol.myAnchorId);
       }
     }
     return answer;
   }
 
   @NotNull
-  private static BitSet calcCoveredFiles(ClassSymbol[] classSymbols) {
+  private BitSet calcCoveredFiles(ClassSymbol[] classSymbols) {
     BitSet problematicFiles = new BitSet();
     BitSet coveredFiles = new BitSet();
     for (ClassSymbol symbol : classSymbols) {
-      coveredFiles.set(symbol.myClassAnchor.myFileId);
+      int fileId = myClassAnchors.getFileId(symbol.myAnchorId);
+      coveredFiles.set(fileId);
       if (symbol.isHierarchyIncomplete()) {
-        problematicFiles.set(symbol.myClassAnchor.myFileId);
+        problematicFiles.set(fileId);
       }
     }
     coveredFiles.andNot(problematicFiles);
@@ -94,13 +101,13 @@ public class SingleClassHierarchy extends ClassHierarchy {
   @Override
   @NotNull
   public List<StubClassAnchor> getCoveredClasses() {
-    return ContainerUtil.filter(myClassAnchors, this::isCovered);
+    return ContainerUtil.filter(getAllClasses(), this::isCovered);
   }
 
   @Override
   @NotNull
   public List<StubClassAnchor> getAllClasses() {
-    return Collections.unmodifiableList(Arrays.asList(myClassAnchors));
+    return IntStream.range(0, myClassAnchors.size()).boxed().map(myClassAnchors::getAnchor).collect(Collectors.toList());
   }
 
   @NotNull
@@ -129,7 +136,7 @@ public class SingleClassHierarchy extends ClassHierarchy {
     }
     StubClassAnchor[] result = new StubClassAnchor[length];
     for (int i = 0; i < length; i++) {
-      result[i] = myClassAnchors[mySubtypes[start + i]];
+      result[i] = myClassAnchors.getAnchor(mySubtypes[start + i]);
     }
     return result;
   }
@@ -163,21 +170,26 @@ public class SingleClassHierarchy extends ClassHierarchy {
     };
   }
 
-  private static StubClassAnchor[] mkByFileId(final StubClassAnchor[] classAnchors) {
-    StubClassAnchor[] result = new StubClassAnchor[classAnchors.length];
-    StubClassAnchor lastProcessedAnchor = null;
-    int i = 0;
-    for (StubClassAnchor classAnchor : classAnchors) {
-      if (lastProcessedAnchor == null || lastProcessedAnchor.myFileId != classAnchor.myFileId) {
+  @NotNull
+  private Integer[] getAnchorsFromDistinctFiles() {
+    if (myClassAnchors.size() == 0) return new Integer[0];
+
+    Integer[] result = new Integer[myClassAnchors.size()];
+    result[0] = 0;
+    int i = 1;
+    for (int classAnchor = 1; classAnchor < result.length; classAnchor++) {
+      if (myClassAnchors.getFileId(classAnchor - 1) != myClassAnchors.getFileId(classAnchor)) {
         result[i++] = classAnchor;
       }
-      lastProcessedAnchor = classAnchor;
     }
-    // compacting
-    result = Arrays.copyOf(result, i);
+    return Arrays.copyOf(result, i);
+  }
 
-    Arrays.sort(result, (a1, a2) -> Integer.compare(a1.myFileId, a2.myFileId));
-    return result;
+  private int[] mkByFileId() {
+    // using a boxed array since there seems to be no easy way to sort int[] with custom comparator
+    Integer[] ids = getAnchorsFromDistinctFiles();
+    Arrays.sort(ids, (a1, a2) -> Integer.compare(myClassAnchors.getFileId(a1), myClassAnchors.getFileId(a2)));
+    return ArrayUtils.toPrimitive(ids);
   }
 
   private void connectSubTypes(ClassSymbol[] classSymbols) {
@@ -196,7 +208,7 @@ public class SingleClassHierarchy extends ClassHierarchy {
     for (int subTypeId = 0; subTypeId < classSymbols.length; subTypeId++) {
       ClassSymbol subType = classSymbols[subTypeId];
       for (ClassSymbol superType : subType.rawSuperClasses()) {
-        int superTypeId = superType.myClassAnchor.myId;
+        int superTypeId = superType.myAnchorId;
         subtypes[starts[superTypeId] + filled[superTypeId]] = subTypeId;
         filled[superTypeId] += 1;
       }
@@ -208,7 +220,7 @@ public class SingleClassHierarchy extends ClassHierarchy {
 
   private void excludeUncoveredFiles(ClassSymbol[] classSymbols) {
     for (ClassSymbol symbol : classSymbols) {
-      if (!isCovered(symbol.myClassAnchor)) {
+      if (!myCoveredFiles.get(myClassAnchors.getFileId(symbol.myAnchorId))) {
         symbol.markHierarchyIncomplete();
       }
     }
@@ -218,8 +230,7 @@ public class SingleClassHierarchy extends ClassHierarchy {
     int[] sizes = new int[classSymbols.length];
     for (ClassSymbol subType : classSymbols) {
       for (ClassSymbol superType : subType.rawSuperClasses()) {
-        int superTypeId = superType.myClassAnchor.myId;
-        sizes[superTypeId] += 1;
+        sizes[superType.myAnchorId] += 1;
       }
     }
     return sizes;
@@ -234,31 +245,29 @@ public class SingleClassHierarchy extends ClassHierarchy {
   }
 
   private StubClassAnchor forPsiClass(int fileId, PsiClass psiClass) {
-    StubClassAnchor anchor = getFirst(fileId, myClassAnchorsByFileIds);
-    if (anchor == null) {
+    int id = getFirst(fileId);
+    if (id == -1) {
       return null;
     }
-    int id = anchor.myId;
-    while (id < myClassAnchors.length) {
-      StubClassAnchor candidate = myClassAnchors[id];
-      if (candidate.myFileId != fileId)
-        return null;
-      if (psiClass.isEquivalentTo(candidate.retrieveClass(psiClass.getProject())))
-        return candidate;
+    while (id < myClassAnchors.size() && myClassAnchors.getFileId(id) == fileId) {
+      if (psiClass.isEquivalentTo(myClassAnchors.retrieveClass(psiClass.getProject(), id))) {
+        return myClassAnchors.getAnchor(id);
+      }
       id++;
     }
     return null;
   }
 
-  private static StubClassAnchor getFirst(int fileId, StubClassAnchor[] byFileIds) {
+  private int getFirst(int fileId) {
     int lo = 0;
-    int hi = byFileIds.length - 1;
+    int hi = myClassAnchorsByFileIds.length - 1;
     while (lo <= hi) {
       int mid = lo + (hi - lo) / 2;
-      if      (fileId < byFileIds[mid].myFileId) hi = mid - 1;
-      else if (fileId > byFileIds[mid].myFileId) lo = mid + 1;
-      else return byFileIds[mid];
+      int midFileId = myClassAnchors.getFileId(myClassAnchorsByFileIds[mid]);
+      if      (fileId < midFileId) hi = mid - 1;
+      else if (fileId > midFileId) lo = mid + 1;
+      else return myClassAnchorsByFileIds[mid];
     }
-    return null;
+    return -1;
   }
 }

java/java-impl/src/com/intellij/psi/stubsHierarchy/impl/StubClassAnchor.java

@@ -15,24 +15,12 @@
  */
 package com.intellij.psi.stubsHierarchy.impl;
 
-import com.intellij.lang.ASTNode;
 import com.intellij.openapi.project.Project;
 import com.intellij.openapi.vfs.VirtualFile;
-import com.intellij.openapi.vfs.newvfs.persistent.PersistentFS;
 import com.intellij.psi.PsiClass;
-import com.intellij.psi.PsiElement;
-import com.intellij.psi.PsiFile;
-import com.intellij.psi.PsiManager;
-import com.intellij.psi.impl.source.PsiFileImpl;
-import com.intellij.psi.impl.source.PsiFileWithStubSupport;
-import com.intellij.psi.stubs.StubBase;
-import com.intellij.psi.stubs.StubElement;
-import com.intellij.psi.stubs.StubTree;
 import com.intellij.psi.stubsHierarchy.SmartClassAnchor;
 import org.jetbrains.annotations.NotNull;
 
-import java.util.List;
-
 public class StubClassAnchor extends SmartClassAnchor {
   public static final StubClassAnchor[] EMPTY_ARRAY = new StubClassAnchor[0];
 
@@ -49,50 +37,13 @@ public class StubClassAnchor extends SmartClassAnchor {
   @Override
   @NotNull
   public VirtualFile retrieveFile() {
-    VirtualFile file = PersistentFS.getInstance().findFileById(myFileId);
-    assert file != null : this;
-    return file;
+    return AnchorRepository.retrieveFile(myFileId);
   }
 
   @Override
   @NotNull
   public PsiClass retrieveClass(@NotNull Project project) {
-    PsiFile psiFile = PsiManager.getInstance(project).findFile(retrieveFile());
-    assert psiFile != null : this;
-    PsiElement element = restoreFromStubIndex((PsiFileWithStubSupport)psiFile, myStubId);
-    if (!(element instanceof PsiClass)) {
-      throw new AssertionError(this + "; " + psiFile);
-    }
-    return (PsiClass)element;
-  }
-
-  private static PsiElement restoreFromStubIndex(PsiFileWithStubSupport fileImpl, int index) {
-    StubTree tree = fileImpl.getStubTree();
-
-    boolean foreign = tree == null;
-    if (foreign) {
-      if (fileImpl instanceof PsiFileImpl) {
-        tree = ((PsiFileImpl)fileImpl).calcStubTree();
-      }
-      else {
-        return null;
-      }
-    }
-
-    List<StubElement<?>> list = tree.getPlainList();
-    if (index >= list.size()) {
-      return null;
-    }
-    StubElement stub = list.get(index);
-
-    if (foreign) {
-      final PsiElement cachedPsi = ((StubBase)stub).getCachedPsi();
-      if (cachedPsi != null) return cachedPsi;
-
-      final ASTNode ast = fileImpl.findTreeForStub(tree, stub);
-      return ast != null ? ast.getPsi() : null;
-    }
-    return stub.getPsi();
+    return AnchorRepository.retrieveClass(project, myFileId, myStubId);
   }
 
   @Override
@@ -102,6 +53,16 @@ public class StubClassAnchor extends SmartClassAnchor {
 
   @Override
   public String toString() {
-    return myStubId + " in " + retrieveFile().getPath();
+    return AnchorRepository.anchorToString(myStubId, myFileId);
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (!(o instanceof StubClassAnchor)) return false;
+    StubClassAnchor anchor = (StubClassAnchor)o;
+    return myId == anchor.myId &&
+           myFileId == anchor.myFileId &&
+           myStubId == anchor.myStubId;
   }
 }

java/java-impl/src/com/intellij/psi/stubsHierarchy/impl/Symbol.java

@@ -88,25 +88,30 @@ public abstract class Symbol {
     private static final int CONNECT_STARTED = 1 << 21;
     public static final ClassSymbol[] EMPTY_ARRAY = new ClassSymbol[0];
 
-    final StubClassAnchor myClassAnchor;
+    final int myAnchorId;
     @CompactArray({QualifiedName.class, ClassSymbol.class}) Object mySuperClasses;
     UnitInfo myUnitInfo;
 
-    ClassSymbol(StubClassAnchor classAnchor,
+    ClassSymbol(int anchorId,
                 int flags,
                 Symbol owner,
                 int name,
                 UnitInfo unitInfo,
                 @CompactArray(QualifiedName.class) Object supers) {
       super(flags | IndexTree.CLASS, owner, name);
-      this.myClassAnchor = classAnchor;
+      this.myAnchorId = anchorId;
       this.mySuperClasses = supers;
       this.myUnitInfo = unitInfo;
     }
 
+    @Override
+    public int hashCode() {
+      return myAnchorId;
+    }
+
     @Override
     public String toString() {
-      return myClassAnchor.toString();
+      return String.valueOf(myAnchorId);
     }
 
     void connect(StubHierarchyConnector connector) {

java/java-impl/src/com/intellij/psi/stubsHierarchy/impl/Symbols.java

@@ -12,6 +12,7 @@ import java.util.List;
 public class Symbols {
   public final PackageSymbol myRootPackage;
   protected final NameEnvironment myNameEnvironment = new NameEnvironment();
+  private final AnchorRepository myClassAnchors = new AnchorRepository();
 
   private List<ClassSymbol> myClassSymbols = new ArrayList<>(0x8000);
   // fullName -> PackageSymbol
@@ -70,8 +71,8 @@ public class Symbols {
                          UnitInfo info,
                          @CompactArray(QualifiedName.class) Object supers,
                          @Nullable QualifiedName qualifiedName) {
-    StubClassAnchor stubClassAnchor = new StubClassAnchor(myClassSymbols.size(), fileId, stubId);
-    ClassSymbol c = new ClassSymbol(stubClassAnchor, flags, owner, shortName, info, supers);
+    int anchorId = myClassAnchors.registerClass(fileId, stubId);
+    ClassSymbol c = new ClassSymbol(anchorId, flags, owner, shortName, info, supers);
     myClassSymbols.add(c);
     if (qualifiedName != null) {
       putClassByName(c, qualifiedName.myId);
@@ -115,6 +116,6 @@ public class Symbols {
   SingleClassHierarchy createHierarchy() {
     ClassSymbol[] array = myClassSymbols.toArray(ClassSymbol.EMPTY_ARRAY);
     myClassSymbols = null;
-    return new SingleClassHierarchy(array);
+    return new SingleClassHierarchy(array, myClassAnchors);
   }
 }

java/java-impl/src/com/intellij/psi/stubsHierarchy/impl/test/TestStubHierarchyAction.java

@@ -89,8 +89,8 @@ public class TestStubHierarchyAction extends InheritanceAction {
 
       if (!stubSupers.containsAll(psiSupers)) {
         psiSupers.removeAll(stubSupers);
-        LOG.info("Inconsistent hierarchy for " + classInfo(subClass) +
-                 "\n  missing " + psiSupers.size() + ": " + StringUtil.join(psiSupers, TestHierarchy::classInfo, ", ")
+        throw new AssertionError("Inconsistent hierarchy for " + classInfo(subClass) +
+                  "\n  missing " + psiSupers.size() + ": " + StringUtil.join(psiSupers, TestHierarchy::classInfo, ", ")
         );
       }
     }