stub hierarchy: serialize hashes instead of names for faster reading, smaller index data and a very unlikely collision probability

2026-09-27 10:03:11 +07:00 · 2016-06-24 17:02:24 +02:00
parent 7f70956afe
commit b4a2271239
7 changed files with 149 additions and 149 deletions
@@ -20,7 +20,6 @@ import com.intellij.openapi.progress.ProgressIndicatorProvider;
 import com.intellij.openapi.project.Project;
 import com.intellij.openapi.roots.ProjectFileIndex;
 import com.intellij.openapi.roots.ProjectRootManager;
-import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.openapi.vfs.VirtualFileManager;
 import com.intellij.openapi.vfs.VirtualFileWithId;
 import com.intellij.psi.impl.java.stubs.hierarchy.IndexTree;
@@ -89,7 +88,7 @@ public class HierarchyServiceImpl extends HierarchyService {
      public boolean process(int fileId, IndexTree.Unit unit) {
        if (indicator != null && ++count % 128 == 0) indicator.checkCanceled();
        if (files.get(fileId)) {
-          QualifiedName pkg = StringUtil.isEmpty(unit.myPackageId) ? null : names.fromString(unit.myPackageId, true);
+          QualifiedName pkg = unit.myPackageName.length == 0 ? null : names.myNamesEnumerator.getFullName(unit.myPackageName, true);
          stubEnter.unitEnter(Translator.internNames(names, unit, fileId, pkg));
        }
        return true;
@@ -16,14 +16,12 @@
 package com.intellij.psi.stubsHierarchy.impl;

 import com.intellij.openapi.util.UserDataHolderBase;
-import com.intellij.openapi.util.text.StringUtil;
+import com.intellij.psi.impl.java.stubs.hierarchy.IndexTree;
 import org.jetbrains.annotations.Nullable;

 import java.util.Arrays;
-import java.util.List;

 public class NameEnvironment extends UserDataHolderBase {
-
  public final QualifiedName empty;
  public final QualifiedName java_lang_Object;
  public final QualifiedName java_lang_Enum;
@@ -40,22 +38,7 @@ public class NameEnvironment extends UserDataHolderBase {

  @Nullable
  public QualifiedName fromString(String s, boolean create) {
-    List<String> comps = StringUtil.split(s, ".");
-    int[] ids = new int[comps.size()];
-    for (int i = 0; i < comps.size(); i++) {
-      int name = simpleName(comps.get(i), create);
-      if (name == NamesEnumerator.NO_NAME) {
-        return null;
-      }
-      ids[i] = name;
-    }
-    return myNamesEnumerator.getFullName(ids, create);
-  }
-
-  public int simpleName(String s, boolean create) {
-    if (s == null)
-      return NamesEnumerator.NO_NAME;
-    return myNamesEnumerator.getSimpleName(s, create);
+    return myNamesEnumerator.getFullName(IndexTree.hashQualifiedName(s), create);
  }

  public QualifiedName prefix(QualifiedName name) {
@@ -15,31 +15,15 @@
 */
 package com.intellij.psi.stubsHierarchy.impl;

-import com.intellij.util.ArrayUtil;
 import gnu.trove.TObjectHashingStrategy;
 import gnu.trove.TObjectIntHashMap;
-import org.jetbrains.annotations.Nullable;

 import java.util.Arrays;

 public class NamesEnumerator {
  final static int NO_NAME = 0;

-  private TObjectIntHashMap<byte[]> myAsciiMap = new TObjectIntHashMap<byte[]>(new TObjectHashingStrategy<byte[]>() {
-    @Override
-    public int computeHashCode(byte[] object) {
-      return Arrays.hashCode(object);
-    }
-
-    @Override
-    public boolean equals(byte[] o1, byte[] o2) {
-      return Arrays.equals(o1, o2);
-    }
-  });
-
-  private TObjectIntHashMap<String> myNonAsciiMap = new TObjectIntHashMap<String>();
-
-  TObjectIntHashMap<int[]> fullNameMap = new TObjectIntHashMap<int[]>(new TObjectHashingStrategy<int[]>() {
+  private final TObjectIntHashMap<int[]> myFullNameMap = new TObjectIntHashMap<int[]>(new TObjectHashingStrategy<int[]>() {
    @Override
    public int computeHashCode(int[] object) {
      return Arrays.hashCode(object);
@@ -50,38 +34,17 @@ public class NamesEnumerator {
      return Arrays.equals(o1, o2);
    }
  });
-
  private QualifiedName[] myQualifiedNames = new QualifiedName[0x8000];

  QualifiedName qualifiedName(int id) {
    return myQualifiedNames[id];
  }

-  public int getSimpleName(String s, boolean create) {
-    byte[] bytes = convertToBytesIfAsciiString(s);
-    if (bytes != null) {
-      int id = myAsciiMap.get(bytes);
-      if (id == 0 && create) {
-        id = myAsciiMap.size() + myNonAsciiMap.size() + 1;
-        myAsciiMap.put(bytes, id);
-      }
-      return id;
-    }
-    else {
-      int id = myNonAsciiMap.get(s);
-      if (id == 0 && create) {
-        id = myAsciiMap.size() + myNonAsciiMap.size() + 1;
-        myNonAsciiMap.put(s, id);
-      }
-      return id;
-    }
-  }
-
  public QualifiedName getFullName(int[] ids, boolean create) {
-    int id = fullNameMap.get(ids);
+    int id = myFullNameMap.get(ids);
    if (id == 0 && create) {
-      id = fullNameMap.size() + 1;
-      fullNameMap.put(ids, id);
+      id = myFullNameMap.size() + 1;
+      myFullNameMap.put(ids, id);
      ensureFullCapacity(id);
      myQualifiedNames[id] = new QualifiedName(id, ids);
    }
@@ -104,20 +67,4 @@ public class NamesEnumerator {
    return currentLength;
  }

-  @Nullable
-  public static byte[] convertToBytesIfAsciiString(CharSequence name) {
-    int length = name.length();
-    if (length == 0) return ArrayUtil.EMPTY_BYTE_ARRAY;
-
-    byte[] bytes = new byte[length];
-    for (int i = 0; i < length; i++) {
-      char c = name.charAt(i);
-      if (c >= 128) {
-        return null;
-      }
-      bytes[i] = (byte)c;
-    }
-    return bytes;
-  }
-
 }
@@ -74,7 +74,7 @@ public class StubHierarchyIndex extends FileBasedIndexExtension<Integer, IndexTr

  @Override
  public int getVersion() {
-    return IndexTree.STUB_HIERARCHY_ENABLED ? 3 + Arrays.stream(ourIndexers).mapToInt(StubHierarchyIndexer::getVersion).sum() : 0;
+    return IndexTree.STUB_HIERARCHY_ENABLED ? 4 + Arrays.stream(ourIndexers).mapToInt(StubHierarchyIndexer::getVersion).sum() : 0;
  }

  @NotNull
@@ -16,15 +16,12 @@
 package com.intellij.psi.stubsHierarchy.impl;

 import com.intellij.openapi.util.Key;
-import com.intellij.openapi.util.text.StringUtil;
-import com.intellij.psi.PsiNameHelper;
 import com.intellij.psi.impl.java.stubs.hierarchy.IndexTree;
 import com.intellij.psi.stubsHierarchy.stubs.*;
 import com.intellij.util.BitUtil;
 import org.jetbrains.annotations.NotNull;

 import java.util.ArrayList;
-import java.util.List;

 public class Translator {
  private static final Key<long[]> DEFAULT_JAVA_IMPORTS_KEY = Key.create("java_imports");
@@ -91,11 +88,10 @@ public class Translator {
  }

  private static ClassDeclaration processClassDecl(NameEnvironment nameEnvironment, int fileId, IndexTree.ClassDecl def) {
-    String stubName = def.myName;
-    int name = stubName == null ? 0 : nameEnvironment.simpleName(stubName, true);
+    int name = def.myName;
    ArrayList<QualifiedName> superList = new ArrayList<QualifiedName>();
-    for (String aSuper : def.mySupers) {
-      superList.add(id(nameEnvironment, aSuper));
+    for (int[] aSuper : def.mySupers) {
+      superList.add(nameEnvironment.concat(aSuper, true));
    }
    if (BitUtil.isSet(def.myMods, IndexTree.ENUM)) {
      superList.add(nameEnvironment.java_lang_Enum);
@@ -130,21 +126,8 @@ public class Translator {
  }

  private static long processImport(NameEnvironment nameEnvironment, IndexTree.Import anImport) {
-    QualifiedName fullname = nameEnvironment.fromString(anImport.myFullname, true);
-    int aliasName = anImport.myAlias == null ? 0 : nameEnvironment.simpleName(anImport.myAlias, true);
-    return Import.mkImport(fullname, anImport.myStaticImport, anImport.myOnDemand, aliasName);
+    QualifiedName fullname = nameEnvironment.myNamesEnumerator.getFullName(anImport.myFullname, true);
+    return Import.mkImport(fullname, anImport.myStaticImport, anImport.myOnDemand, anImport.myAlias);
  }

-  private static QualifiedName id(NameEnvironment nameEnvironment, String s) {
-    s = PsiNameHelper.getQualifiedClassName(s, true);
-    List<String> ids = StringUtil.split(s, ".");
-    int[] comps = new int[ids.size()];
-    int i = 0;
-    for (String id : ids) {
-      int name = nameEnvironment.simpleName(id, true);
-      comps[i] = name;
-      i++;
-    }
-    return nameEnvironment.concat(comps, true);
-  }
 }
@@ -16,6 +16,7 @@
 package com.intellij.psi.impl.java.stubs.index;

 import com.intellij.psi.impl.java.stubs.hierarchy.IndexTree;
+import com.intellij.util.BitUtil;
 import com.intellij.util.io.DataExternalizer;
 import com.intellij.util.io.DataInputOutputUtil;
 import org.jetbrains.annotations.NotNull;
@@ -24,19 +25,34 @@ import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;

+import static com.intellij.psi.impl.java.stubs.index.JavaUnitDescriptor.ImportFlags.*;
+
 public class JavaUnitDescriptor implements DataExternalizer<IndexTree.Unit> {
  public static final JavaUnitDescriptor INSTANCE = new JavaUnitDescriptor();

+  private static void writeIntArray(DataOutput out, int[] array) throws IOException {
+    DataInputOutputUtil.writeINT(out, array.length);
+    for (int i : array) {
+      out.writeInt(i);
+    }
+  }
+  private static int[] readIntArray(DataInput in) throws IOException {
+    int length = DataInputOutputUtil.readINT(in);
+    int[] result = new int[length];
+    for (int i = 0; i < length; i++) {
+      result[i] = in.readInt();
+    }
+    return result;
+  }
+
  @Override
  public void save(@NotNull DataOutput out, IndexTree.Unit value) throws IOException {
-    out.writeUTF(value.myPackageId);
+    writeIntArray(out, value.myPackageName);
    out.writeByte(value.myUnitType);
    if (value.myUnitType != IndexTree.BYTECODE) {
      DataInputOutputUtil.writeINT(out, value.imports.length);
      for (IndexTree.Import anImport : value.imports) {
-        out.writeUTF(anImport.myFullname);
-        out.writeBoolean(anImport.myStaticImport);
-        out.writeBoolean(anImport.myOnDemand);
+        writeImport(out, anImport);
      }
    }
    // class Declaration
@@ -47,12 +63,12 @@ public class JavaUnitDescriptor implements DataExternalizer<IndexTree.Unit> {
  }

  private void saveClassDecl(@NotNull DataOutput out, IndexTree.ClassDecl value) throws IOException {
-    out.writeInt(value.myStubId);
+    DataInputOutputUtil.writeINT(out, value.myStubId);
    DataInputOutputUtil.writeINT(out, value.myMods);
-    out.writeUTF(value.myName == null ? "" : value.myName);
+    out.writeInt(value.myName);
    DataInputOutputUtil.writeINT(out, value.mySupers.length);
-    for (String aSuper : value.mySupers) {
-      out.writeUTF(aSuper);
+    for (int[] aSuper : value.mySupers) {
+      writeIntArray(out, aSuper);
    }
    DataInputOutputUtil.writeINT(out, value.myDecls.length);
    for (IndexTree.Decl def : value.myDecls) {
@@ -76,13 +92,13 @@ public class JavaUnitDescriptor implements DataExternalizer<IndexTree.Unit> {

  @Override
  public IndexTree.Unit read(@NotNull DataInput in) throws IOException {
-    String pid = in.readUTF();
+    int[] pid = readIntArray(in);
    byte type = in.readByte();
    IndexTree.Import[] imports = IndexTree.Import.EMPTY_ARRAY;
    if (type != IndexTree.BYTECODE) {
      imports = new IndexTree.Import[DataInputOutputUtil.readINT(in)];
      for (int i = 0; i < imports.length; i++) {
-        imports[i] = new IndexTree.Import(in.readUTF(), in.readBoolean(), in.readBoolean(), null);
+        imports[i] = readImport(in);
      }
    }
    IndexTree.ClassDecl[] classes = new IndexTree.ClassDecl[DataInputOutputUtil.readINT(in)];
@@ -93,15 +109,12 @@ public class JavaUnitDescriptor implements DataExternalizer<IndexTree.Unit> {
  }

  private IndexTree.ClassDecl readClassDecl(DataInput in) throws IOException {
-    int stubId = in.readInt();
+    int stubId = DataInputOutputUtil.readINT(in);
    int mods = DataInputOutputUtil.readINT(in);
-    String name = in.readUTF();
-    if (name.isEmpty()) {
-      name = null;
-    }
-    String[] supers = new String[DataInputOutputUtil.readINT(in)];
+    int name = in.readInt();
+    int[][] supers = new int[DataInputOutputUtil.readINT(in)][];
    for (int i = 0; i < supers.length; i++) {
-      supers[i] = in.readUTF();
+      supers[i] = readIntArray(in);
    }
    IndexTree.Decl[] decls = new IndexTree.Decl[DataInputOutputUtil.readINT(in)];
    for (int i = 0; i < decls.length; i++) {
@@ -124,4 +137,32 @@ public class JavaUnitDescriptor implements DataExternalizer<IndexTree.Unit> {
    }
  }

+  interface ImportFlags {
+    int IS_STATIC = 1;
+    int IS_ON_DEMAND = 2;
+    int HAS_ALIAS = 4;
+  }
+
+  private static void writeImport(@NotNull DataOutput out, IndexTree.Import anImport) throws IOException {
+    writeIntArray(out, anImport.myFullname);
+    boolean hasAlias = anImport.myAlias != 0;
+    int flags = 0;
+    flags = BitUtil.set(flags, IS_STATIC, anImport.myStaticImport);
+    flags = BitUtil.set(flags, IS_ON_DEMAND, anImport.myOnDemand);
+    flags = BitUtil.set(flags, HAS_ALIAS, hasAlias);
+    out.writeByte(flags);
+    if (hasAlias) {
+      out.writeInt(anImport.myAlias);
+    }
+  }
+
+  @NotNull
+  private static IndexTree.Import readImport(@NotNull DataInput in) throws IOException {
+    int[] fullname = readIntArray(in);
+    int flags = in.readByte();
+    return new IndexTree.Import(fullname,
+                                BitUtil.isSet(flags, IS_STATIC), BitUtil.isSet(flags, IS_ON_DEMAND),
+                                BitUtil.isSet(flags, HAS_ALIAS) ? in.readInt() : 0);
+  }
+
 }
@@ -18,14 +18,18 @@ package com.intellij.psi.impl.java.stubs.hierarchy;

 import com.intellij.openapi.util.registry.Registry;
 import com.intellij.openapi.util.text.StringUtil;
+import com.intellij.psi.PsiNameHelper;
+import com.intellij.util.ArrayUtil;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;

 import java.util.Arrays;
+import java.util.List;

 public class IndexTree {
  public static final boolean STUB_HIERARCHY_ENABLED = Registry.is("java.hierarchy.service");

+  @SuppressWarnings("PointlessBitwiseExpression")
  public final static int PACKAGE = 1 << 0;
  public final static int CLASS = 1 << 1;
  public static final int STATIC       = 1 << 3;
@@ -39,17 +43,52 @@ public class IndexTree {
  public static final byte JAVA = 1;
  public static final byte GROOVY = 2;

+  private static int hashIdentifier(@Nullable String s) {
+    if (s == null) return 0;
+
+    // not using String.hashCode because this way there's less collisions for short package names like 'com'
+    int hash = 0;
+    for (int i = 0; i < s.length(); i++) {
+      hash = hash * 239 + s.charAt(i);
+    }
+    return hash;
+  }
+
+  public static int[] hashQualifiedName(@NotNull String qName) {
+    qName = PsiNameHelper.getQualifiedClassName(qName, true);
+    if (qName.isEmpty()) return ArrayUtil.EMPTY_INT_ARRAY;
+
+    List<String> components = StringUtil.split(qName, ".");
+    int[] result = new int[components.size()];
+    for (int i = 0; i < components.size(); i++) {
+      result[i] = hashIdentifier(components.get(i));
+    }
+    return result;
+  }
+
+  private static int[][] hashQualifiedNameArray(String[] supers) {
+    int[][] superHashes = new int[supers.length][];
+    for (int i = 0; i < supers.length; i++) {
+      superHashes[i] = hashQualifiedName(supers[i]);
+    }
+    return superHashes;
+  }
+
  public static class Unit {
-    @NotNull public final String myPackageId;
+    @NotNull public final int[] myPackageName;
    public final byte myUnitType;
    public final Import[] imports;
    public final ClassDecl[] myDecls;

-    public Unit(@Nullable String packageId, byte unitType, Import[] imports, ClassDecl[] decls) {
-      this.myPackageId = StringUtil.notNullize(packageId);
-      this.myUnitType = unitType;
+    public Unit(@Nullable String packageName, byte unitType, Import[] imports, ClassDecl[] decls) {
+      this(hashQualifiedName(StringUtil.notNullize(packageName)), unitType, imports, decls);
+    }
+
+    public Unit(@NotNull int[] packageName, byte unitType, Import[] imports, ClassDecl[] decls) {
+      myPackageName = packageName;
+      myUnitType = unitType;
      this.imports = imports;
-      this.myDecls = decls;
+      myDecls = decls;
    }

    @Override
@@ -60,7 +99,7 @@ public class IndexTree {
      Unit unit = (Unit)o;

      if (myUnitType != unit.myUnitType) return false;
-      if (!myPackageId.equals(unit.myPackageId)) return false;
+      if (!Arrays.equals(myPackageName, unit.myPackageName)) return false;
      if (!Arrays.equals(imports, unit.imports)) return false;
      if (!Arrays.equals(myDecls, unit.myDecls)) return false;

@@ -69,11 +108,11 @@ public class IndexTree {

    @Override
    public int hashCode() {
-      int hash = myUnitType * 31 + myPackageId.hashCode();
+      int hash = myUnitType * 31 + Arrays.hashCode(myPackageName);
      for (ClassDecl decl : myDecls) {
-        String name = decl.myName;
-        if (name != null) {
-          return hash * 31 + name.hashCode();
+        int name = decl.myName;
+        if (name != 0) {
+          return hash * 31 + name;
        }
      }
      return hash;
@@ -82,16 +121,20 @@ public class IndexTree {

  public static class Import {
    public static final Import[] EMPTY_ARRAY = new Import[0];
-    public final String myFullname;
+    public final int[] myFullname;
    public final boolean myStaticImport;
    public final boolean myOnDemand;
-    public final String myAlias;
+    public final int myAlias;

-    public Import(String fullname, boolean staticImport, boolean onDemand, String alias) {
-      this.myFullname = fullname;
-      this.myStaticImport = staticImport;
-      this.myOnDemand = onDemand;
-      this.myAlias = alias;
+    public Import(String fullname, boolean staticImport, boolean onDemand, @Nullable String alias) {
+      this(hashQualifiedName(fullname), staticImport, onDemand, hashIdentifier(alias));
+    }
+
+    public Import(int[] fullname, boolean staticImport, boolean onDemand, int alias) {
+      myFullname = fullname;
+      myStaticImport = staticImport;
+      myOnDemand = onDemand;
+      myAlias = alias;
    }

    @Override
@@ -103,18 +146,18 @@ public class IndexTree {

      if (myStaticImport != anImport.myStaticImport) return false;
      if (myOnDemand != anImport.myOnDemand) return false;
-      if (!myFullname.equals(anImport.myFullname)) return false;
-      if (myAlias != null ? !myAlias.equals(anImport.myAlias) : anImport.myAlias != null) return false;
+      if (!Arrays.equals(myFullname, anImport.myFullname)) return false;
+      if (myAlias != anImport.myAlias) return false;

      return true;
    }

    @Override
    public int hashCode() {
-      int result = myFullname.hashCode();
+      int result = Arrays.hashCode(myFullname);
      result = 31 * result + (myStaticImport ? 1 : 0);
      result = 31 * result + (myOnDemand ? 1 : 0);
-      result = 31 * result + (myAlias != null ? myAlias.hashCode() : 0);
+      result = 31 * result + myAlias;
      return result;
    }
  }
@@ -132,16 +175,20 @@ public class IndexTree {
    public static final ClassDecl[] EMPTY_ARRAY = new ClassDecl[0];
    public final int myStubId;
    public final int myMods;
-    public final String myName;
-    public final String[] mySupers;
+    public final int myName;
+    public final int[][] mySupers;

-    public ClassDecl(int stubId, int mods, String name, String[] supers, Decl[] decls) {
+    public ClassDecl(int stubId, int mods, @Nullable String name, String[] supers, Decl[] decls) {
+      this(stubId, mods, hashIdentifier(name), hashQualifiedNameArray(supers), decls);
+    }
+
+    public ClassDecl(int stubId, int mods, int name, int[][] supers, Decl[] decls) {
      super(decls);
      assert stubId > 0;
-      this.myStubId = stubId;
-      this.myMods = mods;
-      this.myName = name;
-      this.mySupers = supers;
+      myStubId = stubId;
+      myMods = mods;
+      myName = name;
+      mySupers = supers;
    }

    @Override
@@ -152,8 +199,8 @@ public class IndexTree {
      ClassDecl classDecl = (ClassDecl)o;
      if (myStubId != classDecl.myStubId) return false;
      if (myMods != classDecl.myMods) return false;
-      if (myName != null ? !myName.equals(classDecl.myName) : classDecl.myName != null) return false;
-      if (!Arrays.equals(mySupers, classDecl.mySupers)) return false;
+      if (myName != classDecl.myName) return false;
+      if (!Arrays.deepEquals(mySupers, classDecl.mySupers)) return false;
      if (!Arrays.equals(myDecls, classDecl.myDecls)) return false;
      return true;
    }
@@ -162,7 +209,7 @@ public class IndexTree {
    public int hashCode() {
      int result = myStubId;
      result = 31 * result + myMods;
-      result = 31 * result + (myName != null ? myName.hashCode() : 0);
+      result = 31 * result + myName;
      return result;
    }
  }