From cbd1de59a899cc4e8460c93a531913d720a98236 Mon Sep 17 00:00:00 2001 From: "Maxim.Mossienko" Date: Tue, 31 Jul 2012 13:11:46 +0400 Subject: [PATCH] more optimal storage for mostly used single stub id + StubIdList used instead of raw int[] to avoid Value mapping problems, take 2 --- .../com/intellij/psi/stubs/StubIdList.java | 4 +- .../com/intellij/psi/stubs/StubIndexImpl.java | 57 ++++++++++--------- .../intellij/psi/stubs/StubUpdatingIndex.java | 39 +++++++++---- 3 files changed, 58 insertions(+), 42 deletions(-) diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubIdList.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubIdList.java index b21f088a1724..1d9714250639 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubIdList.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubIdList.java @@ -65,12 +65,12 @@ final class StubIdList { } int size() { - return myArray == null ? myData > 0 ? 1 : 0: myData; + return myArray == null ? myData >= 0 ? 1 : 0: myData; } int get(int i) { if (myArray == null) { - assert myData > 0; + assert myData >= 0; if (i == 0) return myData; throw new IncorrectOperationException(); } else { diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java index e896aa0815c8..2cf8f208ead8 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java @@ -127,14 +127,14 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe for (int attempt = 0; attempt < 2; attempt++) { try { - final MapIndexStorage storage = new MapIndexStorage( + final MapIndexStorage storage = new MapIndexStorage( IndexInfrastructure.getStorageFile(indexKey), extension.getKeyDescriptor(), new StubIdExternalizer(), extension.getCacheSize(), extension.isKeyHighlySelective() ); - final MemoryIndexStorage memStorage = new MemoryIndexStorage(storage); + final MemoryIndexStorage memStorage = new MemoryIndexStorage(storage); myIndices.put(indexKey, new MyIndex(memStorage)); break; } @@ -148,40 +148,41 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe return needRebuild; } - private static class StubIdExternalizer implements DataExternalizer { + private static class StubIdExternalizer implements DataExternalizer { @Override - public void save(final DataOutput out, @NotNull final int[] value) throws IOException { - int size = value.length; + public void save(final DataOutput out, @NotNull final StubIdList value) throws IOException { + int size = value.size(); if (size == 0) { - DataInputOutputUtil.writeSINT(out, Integer.MAX_VALUE); + DataInputOutputUtil.writeINT(out, Integer.MAX_VALUE); } else if (size == 1) { - DataInputOutputUtil.writeSINT(out, -value[0]); + DataInputOutputUtil.writeINT(out, value.get(0)); // most often case } else { - DataInputOutputUtil.writeSINT(out, size); - for (int i = 0; i < size; i++) { - DataInputOutputUtil.writeINT(out, value[i]); + DataInputOutputUtil.writeINT(out, -size); + for(int i = 0; i < size; ++i) { + DataInputOutputUtil.writeINT(out, value.get(i)); } } } @NotNull @Override - public int[] read(final DataInput in) throws IOException { - int size = DataInputOutputUtil.readSINT(in); + public StubIdList read(final DataInput in) throws IOException { + int size = DataInputOutputUtil.readINT(in); if (size == Integer.MAX_VALUE) { - return new int[0]; + return new StubIdList(); } - else if (size <= 0) { - return new int[] {-size}; + else if (size >= 0) { + return new StubIdList(size); } else { + size = -size; int[] result = new int[size]; - for (int i = 0; i < size; i++) { + for(int i = 0; i < size; ++i) { result[i] = DataInputOutputUtil.readINT(in); } - return result; + return new StubIdList(result, size); } } } @@ -216,13 +217,13 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe // disable up-to-date check to avoid locks on attempt to acquire index write lock while holding at the same time the readLock for this index FileBasedIndexImpl.disableUpToDateCheckForCurrentThread(); index.getReadLock().lock(); - final ValueContainer container = index.getData(key); + final ValueContainer container = index.getData(key); final FileBasedIndexImpl.ProjectIndexableFilesFilter projectFilesFilter = fileBasedIndex.projectIndexableFiles(project); - return container.forEach(new ValueContainer.ContainerAction() { + return container.forEach(new ValueContainer.ContainerAction() { @Override - public boolean perform(final int id, @NotNull final int[] value) { + public boolean perform(final int id, @NotNull final StubIdList value) { if (projectFilesFilter != null && !projectFilesFilter.contains(id)) return true; final VirtualFile file = IndexInfrastructure.findFileByIdIfCached(fs, id); if (file == null || scope != null && !scope.contains(file)) { @@ -252,8 +253,8 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe return true; } final List> plained = stubTree.getPlainList(); - for (int i = 0; i < value.length; i++) { - final StubElement stub = plained.get(value[i]); + for (int i = 0, size = value.size(); i < size; i++) { + final StubElement stub = plained.get(value.get(i)); final ASTNode tree = psiFile.findTreeForStub(stubTree, stub); if (tree != null) { @@ -293,8 +294,8 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe } else { final List> plained = stubTree.getPlainList(); - for (int i = 0; i < value.length; i++) { - final int stubTreeIndex = value[i]; + for (int i = 0, size = value.size(); i < size; i++) { + final int stubTreeIndex = value.get(i); if (stubTreeIndex >= plained.size()) { final VirtualFile virtualFile = psiFile.getVirtualFile(); StubTree stubTreeFromIndex = StubTreeLoader.getInstance().readFromVFile(project, file); @@ -473,7 +474,7 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe index.flush(); } - public void updateIndex(@NotNull StubIndexKey key, int fileId, @NotNull final Map oldValues, @NotNull Map newValues) { + public void updateIndex(@NotNull StubIndexKey key, int fileId, @NotNull final Map oldValues, @NotNull Map newValues) { try { final MyIndex index = (MyIndex)myIndices.get(key); index.updateWithMap(fileId, newValues, new Callable>() { @@ -489,13 +490,13 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe } } - private static class MyIndex extends MapReduceIndex { - public MyIndex(final IndexStorage storage) { + private static class MyIndex extends MapReduceIndex { + public MyIndex(final IndexStorage storage) { super(null, null, storage); } @Override - public void updateWithMap(final int inputId, @NotNull final Map newData, @NotNull Callable> oldKeysGetter) throws StorageException { + public void updateWithMap(final int inputId, @NotNull final Map newData, @NotNull Callable> oldKeysGetter) throws StorageException { super.updateWithMap(inputId, newData, oldKeysGetter); } } diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java index b4e6a7182b55..a784e3cb85ef 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java @@ -47,7 +47,7 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi public static final ID INDEX_ID = ID.create("Stubs"); - private static final int VERSION = 20; + private static final int VERSION = 21; private static final DataExternalizer KEY_EXTERNALIZER = new DataExternalizer() { @Override @@ -206,21 +206,25 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi return new MyIndex(indexId, storage, getIndexer()); } - private static void updateStubIndices(@NotNull final Collection indexKeys, final int inputId, @NotNull final Map> oldStubTree, @NotNull final Map> newStubTree) { + private static void updateStubIndices(@NotNull final Collection indexKeys, + final int inputId, + @NotNull final Map> oldStubTree, + @NotNull final Map> newStubTree) { final StubIndexImpl stubIndex = (StubIndexImpl)StubIndex.getInstance(); for (StubIndexKey key : indexKeys) { - final Map oldMap = oldStubTree.get(key); - final Map newMap = newStubTree.get(key); + final Map oldMap = oldStubTree.get(key); + final Map newMap = newStubTree.get(key); - final Map _oldMap = oldMap != null ? oldMap : Collections.emptyMap(); - final Map _newMap = newMap != null ? newMap : Collections.emptyMap(); + final Map _oldMap = oldMap != null ? oldMap : Collections.emptyMap(); + final Map _newMap = newMap != null ? newMap : Collections.emptyMap(); stubIndex.updateIndex(key, inputId, _oldMap, _newMap); } } @NotNull - private static Collection getAffectedIndices(@NotNull final Map> oldStubTree, @NotNull final Map> newStubTree) { + private static Collection getAffectedIndices(@NotNull final Map> oldStubTree, + @NotNull final Map> newStubTree) { Set allIndices = new HashSet(); allIndices.addAll(oldStubTree.keySet()); allIndices.addAll(newStubTree.keySet()); @@ -253,7 +257,7 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi throws StorageException { checkNameStorage(); - final Map> newStubTree = getStubTree(newData); + final Map> newStubTree = getStubTree(newData); final StubIndexImpl stubIndex = getStubIndex(); final Collection allStubIndices = stubIndex.getAllStubIndexKeys(); @@ -267,7 +271,7 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi getWriteLock().lock(); final Map oldData = readOldData(inputId); - final Map> oldStubTree = getStubTree(oldData); + final Map> oldStubTree = getStubTree(oldData); super.updateWithMap(inputId, newData, oldKeysGetter); @@ -301,11 +305,22 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi } } - private static Map> getStubTree(@NotNull final Map data) { - final Map> stubTree; + private static Map> getStubTree(@NotNull final Map data) { + final Map> stubTree; if (!data.isEmpty()) { final SerializedStubTree stub = data.values().iterator().next(); - stubTree = new StubTree((PsiFileStub)stub.getStub(true), false).indexStubTree(); + Map> map = new StubTree((PsiFileStub)stub.getStub(true), false).indexStubTree(); + + // xxx:fix refs inplace + stubTree = (Map)map; + for(StubIndexKey key:map.keySet()) { + Map value = map.get(key); + for(Object k: value.keySet()) { + int[] ints = value.get(k); + StubIdList stubList = ints.length == 1 ? new StubIdList(ints[0]) : new StubIdList(ints, ints.length); + ((Map)(Map)value).put(k, stubList); + } + } } else { stubTree = Collections.emptyMap();