From 8b124a1d3fc73057146318cd60c0812cd140cdda Mon Sep 17 00:00:00 2001 From: "Maxim.Mossienko" Date: Fri, 11 Apr 2014 12:29:49 +0200 Subject: [PATCH] use snapshot mapping for IdIndex / TrigramIndex for storing keys, right now switched on by the flag --- .../psi/impl/cache/impl/id/IdIndex.java | 10 +- .../intellij/find/ngrams/TrigramIndex.java | 3 +- .../com/intellij/psi/stubs/StubIndexImpl.java | 6 +- .../intellij/psi/stubs/StubUpdatingIndex.java | 4 +- .../util/indexing/ContentHashesSupport.java | 75 +++++++ .../util/indexing/FileBasedIndexImpl.java | 5 +- .../util/indexing/MapReduceIndex.java | 205 +++++++++++++----- .../newvfs/persistent/ContentHashesUtil.java | 2 +- 8 files changed, 249 insertions(+), 61 deletions(-) create mode 100644 platform/lang-impl/src/com/intellij/util/indexing/ContentHashesSupport.java diff --git a/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/IdIndex.java b/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/IdIndex.java index 2e25977f9d90..f61eceb7119d 100644 --- a/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/IdIndex.java +++ b/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/IdIndex.java @@ -21,6 +21,8 @@ import com.intellij.openapi.fileTypes.FileType; import com.intellij.openapi.fileTypes.LanguageFileType; import com.intellij.openapi.fileTypes.impl.CustomSyntaxTableFileType; import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.psi.search.UsageSearchContext; +import com.intellij.util.SystemProperties; import com.intellij.util.indexing.*; import com.intellij.util.io.DataExternalizer; import com.intellij.util.io.InlineKeyDescriptor; @@ -48,15 +50,17 @@ public class IdIndex extends FileBasedIndexExtension { } }; + public static final boolean ourSnapshotMappingsEnabled = SystemProperties.getBooleanProperty("idea.index.snapshot.mappings.enabled", false); + private final DataExternalizer myValueExternalizer = new DataExternalizer() { @Override public void save(@NotNull final DataOutput out, final Integer value) throws IOException { - out.writeByte(value.intValue()); + out.write(value.intValue() & UsageSearchContext.ANY); } @Override public Integer read(@NotNull final DataInput in) throws IOException { - return Integer.valueOf(in.readByte()); + return Integer.valueOf(in.readByte() & UsageSearchContext.ANY); } }; @@ -87,7 +91,7 @@ public class IdIndex extends FileBasedIndexExtension { @Override public int getVersion() { - return 11; // TODO: version should enumerate all word scanner versions and build version upon that set + return 11 + (ourSnapshotMappingsEnabled ? 0xFF:0); // TODO: version should enumerate all word scanner versions and build version upon that set } @Override diff --git a/platform/lang-impl/src/com/intellij/find/ngrams/TrigramIndex.java b/platform/lang-impl/src/com/intellij/find/ngrams/TrigramIndex.java index 36d1ea401f47..e769472503e6 100644 --- a/platform/lang-impl/src/com/intellij/find/ngrams/TrigramIndex.java +++ b/platform/lang-impl/src/com/intellij/find/ngrams/TrigramIndex.java @@ -21,6 +21,7 @@ package com.intellij.find.ngrams; import com.intellij.openapi.util.text.TrigramBuilder; import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.psi.impl.cache.impl.id.IdIndex; import com.intellij.util.indexing.*; import com.intellij.util.io.EnumeratorIntegerDescriptor; import com.intellij.util.io.KeyDescriptor; @@ -100,7 +101,7 @@ public class TrigramIndex extends ScalarIndexExtension { @Override public int getVersion() { - return ENABLED ? 2 : 1; + return ENABLED ? 2 + (IdIndex.ourSnapshotMappingsEnabled ? 0xFF:0) : 1; } @Override diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java index 61a521e729eb..cb925816ca58 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java @@ -428,7 +428,7 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe public void updateIndex(@NotNull StubIndexKey key, int fileId, @NotNull final Map oldValues, @NotNull Map newValues) { try { final MyIndex index = (MyIndex)myIndices.get(key); - index.updateWithMap(fileId, newValues, new NotNullComputable>() { + index.updateWithMap(fileId, fileId, newValues, new NotNullComputable>() { @NotNull @Override public Collection compute() { @@ -449,9 +449,9 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe @Override public void updateWithMap(final int inputId, - @NotNull final Map newData, + int savedInputId, @NotNull final Map newData, @NotNull NotNullComputable> oldKeysGetter) throws StorageException { - super.updateWithMap(inputId, newData, oldKeysGetter); + super.updateWithMap(inputId, savedInputId, newData, oldKeysGetter); } } diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java index 5a735a85be70..0b565e657cd7 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java @@ -293,7 +293,7 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi @Override protected void updateWithMap(final int inputId, - @NotNull final Map newData, + int savedInputId, @NotNull final Map newData, @NotNull NotNullComputable> oldKeysGetter) throws StorageException { @@ -327,7 +327,7 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi throw new StorageException(e); } - super.updateWithMap(inputId, newData, oldKeysGetter); + super.updateWithMap(inputId, savedInputId, newData, oldKeysGetter); updateStubIndices(getAffectedIndices(oldStubTree, newStubTree), inputId, oldStubTree, newStubTree); } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/ContentHashesSupport.java b/platform/lang-impl/src/com/intellij/util/indexing/ContentHashesSupport.java new file mode 100644 index 000000000000..8e4bf27e2951 --- /dev/null +++ b/platform/lang-impl/src/com/intellij/util/indexing/ContentHashesSupport.java @@ -0,0 +1,75 @@ +/* + * Copyright 2000-2014 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.indexing; + +import com.intellij.openapi.application.PathManager; +import com.intellij.openapi.fileTypes.FileType; +import com.intellij.openapi.util.ThrowableComputable; +import com.intellij.openapi.vfs.newvfs.persistent.ContentHashesUtil; +import com.intellij.openapi.vfs.newvfs.persistent.FlushingDaemon; +import com.intellij.util.io.IOUtil; +import org.jetbrains.annotations.NotNull; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.security.MessageDigest; + +/** + * @author Maxim.Mossienko + * @since 4/10/2014. + */ +class ContentHashesSupport { + private static final ContentHashesUtil.HashEnumerator ourHashesWithFileType; + + static { + ContentHashesUtil.HashEnumerator hashEnumerator = null; + try { + final File hashEnumeratorFile = new File(PathManager.getIndexRoot(), "hashesWithFileType"); + hashEnumerator = IOUtil.openCleanOrResetBroken(new ThrowableComputable() { + @Override + public ContentHashesUtil.HashEnumerator compute() throws IOException { + return new ContentHashesUtil.HashEnumerator(hashEnumeratorFile, null); + } + }, hashEnumeratorFile); + FlushingDaemon.everyFiveSeconds(new Runnable() { + @Override + public void run() { + if (ourHashesWithFileType.isDirty()) ourHashesWithFileType.force(); + } + }); + } catch (IOException ex) { + throw new RuntimeException(ex); + } finally { + ourHashesWithFileType = hashEnumerator; + } + } + + + static int calcContentHashIdWithFileType(@NotNull byte[] bytes, @NotNull FileType fileType) throws IOException { + MessageDigest messageDigest = ContentHashesUtil.HASHER_CACHE.getValue(); + + Charset defaultCharset = Charset.defaultCharset(); + messageDigest.update(fileType.getName().getBytes(defaultCharset)); + messageDigest.update((byte)0); + messageDigest.update(String.valueOf(bytes.length).getBytes(defaultCharset)); + messageDigest.update((byte)0); + messageDigest.update(bytes, 0, bytes.length); + byte[] digest = messageDigest.digest(); + + return ourHashesWithFileType.enumerate(digest); + } +} diff --git a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java index c26ee6c8cfe0..11c31f8b0e6e 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java @@ -61,6 +61,7 @@ import com.intellij.psi.impl.PsiDocumentTransactionListener; import com.intellij.psi.impl.PsiManagerImpl; import com.intellij.psi.impl.PsiTreeChangeEventImpl; import com.intellij.psi.impl.PsiTreeChangePreprocessor; +import com.intellij.psi.impl.cache.impl.id.IdIndex; import com.intellij.psi.impl.cache.impl.id.PlatformIdTableBuilding; import com.intellij.psi.impl.source.PsiFileImpl; import com.intellij.psi.search.EverythingGlobalScope; @@ -141,7 +142,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { private final ConcurrentHashSet myProjectsBeingUpdated = new ConcurrentHashSet(); @SuppressWarnings({"FieldCanBeLocal", "UnusedDeclaration"}) private volatile boolean myInitialized; - // need this variable for memory barrier + // need this variable for memory barrier public FileBasedIndexImpl(@SuppressWarnings("UnusedParameters") VirtualFileManager vfManager, FileDocumentManager fdm, @@ -514,7 +515,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { index = (MapReduceIndex)custom; } else { - index = new MapReduceIndex(indexId, extension.getIndexer(), storage); + index = new MapReduceIndex(indexId, extension.getIndexer(), storage, extension.hasSnapshotMapping() && IdIndex.ourSnapshotMappingsEnabled ? extension.getKeyDescriptor() : null); } final KeyDescriptor keyDescriptor = extension.getKeyDescriptor(); diff --git a/platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java b/platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java index a19e2d92cddf..3220dcb3a68a 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java @@ -18,14 +18,13 @@ package com.intellij.util.indexing; import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.progress.ProgressManager; -import com.intellij.openapi.util.Computable; -import com.intellij.openapi.util.Factory; -import com.intellij.openapi.util.NotNullComputable; -import com.intellij.openapi.util.Ref; +import com.intellij.openapi.util.*; import com.intellij.openapi.util.io.FileUtil; import com.intellij.psi.search.GlobalSearchScope; import com.intellij.util.Processor; -import com.intellij.util.SmartList; +import com.intellij.util.io.EnumeratorIntegerDescriptor; +import com.intellij.util.io.IOUtil; +import com.intellij.util.io.KeyDescriptor; import com.intellij.util.io.PersistentHashMap; import gnu.trove.THashMap; import gnu.trove.TObjectObjectProcedure; @@ -47,19 +46,36 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; */ public class MapReduceIndex implements UpdatableIndex { private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.MapReduceIndex"); + private static final int NULL_MAPPING = 0; @Nullable private final ID myIndexId; private final DataIndexer myIndexer; @NotNull protected final IndexStorage myStorage; - @Nullable private PersistentHashMap> myInputsIndex; + private final boolean myHasSnapshotMapping; + private final KeyDescriptor myKeyDescriptorForSnapshotIndex; + + private PersistentHashMap> myInputsIndex; + private PersistentHashMap> mySnapshotMapping; + private PersistentHashMap myInputsSnapshotMapping; private final ReentrantReadWriteLock myLock = new ReentrantReadWriteLock(); private Factory>> myInputsIndexFactory; - public MapReduceIndex(@Nullable final ID indexId, DataIndexer indexer, @NotNull IndexStorage storage) { + public MapReduceIndex(@Nullable final ID indexId, + DataIndexer indexer, + @NotNull IndexStorage storage) { + this(indexId, indexer, storage, null); + } + + public MapReduceIndex(@Nullable final ID indexId, + DataIndexer indexer, + @NotNull IndexStorage storage, + KeyDescriptor keyDescriptorForSnapshotIndex) { myIndexId = indexId; myIndexer = indexer; myStorage = storage; + myHasSnapshotMapping = keyDescriptorForSnapshotIndex != null; + myKeyDescriptorForSnapshotIndex = keyDescriptorForSnapshotIndex; } @NotNull @@ -73,16 +89,17 @@ public class MapReduceIndex implements UpdatableIndex implements UpdatableIndex createInputSnapshotMapping() throws IOException { + assert myIndexId != null; + final File fileIdToHashIdFile = new File(IndexInfrastructure.getIndexRootDir(myIndexId), "fileIdToHashId"); + return IOUtil.openCleanOrResetBroken(new ThrowableComputable, IOException>() { + @Override + public PersistentHashMap compute() throws IOException { + return new PersistentHashMap(fileIdToHashIdFile, EnumeratorIntegerDescriptor.INSTANCE, EnumeratorIntegerDescriptor.INSTANCE, 4096) { + @Override + protected boolean wantCompactIntegralValues() { + return true; + } + }; + } + }, fileIdToHashIdFile); + } + + private static void cleanMapping(@NotNull PersistentHashMap index) { + final File baseFile = index.getBaseFile(); + try { + index.close(); + } + catch (IOException ignored) { + } + + FileUtil.delete(baseFile); + } + @Override public void flush() throws StorageException{ try { getReadLock().lock(); - final PersistentHashMap> inputsIndex = myInputsIndex; - if (inputsIndex != null && inputsIndex.isDirty()) { - inputsIndex.force(); - } + doForce(myInputsIndex); + doForce(myInputsSnapshotMapping); + doForce(mySnapshotMapping); myStorage.flush(); } catch (IOException e) { @@ -122,6 +165,12 @@ public class MapReduceIndex implements UpdatableIndex inputsIndex) { + if (inputsIndex != null && inputsIndex.isDirty()) { + inputsIndex.force(); + } + } + @Override public void dispose() { final Lock lock = getWriteLock(); @@ -131,14 +180,9 @@ public class MapReduceIndex implements UpdatableIndex implements UpdatableIndex index) { + if (index != null) { + try { + index.close(); + } + catch (IOException e) { + LOG.error(e); + } + } + } + @NotNull @Override public final Lock getReadLock() { @@ -188,9 +243,25 @@ public class MapReduceIndex implements UpdatableIndex>> factory) throws IOException { myInputsIndexFactory = factory; + if (myHasSnapshotMapping) { + myInputsSnapshotMapping = createInputSnapshotMapping(); + mySnapshotMapping = createSnapshotMappingIndex(); + } myInputsIndex = createInputsIndex(); } + private PersistentHashMap> createSnapshotMappingIndex() throws IOException { + assert myIndexId != null; + final File hashIdToKeysFile = new File(IndexInfrastructure.getIndexRootDir(myIndexId), "hashIdToKeys"); + final InputIndexDataExternalizer externalizer = new InputIndexDataExternalizer(myKeyDescriptorForSnapshotIndex, myIndexId); + return IOUtil.openCleanOrResetBroken(new ThrowableComputable>, IOException>() { + @Override + public PersistentHashMap> compute() throws IOException { + return new PersistentHashMap>(hashIdToKeysFile, EnumeratorIntegerDescriptor.INSTANCE, externalizer); + } + }, hashIdToKeysFile); + } + @Nullable private PersistentHashMap> createInputsIndex() throws IOException { Factory>> factory = myInputsIndexFactory; @@ -216,6 +287,50 @@ public class MapReduceIndex implements UpdatableIndex> oldKeysGetter; + final int savedInputId; + + if (myHasSnapshotMapping && !((MemoryIndexStorage)getStorage()).isBufferingEnabled()) { + oldKeysGetter = new NotNullComputable>() { + @NotNull + @Override + public Collection compute() { + try { + Integer hashId = myInputsSnapshotMapping.get(inputId); + Collection keys = hashId != null ? mySnapshotMapping.get(hashId): null; + return keys == null ? Collections.emptyList() : keys; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + }; + try { + if (content instanceof FileContent) { + FileContent fileContent = (FileContent)content; + savedInputId = ContentHashesSupport.calcContentHashIdWithFileType(fileContent.getContent(), fileContent.getFileType()); + } else { + savedInputId = NULL_MAPPING; + } + } catch (IOException ex) { + throw new RuntimeException(ex); + } + } else { + oldKeysGetter = new NotNullComputable>() { + @NotNull + @Override + public Collection compute() { + try { + Collection oldKeys = myInputsIndex.get(inputId); + return oldKeys == null? Collections.emptyList() : oldKeys; + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + }; + savedInputId = inputId; + } + // do not depend on content! return new Computable() { @Override @@ -225,22 +340,7 @@ public class MapReduceIndex implements UpdatableIndex>() { - @NotNull - @Override - public Collection compute() { - if (myInputsIndex == null) { - return new SmartList((Key)(Integer)inputId); - } - try { - Collection oldKeys = myInputsIndex.get(inputId); - return oldKeys == null? Collections.emptyList() : oldKeys; - } - catch (IOException e) { - throw new RuntimeException(e); - } - } - }); + updateWithMap(inputId, savedInputId, data, oldKeysGetter); } catch (StorageException ex) { exRef.set(ex); @@ -248,6 +348,7 @@ public class MapReduceIndex implements UpdatableIndex implements UpdatableIndex newData, + int savedInputId, @NotNull Map newData, @NotNull NotNullComputable> oldKeysGetter) throws StorageException { getWriteLock().lock(); try { @@ -295,8 +396,14 @@ public class MapReduceIndex implements UpdatableIndex newKeys = newData.keySet(); if (newKeys.size() > 0) { myInputsIndex.put(inputId, newKeys); @@ -305,9 +412,9 @@ public class MapReduceIndex implements UpdatableIndex { public HashEnumerator(File contentsHashesFile, PagedFileStorage.StorageLockContext storageLockContext) throws IOException { - super(contentsHashesFile, new ContentHashesDescriptor(), 1024 * 1024, storageLockContext); + super(contentsHashesFile, new ContentHashesDescriptor(), 64 * 1024, storageLockContext); } @Override