MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

use snapshot mapping for IdIndex / TrigramIndex for storing keys, right now switched on by the flag

Browse Source
This commit is contained in:
Maxim.Mossienko
2014-04-11 12:32:31 +02:00
parent f8863d6762
commit 8b124a1d3f
8 changed files with 249 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/IdIndex.java
Vendored
+7 -3
View File
@@ -21,6 +21,8 @@ import com.intellij.openapi.fileTypes.FileType;
import com.intellij.openapi.fileTypes.LanguageFileType;
import com.intellij.openapi.fileTypes.impl.CustomSyntaxTableFileType;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.psi.search.UsageSearchContext;
import com.intellij.util.SystemProperties;
import com.intellij.util.indexing.*;
import com.intellij.util.io.DataExternalizer;
import com.intellij.util.io.InlineKeyDescriptor;
@@ -48,15 +50,17 @@ public class IdIndex extends FileBasedIndexExtension<IdIndexEntry, Integer> {
}
};
public static final boolean ourSnapshotMappingsEnabled = SystemProperties.getBooleanProperty("idea.index.snapshot.mappings.enabled", false);
private final DataExternalizer<Integer> myValueExternalizer = new DataExternalizer<Integer>() {
@Override
public void save(@NotNull final DataOutput out, final Integer value) throws IOException {
out.writeByte(value.intValue());
out.write(value.intValue() & UsageSearchContext.ANY);
}
@Override
public Integer read(@NotNull final DataInput in) throws IOException {
return Integer.valueOf(in.readByte());
return Integer.valueOf(in.readByte() & UsageSearchContext.ANY);
}
};
@@ -87,7 +91,7 @@ public class IdIndex extends FileBasedIndexExtension<IdIndexEntry, Integer> {
@Override
public int getVersion() {
return 11; // TODO: version should enumerate all word scanner versions and build version upon that set
return 11 + (ourSnapshotMappingsEnabled ? 0xFF:0); // TODO: version should enumerate all word scanner versions and build version upon that set
}
@Override
platform/lang-impl/src/com/intellij/find/ngrams/TrigramIndex.java
+2 -1
View File
@@ -21,6 +21,7 @@ package com.intellij.find.ngrams;
import com.intellij.openapi.util.text.TrigramBuilder;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.psi.impl.cache.impl.id.IdIndex;
import com.intellij.util.indexing.*;
import com.intellij.util.io.EnumeratorIntegerDescriptor;
import com.intellij.util.io.KeyDescriptor;
@@ -100,7 +101,7 @@ public class TrigramIndex extends ScalarIndexExtension<Integer> {
@Override
public int getVersion() {
return ENABLED ? 2 : 1;
return ENABLED ? 2 + (IdIndex.ourSnapshotMappingsEnabled ? 0xFF:0) : 1;
}
@Override
platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java
+3 -3
View File
@@ -428,7 +428,7 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe
public <K> void updateIndex(@NotNull StubIndexKey key, int fileId, @NotNull final Map<K, StubIdList> oldValues, @NotNull Map<K, StubIdList> newValues) {
try {
final MyIndex<K> index = (MyIndex<K>)myIndices.get(key);
index.updateWithMap(fileId, newValues, new NotNullComputable<Collection<K>>() {
index.updateWithMap(fileId, fileId, newValues, new NotNullComputable<Collection<K>>() {
@NotNull
@Override
public Collection<K> compute() {
@@ -449,9 +449,9 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe
@Override
public void updateWithMap(final int inputId,
@NotNull final Map<K, StubIdList> newData,
int savedInputId, @NotNull final Map<K, StubIdList> newData,
@NotNull NotNullComputable<Collection<K>> oldKeysGetter) throws StorageException {
super.updateWithMap(inputId, newData, oldKeysGetter);
super.updateWithMap(inputId, savedInputId, newData, oldKeysGetter);
}
}
platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java
+2 -2
View File
@@ -293,7 +293,7 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi
@Override
protected void updateWithMap(final int inputId,
@NotNull final Map<Integer, SerializedStubTree> newData,
int savedInputId, @NotNull final Map<Integer, SerializedStubTree> newData,
@NotNull NotNullComputable<Collection<Integer>> oldKeysGetter)
throws StorageException {
@@ -327,7 +327,7 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi
throw new StorageException(e);
}
super.updateWithMap(inputId, newData, oldKeysGetter);
super.updateWithMap(inputId, savedInputId, newData, oldKeysGetter);
updateStubIndices(getAffectedIndices(oldStubTree, newStubTree), inputId, oldStubTree, newStubTree);
}
platform/lang-impl/src/com/intellij/util/indexing/ContentHashesSupport.java
+75
View File
@@ -0,0 +1,75 @@
/*
* Copyright 2000-2014 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.util.indexing;
import com.intellij.openapi.application.PathManager;
import com.intellij.openapi.fileTypes.FileType;
import com.intellij.openapi.util.ThrowableComputable;
import com.intellij.openapi.vfs.newvfs.persistent.ContentHashesUtil;
import com.intellij.openapi.vfs.newvfs.persistent.FlushingDaemon;
import com.intellij.util.io.IOUtil;
import org.jetbrains.annotations.NotNull;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.security.MessageDigest;
/**
* @author Maxim.Mossienko
* @since 4/10/2014.
*/
class ContentHashesSupport {
private static final ContentHashesUtil.HashEnumerator ourHashesWithFileType;
static {
ContentHashesUtil.HashEnumerator hashEnumerator = null;
try {
final File hashEnumeratorFile = new File(PathManager.getIndexRoot(), "hashesWithFileType");
hashEnumerator = IOUtil.openCleanOrResetBroken(new ThrowableComputable<ContentHashesUtil.HashEnumerator, IOException>() {
@Override
public ContentHashesUtil.HashEnumerator compute() throws IOException {
return new ContentHashesUtil.HashEnumerator(hashEnumeratorFile, null);
}
}, hashEnumeratorFile);
FlushingDaemon.everyFiveSeconds(new Runnable() {
@Override
public void run() {
if (ourHashesWithFileType.isDirty()) ourHashesWithFileType.force();
}
});
} catch (IOException ex) {
throw new RuntimeException(ex);
} finally {
ourHashesWithFileType = hashEnumerator;
}
}
static int calcContentHashIdWithFileType(@NotNull byte[] bytes, @NotNull FileType fileType) throws IOException {
MessageDigest messageDigest = ContentHashesUtil.HASHER_CACHE.getValue();
Charset defaultCharset = Charset.defaultCharset();
messageDigest.update(fileType.getName().getBytes(defaultCharset));
messageDigest.update((byte)0);
messageDigest.update(String.valueOf(bytes.length).getBytes(defaultCharset));
messageDigest.update((byte)0);
messageDigest.update(bytes, 0, bytes.length);
byte[] digest = messageDigest.digest();
return ourHashesWithFileType.enumerate(digest);
}
}
platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java
+3 -2
View File
@@ -61,6 +61,7 @@ import com.intellij.psi.impl.PsiDocumentTransactionListener;
import com.intellij.psi.impl.PsiManagerImpl;
import com.intellij.psi.impl.PsiTreeChangeEventImpl;
import com.intellij.psi.impl.PsiTreeChangePreprocessor;
import com.intellij.psi.impl.cache.impl.id.IdIndex;
import com.intellij.psi.impl.cache.impl.id.PlatformIdTableBuilding;
import com.intellij.psi.impl.source.PsiFileImpl;
import com.intellij.psi.search.EverythingGlobalScope;
@@ -141,7 +142,7 @@ public class FileBasedIndexImpl extends FileBasedIndex {
private final ConcurrentHashSet<Project> myProjectsBeingUpdated = new ConcurrentHashSet<Project>();
@SuppressWarnings({"FieldCanBeLocal", "UnusedDeclaration"}) private volatile boolean myInitialized;
// need this variable for memory barrier
// need this variable for memory barrier
public FileBasedIndexImpl(@SuppressWarnings("UnusedParameters") VirtualFileManager vfManager,
FileDocumentManager fdm,
@@ -514,7 +515,7 @@ public class FileBasedIndexImpl extends FileBasedIndex {
index = (MapReduceIndex<K, V, FileContent>)custom;
}
else {
index = new MapReduceIndex<K, V, FileContent>(indexId, extension.getIndexer(), storage);
index = new MapReduceIndex<K, V, FileContent>(indexId, extension.getIndexer(), storage, extension.hasSnapshotMapping() && IdIndex.ourSnapshotMappingsEnabled ? extension.getKeyDescriptor() : null);
}
final KeyDescriptor<K> keyDescriptor = extension.getKeyDescriptor();
platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java
+156 -49
View File
@@ -18,14 +18,13 @@ package com.intellij.util.indexing;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.progress.ProgressManager;
import com.intellij.openapi.util.Computable;
import com.intellij.openapi.util.Factory;
import com.intellij.openapi.util.NotNullComputable;
import com.intellij.openapi.util.Ref;
import com.intellij.openapi.util.*;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.psi.search.GlobalSearchScope;
import com.intellij.util.Processor;
import com.intellij.util.SmartList;
import com.intellij.util.io.EnumeratorIntegerDescriptor;
import com.intellij.util.io.IOUtil;
import com.intellij.util.io.KeyDescriptor;
import com.intellij.util.io.PersistentHashMap;
import gnu.trove.THashMap;
import gnu.trove.TObjectObjectProcedure;
@@ -47,19 +46,36 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
*/
public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Value, Input> {
private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.MapReduceIndex");
private static final int NULL_MAPPING = 0;
@Nullable private final ID<Key, Value> myIndexId;
private final DataIndexer<Key, Value, Input> myIndexer;
@NotNull protected final IndexStorage<Key, Value> myStorage;
@Nullable private PersistentHashMap<Integer, Collection<Key>> myInputsIndex;
private final boolean myHasSnapshotMapping;
private final KeyDescriptor<Key> myKeyDescriptorForSnapshotIndex;
private PersistentHashMap<Integer, Collection<Key>> myInputsIndex;
private PersistentHashMap<Integer, Collection<Key>> mySnapshotMapping;
private PersistentHashMap<Integer, Integer> myInputsSnapshotMapping;
private final ReentrantReadWriteLock myLock = new ReentrantReadWriteLock();
private Factory<PersistentHashMap<Integer, Collection<Key>>> myInputsIndexFactory;
public MapReduceIndex(@Nullable final ID<Key, Value> indexId, DataIndexer<Key, Value, Input> indexer, @NotNull IndexStorage<Key, Value> storage) {
public MapReduceIndex(@Nullable final ID<Key, Value> indexId,
DataIndexer<Key, Value, Input> indexer,
@NotNull IndexStorage<Key, Value> storage) {
this(indexId, indexer, storage, null);
}
public MapReduceIndex(@Nullable final ID<Key, Value> indexId,
DataIndexer<Key, Value, Input> indexer,
@NotNull IndexStorage<Key, Value> storage,
KeyDescriptor<Key> keyDescriptorForSnapshotIndex) {
myIndexId = indexId;
myIndexer = indexer;
myStorage = storage;
myHasSnapshotMapping = keyDescriptorForSnapshotIndex != null;
myKeyDescriptorForSnapshotIndex = keyDescriptorForSnapshotIndex;
}
@NotNull
@@ -73,16 +89,17 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
getWriteLock().lock();
myStorage.clear();
if (myInputsIndex != null) {
final File baseFile = myInputsIndex.getBaseFile();
try {
myInputsIndex.close();
}
catch (IOException ignored) {
}
FileUtil.delete(baseFile);
cleanMapping(myInputsIndex);
myInputsIndex = createInputsIndex();
}
if (myInputsSnapshotMapping != null) {
cleanMapping(myInputsSnapshotMapping);
myInputsSnapshotMapping = createInputSnapshotMapping();
}
if (mySnapshotMapping != null) {
cleanMapping(mySnapshotMapping);
mySnapshotMapping = createSnapshotMappingIndex();
}
}
catch (StorageException e) {
LOG.error(e);
@@ -95,14 +112,40 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
}
}
private PersistentHashMap<Integer, Integer> createInputSnapshotMapping() throws IOException {
assert myIndexId != null;
final File fileIdToHashIdFile = new File(IndexInfrastructure.getIndexRootDir(myIndexId), "fileIdToHashId");
return IOUtil.openCleanOrResetBroken(new ThrowableComputable<PersistentHashMap<Integer, Integer>, IOException>() {
@Override
public PersistentHashMap<Integer, Integer> compute() throws IOException {
return new PersistentHashMap<Integer, Integer>(fileIdToHashIdFile, EnumeratorIntegerDescriptor.INSTANCE, EnumeratorIntegerDescriptor.INSTANCE, 4096) {
@Override
protected boolean wantCompactIntegralValues() {
return true;
}
};
}
}, fileIdToHashIdFile);
}
private static void cleanMapping(@NotNull PersistentHashMap<?, ?> index) {
final File baseFile = index.getBaseFile();
try {
index.close();
}
catch (IOException ignored) {
}
FileUtil.delete(baseFile);
}
@Override
public void flush() throws StorageException{
try {
getReadLock().lock();
final PersistentHashMap<Integer, Collection<Key>> inputsIndex = myInputsIndex;
if (inputsIndex != null && inputsIndex.isDirty()) {
inputsIndex.force();
}
doForce(myInputsIndex);
doForce(myInputsSnapshotMapping);
doForce(mySnapshotMapping);
myStorage.flush();
}
catch (IOException e) {
@@ -122,6 +165,12 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
}
}
private static void doForce(PersistentHashMap<?, ?> inputsIndex) {
if (inputsIndex != null && inputsIndex.isDirty()) {
inputsIndex.force();
}
}
@Override
public void dispose() {
final Lock lock = getWriteLock();
@@ -131,14 +180,9 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
myStorage.close();
}
finally {
if (myInputsIndex != null) {
try {
myInputsIndex.close();
}
catch (IOException e) {
LOG.error(e);
}
}
doClose(myInputsIndex);
doClose(myInputsSnapshotMapping);
doClose(mySnapshotMapping);
}
}
catch (StorageException e) {
@@ -149,6 +193,17 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
}
}
private static void doClose(PersistentHashMap<?, ?> index) {
if (index != null) {
try {
index.close();
}
catch (IOException e) {
LOG.error(e);
}
}
}
@NotNull
@Override
public final Lock getReadLock() {
@@ -188,9 +243,25 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
public void setInputIdToDataKeysIndex(Factory<PersistentHashMap<Integer, Collection<Key>>> factory) throws IOException {
myInputsIndexFactory = factory;
if (myHasSnapshotMapping) {
myInputsSnapshotMapping = createInputSnapshotMapping();
mySnapshotMapping = createSnapshotMappingIndex();
}
myInputsIndex = createInputsIndex();
}
private PersistentHashMap<Integer, Collection<Key>> createSnapshotMappingIndex() throws IOException {
assert myIndexId != null;
final File hashIdToKeysFile = new File(IndexInfrastructure.getIndexRootDir(myIndexId), "hashIdToKeys");
final InputIndexDataExternalizer<Key> externalizer = new InputIndexDataExternalizer<Key>(myKeyDescriptorForSnapshotIndex, myIndexId);
return IOUtil.openCleanOrResetBroken(new ThrowableComputable<PersistentHashMap<Integer, Collection<Key>>, IOException>() {
@Override
public PersistentHashMap<Integer, Collection<Key>> compute() throws IOException {
return new PersistentHashMap<Integer, Collection<Key>>(hashIdToKeysFile, EnumeratorIntegerDescriptor.INSTANCE, externalizer);
}
}, hashIdToKeysFile);
}
@Nullable
private PersistentHashMap<Integer, Collection<Key>> createInputsIndex() throws IOException {
Factory<PersistentHashMap<Integer, Collection<Key>>> factory = myInputsIndexFactory;
@@ -216,6 +287,50 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
ProgressManager.checkCanceled();
final NotNullComputable<Collection<Key>> oldKeysGetter;
final int savedInputId;
if (myHasSnapshotMapping && !((MemoryIndexStorage)getStorage()).isBufferingEnabled()) {
oldKeysGetter = new NotNullComputable<Collection<Key>>() {
@NotNull
@Override
public Collection<Key> compute() {
try {
Integer hashId = myInputsSnapshotMapping.get(inputId);
Collection<Key> keys = hashId != null ? mySnapshotMapping.get(hashId): null;
return keys == null ? Collections.<Key>emptyList() : keys;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
};
try {
if (content instanceof FileContent) {
FileContent fileContent = (FileContent)content;
savedInputId = ContentHashesSupport.calcContentHashIdWithFileType(fileContent.getContent(), fileContent.getFileType());
} else {
savedInputId = NULL_MAPPING;
}
} catch (IOException ex) {
throw new RuntimeException(ex);
}
} else {
oldKeysGetter = new NotNullComputable<Collection<Key>>() {
@NotNull
@Override
public Collection<Key> compute() {
try {
Collection<Key> oldKeys = myInputsIndex.get(inputId);
return oldKeys == null? Collections.<Key>emptyList() : oldKeys;
}
catch (IOException e) {
throw new RuntimeException(e);
}
}
};
savedInputId = inputId;
}
// do not depend on content!
return new Computable<Boolean>() {
@Override
@@ -225,22 +340,7 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
@Override
public void run() {
try {
updateWithMap(inputId, data, new NotNullComputable<Collection<Key>>() {
@NotNull
@Override
public Collection<Key> compute() {
if (myInputsIndex == null) {
return new SmartList<Key>((Key)(Integer)inputId);
}
try {
Collection<Key> oldKeys = myInputsIndex.get(inputId);
return oldKeys == null? Collections.<Key>emptyList() : oldKeys;
}
catch (IOException e) {
throw new RuntimeException(e);
}
}
});
updateWithMap(inputId, savedInputId, data, oldKeysGetter);
}
catch (StorageException ex) {
exRef.set(ex);
@@ -248,6 +348,7 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
}
});
//noinspection ThrowableResultOfMethodCallIgnored
if (exRef.get() != null) {
LOG.info(exRef.get());
FileBasedIndex.getInstance().requestRebuild(myIndexId);
@@ -259,7 +360,7 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
}
protected void updateWithMap(final int inputId,
@NotNull Map<Key, Value> newData,
int savedInputId, @NotNull Map<Key, Value> newData,
@NotNull NotNullComputable<Collection<Key>> oldKeysGetter) throws StorageException {
getWriteLock().lock();
try {
@@ -295,8 +396,14 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
myStorage.addValue(entry.getKey(), inputId, entry.getValue());
}
}
if (myInputsIndex != null) {
try {
try {
if (myHasSnapshotMapping && !((MemoryIndexStorage)getStorage()).isBufferingEnabled()) {
if (savedInputId != NULL_MAPPING && !mySnapshotMapping.containsMapping(savedInputId)) {
mySnapshotMapping.put(savedInputId, newData.keySet());
}
myInputsSnapshotMapping.put(inputId, savedInputId);
} else if (myInputsIndex != null) {
final Set<Key> newKeys = newData.keySet();
if (newKeys.size() > 0) {
myInputsIndex.put(inputId, newKeys);
@@ -305,9 +412,9 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
myInputsIndex.remove(inputId);
}
}
catch (IOException e) {
throw new StorageException(e);
}
}
catch (IOException e) {
throw new StorageException(e);
}
}
finally {
platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/ContentHashesUtil.java
+1 -1
View File
@@ -59,7 +59,7 @@ public class ContentHashesUtil {
public static class HashEnumerator extends PersistentBTreeEnumerator<byte[]> {
public HashEnumerator(File contentsHashesFile, PagedFileStorage.StorageLockContext storageLockContext) throws IOException {
super(contentsHashesFile, new ContentHashesDescriptor(), 1024 * 1024, storageLockContext);
super(contentsHashesFile, new ContentHashesDescriptor(), 64 * 1024, storageLockContext);
}
@Override