MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

new api / impl for processing keys in filebased / stub index and processing names in goto file/class.

Browse Source
This commit is contained in:
Maxim.Mossienko
2013-08-21 17:30:34 +02:00
parent 4872c6937c
commit 75c58e1845
18 changed files with 311 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/indexing-api/src/com/intellij/psi/stubs/StringStubIndexExtension.java
+4
View File
@@ -33,4 +33,8 @@ public abstract class StringStubIndexExtension<Psi extends PsiElement> extends A
public KeyDescriptor<String> getKeyDescriptor() {
return new EnumeratorStringDescriptor();
}
public boolean traceKeyHashToVirtualFileMapping() {
return false;
}
}
platform/indexing-api/src/com/intellij/psi/stubs/StubIndex.java
+6
View File
@@ -28,7 +28,9 @@ import com.intellij.psi.search.GlobalSearchScope;
import com.intellij.psi.util.PsiUtilCore;
import com.intellij.util.Processor;
import com.intellij.util.indexing.FileBasedIndex;
import com.intellij.util.indexing.IdFilter;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.util.Collection;
import java.util.Iterator;
@@ -59,6 +61,10 @@ public abstract class StubIndex {
public abstract <K> boolean processAllKeys(@NotNull StubIndexKey<K, ?> indexKey, @NotNull Project project, Processor<K> processor);
public <K> boolean processAllKeys(@NotNull StubIndexKey<K, ?> indexKey, Processor<K> processor, GlobalSearchScope scope, @Nullable IdFilter idFilter) {
return processAllKeys(indexKey, scope.getProject(), processor);
}
public <Key, Psi extends PsiElement> Collection<Psi> safeGet(@NotNull StubIndexKey<Key, Psi> indexKey,
@NotNull Key key,
@NotNull final Project project,
platform/indexing-api/src/com/intellij/util/indexing/FileBasedIndex.java
+7
View File
@@ -126,6 +126,10 @@ public abstract class FileBasedIndex implements BaseComponent {
*/
public abstract <K> boolean processAllKeys(@NotNull ID<K, ?> indexId, Processor<K> processor, @Nullable Project project);
public <K> boolean processAllKeys(@NotNull ID<K, ?> indexId, @NotNull Processor<K> processor, @NotNull GlobalSearchScope scope, @Nullable IdFilter idFilter) {
return processAllKeys(indexId, processor, scope.getProject());
}
public interface ValueProcessor<V> {
/**
* @param value a value to process
@@ -145,4 +149,7 @@ public abstract class FileBasedIndex implements BaseComponent {
public interface FileTypeSpecificInputFilter extends InputFilter {
void registerFileTypesUsedForIndexing(@NotNull Consumer<FileType> fileTypeSink);
}
// TODO: remove once changes becomes permamnent
public static final boolean ourEnableTracingOfKeyHashToVirtualFileMapping = ApplicationManager.getApplication().isInternal();
}
platform/indexing-api/src/com/intellij/util/indexing/FileBasedIndexExtension.java
+4
View File
@@ -80,6 +80,10 @@ public abstract class FileBasedIndexExtension<K, V> {
return false;
}
public boolean traceKeyHashToVirtualFileMapping() {
return false;
}
/** Per-filetype index version support */
public Map<FileType, Integer> getVersionMap() {
return Collections.emptyMap();
platform/indexing-api/src/com/intellij/util/indexing/IdFilter.java
+23
View File
@@ -0,0 +1,23 @@
/*
* Copyright 2000-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.util.indexing;
/**
* Created by Maxim.Mossienko on 8/14/13.
*/
public abstract class IdFilter {
public abstract boolean contains(int id);
}
platform/lang-impl/src/com/intellij/ide/util/gotoByName/ChooseByNameBase.java
+13 -3
View File
@@ -70,6 +70,7 @@ import com.intellij.util.Alarm;
import com.intellij.util.Consumer;
import com.intellij.util.Processor;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.indexing.FileBasedIndex;
import com.intellij.util.text.Matcher;
import com.intellij.util.text.MatcherHolder;
import com.intellij.util.ui.AsyncProcessIcon;
@@ -153,6 +154,7 @@ public abstract class ChooseByNameBase {
private String myFindUsagesTitle;
private ShortcutSet myCheckBoxShortcut;
protected boolean myInitIsDone;
static final boolean ourLoadNamesEachTime = FileBasedIndex.ourEnableTracingOfKeyHashToVirtualFileMapping;
public boolean checkDisposed() {
if (myDisposedFlag && myPostponedOkAction != null && !myPostponedOkAction.isProcessed()) {
@@ -795,7 +797,10 @@ public abstract class ChooseByNameBase {
}
}
if (index == 1 && myModel instanceof ContributorsBasedGotoByModel && myNames[0] != null) {
if (index == 1 &&
myModel instanceof ContributorsBasedGotoByModel &&
((ContributorsBasedGotoByModel)myModel).sameNamesForProjectAndLibraries() &&
myNames[0] != null) {
// there is no way in indices to have different keys for project symbols vs libraries, we always have same ones
myNames[1] = myNames[0];
return;
@@ -814,6 +819,10 @@ public abstract class ChooseByNameBase {
@NotNull
public String[] getNames(boolean checkboxState) {
if (ourLoadNamesEachTime) {
myNames[checkboxState ? 1 : 0] = null;
ensureNamesLoaded(checkboxState);
}
return checkboxState ? myNames[1] : myNames[0];
}
@@ -1193,8 +1202,9 @@ public abstract class ChooseByNameBase {
}
final String text = myTextField.getText();
if (text.length() == 0) return Collections.emptyList();
final boolean checkBoxState = myCheckBox.isSelected();
//ensureNamesLoaded(checkBoxState);
if (ourLoadNamesEachTime) ensureNamesLoaded(checkBoxState);
final String[] names = checkBoxState ? myNames[1] : myNames[0];
if (names == null) return Collections.emptyList();
@@ -1489,7 +1499,7 @@ public abstract class ChooseByNameBase {
public void run() {
try {
boolean everywhere = myCheckboxState;
ensureNamesLoaded(everywhere);
if (!ourLoadNamesEachTime) ensureNamesLoaded(everywhere);
addElementsByPattern(myPattern, elements, myCancelled, everywhere);
}
catch (ProcessCanceledException e) {
platform/lang-impl/src/com/intellij/ide/util/gotoByName/ContributorsBasedGotoByModel.java
+61 -11
View File
@@ -20,6 +20,7 @@ import com.intellij.diagnostic.PluginException;
import com.intellij.ide.plugins.PluginManager;
import com.intellij.ide.util.NavigationItemListCellRenderer;
import com.intellij.navigation.ChooseByNameContributor;
import com.intellij.navigation.EfficientChooseByNameContributor;
import com.intellij.navigation.NavigationItem;
import com.intellij.openapi.application.ReadActionProcessor;
import com.intellij.openapi.diagnostic.Logger;
@@ -32,9 +33,12 @@ import com.intellij.openapi.project.DumbService;
import com.intellij.openapi.project.IndexNotReadyException;
import com.intellij.openapi.project.Project;
import com.intellij.util.ArrayUtil;
import com.intellij.util.CommonProcessors;
import com.intellij.util.Processor;
import com.intellij.util.containers.ConcurrentHashMap;
import com.intellij.util.containers.ContainerUtil;
import gnu.trove.THashSet;
import gnu.trove.TIntHashSet;
import org.jetbrains.annotations.NotNull;
import javax.swing.*;
@@ -47,7 +51,7 @@ import java.util.List;
/**
* Contributor-based goto model
*/
public abstract class ContributorsBasedGotoByModel implements ChooseByNameModel {
public abstract class ContributorsBasedGotoByModel implements EfficientChooseByNameModel {
public static final Logger LOG = Logger.getInstance("#com.intellij.ide.util.gotoByName.ContributorsBasedGotoByModel");
protected final Project myProject;
@@ -74,11 +78,14 @@ public abstract class ContributorsBasedGotoByModel implements ChooseByNameModel
};
}
@NotNull
@Override
public String[] getNames(final boolean checkBoxState) {
final THashSet<String> allNames = ContainerUtil.newTroveSet();
public boolean sameNamesForProjectAndLibraries() {
return !ChooseByNameBase.ourLoadNamesEachTime;
}
private final ConcurrentHashMap<ChooseByNameContributor, TIntHashSet> myContributorToItsSymbolsMap = new ConcurrentHashMap<ChooseByNameContributor, TIntHashSet>();
@Override
public void processNames(final Processor<String> nameProcessor, final boolean checkBoxState) {
long start = System.currentTimeMillis();
List<ChooseByNameContributor> liveContribs = filterDumb(myContributors);
ProgressIndicator indicator = ProgressManager.getInstance().getProgressIndicator();
@@ -87,10 +94,30 @@ public abstract class ContributorsBasedGotoByModel implements ChooseByNameModel
public boolean processInReadAction(ChooseByNameContributor contributor) {
try {
if (!myProject.isDisposed()) {
String[] names = contributor.getNames(myProject, checkBoxState);
synchronized (allNames) {
allNames.ensureCapacity(names.length);
ContainerUtil.addAll(allNames, names);
long contributorStarted = System.currentTimeMillis();
final TIntHashSet filter = new TIntHashSet(1000);
myContributorToItsSymbolsMap.put(contributor, filter);
if (contributor instanceof EfficientChooseByNameContributor) {
((EfficientChooseByNameContributor)contributor).processNames(new Processor<String>() {
@Override
public boolean process(String s) {
if (nameProcessor.process(s)) {
filter.add(s.hashCode());
}
return true;
}
}, DefaultFileNavigationContributor.getScope(myProject, checkBoxState), DefaultFileNavigationContributor.getFilter(myProject, checkBoxState));
} else {
String[] names = contributor.getNames(myProject, checkBoxState);
for (String element : names) {
if (nameProcessor.process(element)) {
filter.add(element.hashCode());
}
}
}
if (LOG.isDebugEnabled()) {
LOG.debug(contributor + " for " + (System.currentTimeMillis() - contributorStarted));
}
}
}
@@ -114,7 +141,18 @@ public abstract class ContributorsBasedGotoByModel implements ChooseByNameModel
}
long finish = System.currentTimeMillis();
if (LOG.isDebugEnabled()) {
LOG.debug("getNames(): "+(finish-start)+"ms; (got "+allNames.size()+" elements)");
LOG.debug("processNames(): "+(finish-start)+"ms;");
}
}
@NotNull
@Override
public String[] getNames(final boolean checkBoxState) {
final THashSet<String> allNames = ContainerUtil.newTroveSet();
processNames(new CommonProcessors.CollectProcessor<String>(allNames), checkBoxState);
if (LOG.isDebugEnabled()) {
LOG.debug("getNames(): (got "+allNames.size()+" elements)");
}
return ArrayUtil.toStringArray(allNames);
}
@@ -133,6 +171,7 @@ public abstract class ContributorsBasedGotoByModel implements ChooseByNameModel
@NotNull
public Object[] getElementsByName(final String name, final boolean checkBoxState, final String pattern, @NotNull final ProgressIndicator canceled) {
long elementByNameStarted = System.currentTimeMillis();
final List<NavigationItem> items = Collections.synchronizedList(new ArrayList<NavigationItem>());
Processor<ChooseByNameContributor> processor = new Processor<ChooseByNameContributor>() {
@@ -141,8 +180,12 @@ public abstract class ContributorsBasedGotoByModel implements ChooseByNameModel
if (myProject.isDisposed()) {
return true;
}
TIntHashSet filter = myContributorToItsSymbolsMap.get(contributor);
if (!filter.contains(name.hashCode())) return true;
try {
for (NavigationItem item : contributor.getItemsByName(name, pattern, myProject, checkBoxState)) {
long contributorStarted = System.currentTimeMillis();
NavigationItem[] itemsByName = contributor.getItemsByName(name, pattern, myProject, checkBoxState);
for (NavigationItem item : itemsByName) {
canceled.checkCanceled();
if (item == null) {
PluginId pluginId = PluginManager.getPluginByClassName(contributor.getClass().getName());
@@ -159,6 +202,10 @@ public abstract class ContributorsBasedGotoByModel implements ChooseByNameModel
items.add(item);
}
}
if (LOG.isDebugEnabled()) {
LOG.debug(System.currentTimeMillis() - contributorStarted + "," + contributor + "," + itemsByName.length);
}
}
catch (ProcessCanceledException ex) {
// index corruption detected, ignore
@@ -173,6 +220,9 @@ public abstract class ContributorsBasedGotoByModel implements ChooseByNameModel
canceled.cancel();
}
canceled.checkCanceled(); // if parallel job execution was canceled because of PCE, rethrow it from here
if (LOG.isDebugEnabled()) {
LOG.debug("Retrieving " + name + ":" + items.size() + " for " + (System.currentTimeMillis() - elementByNameStarted));
}
return ArrayUtil.toObjectArray(items);
}
platform/lang-impl/src/com/intellij/ide/util/gotoByName/DefaultChooseByNameItemProvider.java
+34 -7
View File
@@ -47,11 +47,11 @@ public class DefaultChooseByNameItemProvider implements ChooseByNameItemProvider
}
@Override
public boolean filterElements(@NotNull ChooseByNameBase base,
@NotNull String pattern,
public boolean filterElements(@NotNull final ChooseByNameBase base,
@NotNull final String pattern,
boolean everywhere,
@NotNull ProgressIndicator indicator,
@NotNull Processor<Object> consumer) {
@NotNull final Processor<Object> consumer) {
String namePattern = getNamePattern(base, pattern);
String qualifierPattern = getQualifierPattern(base, pattern);
@@ -60,13 +60,40 @@ public class DefaultChooseByNameItemProvider implements ChooseByNameItemProvider
final ChooseByNameModel model = base.getModel();
String matchingPattern = convertToMatchingPattern(base, namePattern);
List<MatchResult> namesList = new ArrayList<MatchResult>();
String[] names = base.getNames(everywhere);
CollectConsumer<MatchResult> collect = new SynchronizedCollectConsumer<MatchResult>(namesList);
processNamesByPattern(base, names, matchingPattern, indicator, collect);
final CollectConsumer<MatchResult> collect = new SynchronizedCollectConsumer<MatchResult>(namesList);
long started;
if (model instanceof EfficientChooseByNameModel) {
indicator.checkCanceled();
started = System.currentTimeMillis();
final MinusculeMatcher matcher = buildPatternMatcher(matchingPattern, NameUtil.MatchingCaseSensitivity.NONE);
((EfficientChooseByNameModel)model).processNames(new Processor<String>() {
@Override
public boolean process(String sequence) {
ProgressManager.checkCanceled();
MatchResult result = matches(base, pattern, matcher, sequence);
if (result != null) {
collect.consume(result);
return true;
}
return false;
}
}, everywhere);
System.out.println("loaded + matched:"+ (System.currentTimeMillis() - started)+ "," + collect.getResult().size());
} else {
String[] names = base.getNames(everywhere);
started = System.currentTimeMillis();
processNamesByPattern(base, names, matchingPattern, indicator, collect);
System.out.println("matched:"+ (System.currentTimeMillis() - started)+ "," + names.length);
}
indicator.checkCanceled();
sortNamesList(matchingPattern, (List<MatchResult>)collect.getResult());
started = System.currentTimeMillis();
List<MatchResult> results = (List<MatchResult>)collect.getResult();
sortNamesList(matchingPattern, results);
System.out.println("sorted:"+ (System.currentTimeMillis() - started) + ",results:" + results.size());
indicator.checkCanceled();
List<Object> sameNameElements = new SmartList<Object>();
platform/lang-impl/src/com/intellij/ide/util/gotoByName/EfficientChooseByNameModel.java
+22
View File
@@ -0,0 +1,22 @@
/*
* Copyright 2000-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.ide.util.gotoByName;
import com.intellij.util.Processor;
public interface EfficientChooseByNameModel extends ChooseByNameModel {
void processNames(Processor<String> processor, boolean inLibraries);
}
platform/lang-impl/src/com/intellij/navigation/EfficientChooseByNameContributor.java
+25
View File
@@ -0,0 +1,25 @@
/*
* Copyright 2000-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.navigation;
import com.intellij.openapi.project.Project;
import com.intellij.psi.search.GlobalSearchScope;
import com.intellij.util.Processor;
import com.intellij.util.indexing.IdFilter;
public interface EfficientChooseByNameContributor extends ChooseByNameContributor {
void processNames(Processor<String> processor, GlobalSearchScope scope, IdFilter filter);
}
platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java
+10 -3
View File
@@ -142,7 +142,9 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe
IndexInfrastructure.getStorageFile(indexKey),
extension.getKeyDescriptor(),
new StubIdExternalizer(),
extension.getCacheSize()
extension.getCacheSize(),
false,
extension instanceof StringStubIndexExtension && ((StringStubIndexExtension)extension).traceKeyHashToVirtualFileMapping()
);
final MemoryIndexStorage<K, StubIdList> memStorage = new MemoryIndexStorage<K, StubIdList>(storage);
@@ -285,11 +287,16 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe
@Override
public <K> boolean processAllKeys(@NotNull StubIndexKey<K, ?> indexKey, @NotNull Project project, Processor<K> processor) {
FileBasedIndex.getInstance().ensureUpToDate(StubUpdatingIndex.INDEX_ID, project, GlobalSearchScope.allScope(project));
return processAllKeys(indexKey, processor, GlobalSearchScope.allScope(project), null);
}
public <K> boolean processAllKeys(@NotNull StubIndexKey<K, ?> indexKey, Processor<K> processor, GlobalSearchScope scope, @Nullable IdFilter idFilter) {
FileBasedIndex.getInstance().ensureUpToDate(StubUpdatingIndex.INDEX_ID, scope.getProject(), scope);
final MyIndex<K> index = (MyIndex<K>)myIndices.get(indexKey);
try {
return index.processAllKeys(processor);
return index.processAllKeys(processor, idFilter);
}
catch (StorageException e) {
forceRebuild(e);
platform/lang-impl/src/com/intellij/util/indexing/AbstractIndex.java
+2 -1
View File
@@ -18,6 +18,7 @@ package com.intellij.util.indexing;
import com.intellij.util.Processor;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
/**
* @author Eugene Zhuravlev
@@ -27,5 +28,5 @@ public interface AbstractIndex<Key, Value> {
@NotNull
ValueContainer<Value> getData(Key key) throws StorageException;
boolean processAllKeys(Processor<Key> processor) throws StorageException;
boolean processAllKeys(Processor<Key> processor, @Nullable IdFilter idFilter) throws StorageException;
}
platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java
+12 -5
View File
@@ -413,7 +413,8 @@ public class FileBasedIndexImpl extends FileBasedIndex {
extension.getKeyDescriptor(),
extension.getValueExternalizer(),
extension.getCacheSize(),
extension.isKeyHighlySelective()
extension.isKeyHighlySelective(),
extension.traceKeyHashToVirtualFileMapping()
);
final MemoryIndexStorage<K, V> memStorage = new MemoryIndexStorage<K, V>(storage);
@@ -722,13 +723,17 @@ public class FileBasedIndexImpl extends FileBasedIndex {
@Override
public <K> boolean processAllKeys(@NotNull final ID<K, ?> indexId, Processor<K> processor, @Nullable Project project) {
return processAllKeys(indexId, processor, project != null ? GlobalSearchScope.allScope(project) : new EverythingGlobalScope(), null);
}
public <K> boolean processAllKeys(@NotNull ID<K, ?> indexId, Processor<K> processor, @NotNull GlobalSearchScope scope, @Nullable IdFilter idFilter) {
try {
final UpdatableIndex<K, ?, FileContent> index = getIndex(indexId);
if (index == null) {
return true;
}
ensureUpToDate(indexId, project, project != null ? GlobalSearchScope.allScope(project) : new EverythingGlobalScope());
return index.processAllKeys(processor);
ensureUpToDate(indexId, scope.getProject(), scope);
return index.processAllKeys(processor, idFilter);
}
catch (StorageException e) {
scheduleRebuild(indexId, e);
@@ -1014,7 +1019,7 @@ public class FileBasedIndexImpl extends FileBasedIndex {
myContentlessIndicesUpdateQueue.signalUpdateEnd();
}
public static final class ProjectIndexableFilesFilter {
public static final class ProjectIndexableFilesFilter extends IdFilter {
private static final int SHIFT = 6;
private static final int MASK = (1 << SHIFT) - 1;
private final long[] myBitMask;
@@ -1735,7 +1740,9 @@ public class FileBasedIndexImpl extends FileBasedIndex {
}
FileType fileType = file.getFileType();
if(isProjectOrWorkspaceFile(file, fileType)) return Collections.emptyList();
if (fileType == StdFileTypes.HTML || fileType == StdFileTypes.XML) {
int a = 1;
}
List<ID<?, ?>> ids = myFileType2IndicesWithFileTypeInfoMap.get(fileType);
if (ids == null) ids = myIndicesWithoutFileTypeInfo;
return ids;
platform/lang-impl/src/com/intellij/util/indexing/IndexStorage.java
+2 -1
View File
@@ -18,6 +18,7 @@ package com.intellij.util.indexing;
import com.intellij.util.Processor;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.io.Flushable;
import java.io.IOException;
@@ -38,7 +39,7 @@ public interface IndexStorage<Key, Value> extends Flushable {
@NotNull
ValueContainer<Value> read(Key key) throws StorageException;
boolean processKeys(Processor<Key> processor) throws StorageException;
boolean processKeys(Processor<Key> processor, @Nullable IdFilter idFilter) throws StorageException;
Collection<Key> getKeys() throws StorageException;
platform/lang-impl/src/com/intellij/util/indexing/MapIndexStorage.java
+74 -7
View File
@@ -22,12 +22,13 @@ import com.intellij.openapi.util.io.FileUtil;
import com.intellij.util.CommonProcessors;
import com.intellij.util.Processor;
import com.intellij.util.containers.SLRUCache;
import com.intellij.util.io.DataExternalizer;
import com.intellij.util.io.KeyDescriptor;
import com.intellij.util.io.PersistentMap;
import com.intellij.util.io.*;
import gnu.trove.TIntHashSet;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
@@ -42,7 +43,9 @@ import java.util.concurrent.locks.ReentrantLock;
*/
public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Value>{
private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.MapIndexStorage");
private final boolean myBuildKeyHashToVirtualFileMapping;
private PersistentMap<Key, ValueContainer<Value>> myMap;
private PersistentBTreeEnumerator<int[]> myKeyHashToVirtualFileMapping;
private SLRUCache<Key, ChangeTrackingValueContainer<Value>> myCache;
private final File myStorageFile;
private final KeyDescriptor<Key> myKeyDescriptor;
@@ -71,20 +74,23 @@ public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Valu
@NotNull DataExternalizer<Value> valueExternalizer,
final int cacheSize
) throws IOException {
this(storageFile, keyDescriptor, valueExternalizer, cacheSize, false);
this(storageFile, keyDescriptor, valueExternalizer, cacheSize, false, false);
}
public MapIndexStorage(@NotNull File storageFile,
@NotNull KeyDescriptor<Key> keyDescriptor,
@NotNull DataExternalizer<Value> valueExternalizer,
final int cacheSize,
boolean highKeySelectivity) throws IOException {
boolean highKeySelectivity,
boolean buildKeyHashToVirtualFileMapping
) throws IOException {
myStorageFile = storageFile;
myKeyDescriptor = keyDescriptor;
myCacheSize = cacheSize;
myDataExternalizer = valueExternalizer;
myHighKeySelectivity = highKeySelectivity;
myBuildKeyHashToVirtualFileMapping = buildKeyHashToVirtualFileMapping && FileBasedIndex.ourEnableTracingOfKeyHashToVirtualFileMapping;
initMapAndCache();
}
@@ -133,6 +139,37 @@ public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Valu
};
myMap = map;
myKeyHashToVirtualFileMapping = myBuildKeyHashToVirtualFileMapping ? new PersistentBTreeEnumerator<int[]>(getProjectFile(), new KeyDescriptor<int[]>() {
@Override
public void save(DataOutput out, int[] value) throws IOException {
DataInputOutputUtil.writeINT(out, value[0]);
DataInputOutputUtil.writeINT(out, value[1]);
}
@Override
public int[] read(DataInput in) throws IOException {
return new int[] {DataInputOutputUtil.readINT(in), DataInputOutputUtil.readINT(in)};
}
@Override
public int getHashCode(int[] value) {
return value[0] * 31 + value[1];
}
@Override
public boolean isEqual(int[] val1, int[] val2) {
return val1[0] == val2[0] && val1[1] == val2[1];
}
}, 4096) {
protected boolean serializationEquivalenceIsExhausting() {
return true;
}
}: null;
}
private File getProjectFile() {
return new File(myStorageFile.getPath() + ".project");
}
@Override
@@ -143,6 +180,7 @@ public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Valu
myCache.clear();
myMap.force();
}
if (myKeyHashToVirtualFileMapping != null) myKeyHashToVirtualFileMapping.force();
}
finally {
l.unlock();
@@ -154,6 +192,7 @@ public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Valu
try {
myLowMemoryFlusher.stop();
flush();
if (myKeyHashToVirtualFileMapping != null) myKeyHashToVirtualFileMapping.close();
myMap.close();
}
catch (IOException e) {
@@ -175,12 +214,14 @@ public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Valu
public void clear() throws StorageException{
try {
myMap.close();
if (myKeyHashToVirtualFileMapping != null) myKeyHashToVirtualFileMapping.close();
}
catch (IOException e) {
LOG.error(e);
}
try {
FileUtil.delete(myStorageFile);
if (myKeyHashToVirtualFileMapping != null) IOUtil.deleteAllFilesStartingWith(getProjectFile());
initMapAndCache();
}
catch (IOException e) {
@@ -199,10 +240,32 @@ public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Valu
}
@Override
public boolean processKeys(final Processor<Key> processor) throws StorageException {
public boolean processKeys(final Processor<Key> processor, final IdFilter idFilter) throws StorageException {
l.lock();
try {
myCache.clear(); // this will ensure that all new keys are made into the map
if (myBuildKeyHashToVirtualFileMapping && idFilter != null) {
final TIntHashSet hashMaskSet = new TIntHashSet(1000);
long l = System.currentTimeMillis();
myKeyHashToVirtualFileMapping.iterateData(new Processor<int[]>() {
@Override
public boolean process(int[] key) {
if (!idFilter.contains(key[1])) return true;
hashMaskSet.add(key[0]);
return true;
}
});
if (LOG.isDebugEnabled()) {
LOG.debug("Scanned keyHashToVirtualFileMapping of " + myStorageFile + " for " + (System.currentTimeMillis() - l));
}
return myMap.processKeys(new Processor<Key>() {
@Override
public boolean process(Key key) {
if (!hashMaskSet.contains(myKeyDescriptor.getHashCode(key))) return true;
return processor.process(key);
}
});
}
return myMap.processKeys(processor);
}
catch (IOException e) {
@@ -227,7 +290,7 @@ public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Valu
@Override
public Collection<Key> getKeys() throws StorageException {
List<Key> keys = new ArrayList<Key>();
processKeys(new CommonProcessors.CollectProcessor<Key>(keys));
processKeys(new CommonProcessors.CollectProcessor<Key>(keys), null);
return keys;
}
@@ -256,6 +319,10 @@ public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Valu
@Override
public void addValue(final Key key, final int inputId, final Value value) throws StorageException {
try {
if (myKeyHashToVirtualFileMapping != null) {
myKeyHashToVirtualFileMapping.enumerate(new int[] { myKeyDescriptor.getHashCode(key), inputId });
}
myMap.markDirty();
if (!myHighKeySelectivity) {
read(key).addValue(inputId, value);
platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java
+2 -2
View File
@@ -155,11 +155,11 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
}
@Override
public boolean processAllKeys(Processor<Key> processor) throws StorageException {
public boolean processAllKeys(Processor<Key> processor, IdFilter idFilter) throws StorageException {
final Lock lock = getReadLock();
try {
lock.lock();
return myStorage.processKeys(processor);
return myStorage.processKeys(processor, idFilter);
}
finally {
lock.unlock();
platform/lang-impl/src/com/intellij/util/indexing/MemoryIndexStorage.java
+3 -3
View File
@@ -101,12 +101,12 @@ public class MemoryIndexStorage<Key, Value> implements IndexStorage<Key, Value>
@Override
public Collection<Key> getKeys() throws StorageException {
final Set<Key> keys = new HashSet<Key>();
processKeys(new CommonProcessors.CollectProcessor<Key>(keys));
processKeys(new CommonProcessors.CollectProcessor<Key>(keys), null);
return keys;
}
@Override
public boolean processKeys(final Processor<Key> processor) throws StorageException {
public boolean processKeys(final Processor<Key> processor, IdFilter idFilter) throws StorageException {
final Set<Key> stopList = new HashSet<Key>();
Processor<Key> decoratingProcessor = new Processor<Key>() {
@@ -128,7 +128,7 @@ public class MemoryIndexStorage<Key, Value> implements IndexStorage<Key, Value>
}
stopList.add(key);
}
return myBackendStorage.processKeys(decoratingProcessor);
return myBackendStorage.processKeys(stopList.size() == 0 && myMap.size() == 0 ? processor : decoratingProcessor, idFilter);
}
@Override
platform/util/src/com/intellij/util/io/PersistentEnumeratorBase.java
+7 -3
View File
@@ -39,7 +39,7 @@ import java.util.List;
* @author jeka
*/
@SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
abstract class PersistentEnumeratorBase<Data> implements Forceable, Closeable {
abstract public class PersistentEnumeratorBase<Data> implements Forceable, Closeable {
protected static final Logger LOG = Logger.getInstance("#com.intellij.util.io.PersistentEnumerator");
protected static final int NULL_ID = 0;
@@ -427,10 +427,14 @@ abstract class PersistentEnumeratorBase<Data> implements Forceable, Closeable {
comparer.close();
if (sameValue[0]) return true;
if (serializationEquivalenceIsExhausting()) return false;
return myDataDescriptor.isEqual(valueOf(idx), value);
}
protected boolean serializationEquivalenceIsExhausting() {
return false;
}
protected int writeData(final Data value, int hashCode) {
try {
markDirty(true);
@@ -498,7 +502,7 @@ abstract class PersistentEnumeratorBase<Data> implements Forceable, Closeable {
return pos;
}
protected boolean iterateData(final Processor<Data> processor) throws IOException {
public boolean iterateData(final Processor<Data> processor) throws IOException {
lockStorage();
try {
if (myKeyStorage == null) {