MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

when processing indices, intersect file ids with project filter to avoid searching virtual files by id using disk

Browse Source
This commit is contained in:
Maxim.Mossienko
2012-02-21 17:41:11 +04:00
parent d0c2ec1331
commit fccc2ddce4
7 changed files with 154 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/lang-impl/src/com/intellij/find/impl/FindInProjectUtil.java
+1 -9
View File
@@ -509,15 +509,7 @@ public class FindInProjectUtil {
fast |= findModel.isWholeWordsOnly() && findModel.getStringToFind().indexOf('$') < 0;
List<String> words = StringUtil.getWordsIn(findModel.getStringToFind());
// hope long words are rare
Collections.sort(words, new Comparator<String>() {
@Override
public int compare(final String o1, final String o2) {
return o2.length() - o1.length();
}
});
List<String> words = StringUtil.getWordsInStringLongestFirst(findModel.getStringToFind());
for (int i = 0; i < words.size(); i++) {
String word = words.get(i);
platform/lang-impl/src/com/intellij/psi/impl/search/PsiSearchHelperImpl.java
+2 -2
View File
@@ -781,7 +781,7 @@ public class PsiSearchHelperImpl implements PsiSearchHelper {
if (scope instanceof LocalSearchScope) {
registerRequest(locals, primitive, processor);
} else {
final List<String> words = StringUtil.getWordsIn(primitive.word);
final List<String> words = StringUtil.getWordsInStringLongestFirst(primitive.word);
final Set<IdIndexEntry> key = new HashSet<IdIndexEntry>(words.size() * 2);
for (String word : words) {
key.add(new IdIndexEntry(word, primitive.caseSensitive));
@@ -857,7 +857,7 @@ public class PsiSearchHelperImpl implements PsiSearchHelper {
}
private static ArrayList<IdIndexEntry> getWordEntries(String name, boolean caseSensitively) {
List<String> words = StringUtil.getWordsIn(name);
List<String> words = StringUtil.getWordsInStringLongestFirst(name);
final ArrayList<IdIndexEntry> keys = new ArrayList<IdIndexEntry>();
for (String word : words) {
keys.add(new IdIndexEntry(word, caseSensitively));
platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java
+3
View File
@@ -199,10 +199,13 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe
index.getReadLock().lock();
final ValueContainer<TIntArrayList> container = index.getData(key);
final FileBasedIndex.ProjectIndexableFilesFilter projectFilesFilter = FileBasedIndex.getInstance().projectIndexableFiles(project);
container.forEach(new ValueContainer.ContainerAction<TIntArrayList>() {
@Override
public void perform(final int id, final TIntArrayList value) {
ProgressManager.checkCanceled();
if (projectFilesFilter != null && !projectFilesFilter.contains(id)) return;
final VirtualFile file = IndexInfrastructure.findFileByIdIfCached(fs, id);
if (file == null || scope != null && !scope.contains(file)) {
return;
platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndex.java
+65 -10
View File
@@ -77,6 +77,7 @@ import org.jetbrains.annotations.Nullable;
import javax.swing.*;
import java.io.*;
import java.lang.ref.SoftReference;
import java.util.*;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ScheduledFuture;
@@ -122,6 +123,7 @@ public class FileBasedIndex implements ApplicationComponent {
private final boolean myIsUnitTestMode;
private ScheduledFuture<?> myFlushingFuture;
private volatile int myLocalModCount;
private volatile int myFilesModCount;
public void requestReindex(final VirtualFile file) {
myChangedFilesCollector.invalidateIndices(file, true);
@@ -860,7 +862,7 @@ public class FileBasedIndex implements ApplicationComponent {
private <K, V, R> R processExceptions(final ID<K, V> indexId,
@Nullable final VirtualFile restrictToFile,
@Nullable final VirtualFile restrictToFile,
final GlobalSearchScope filter,
ThrowableConvertor<UpdatableIndex<K, V, FileContent>, R, StorageException> computable) {
try {
@@ -921,10 +923,12 @@ public class FileBasedIndex implements ApplicationComponent {
}
else {
final PersistentFS fs = (PersistentFS)ManagingFS.getInstance();
ProjectIndexableFilesFilter projectFilesSet = projectIndexableFiles(filter.getProject());
VALUES_LOOP: for (final Iterator<V> valueIt = container.getValueIterator(); valueIt.hasNext();) {
final V value = valueIt.next();
for (final ValueContainer.IntIterator inputIdsIterator = container.getInputIdsIterator(value); inputIdsIterator.hasNext();) {
final int id = inputIdsIterator.next();
if (projectFilesSet != null && !projectFilesSet.contains(id)) continue;
VirtualFile file = IndexInfrastructure.findFileByIdIfCached(fs, id);
if (file != null && filter.accept(file)) {
shouldContinue = processor.process(file, value);
@@ -944,24 +948,70 @@ public class FileBasedIndex implements ApplicationComponent {
final Boolean result = processExceptions(indexId, restrictToFile, filter, keyProcessor);
return result == null || result.booleanValue();
}
public <K, V> boolean processFilesContainingAllKeys(final ID<K, V> indexId,
final Collection<K> dataKeys,
final GlobalSearchScope filter,
@Nullable Condition<V> valueChecker,
final Processor<VirtualFile> processor) {
final TIntHashSet set = collectFileIdsContainingAllKeys(indexId, dataKeys, filter, valueChecker);
ProjectIndexableFilesFilter filesSet = projectIndexableFiles(filter.getProject());
final TIntHashSet set = collectFileIdsContainingAllKeys(indexId, dataKeys, filter, valueChecker, filesSet);
if (set == null) {
return false;
}
return processVirtualFiles(set, filter, processor);
}
private static final Key<SoftReference<ProjectIndexableFilesFilter>> ourProjectFilesSetKey = Key.create("projectFiles");
public static final class ProjectIndexableFilesFilter extends BloomFilterBase {
private static final int MAGIC = 0x278DDE6D;
private final int myModificationCount;
private ProjectIndexableFilesFilter(TIntHashSet set, int modificationCount) {
super(set.size(), 0.005d);
myModificationCount = modificationCount;
set.forEach(new TIntProcedure() {
@Override
public boolean execute(int value) {
addIt(value, value * MAGIC);
return true;
}
});
}
public boolean contains(int id) {
return maybeContains(id, id * MAGIC);
}
}
public @Nullable ProjectIndexableFilesFilter projectIndexableFiles(Project project) {
if (project == null || HeavyProcessLatch.INSTANCE.isRunning()) return null;
SoftReference<ProjectIndexableFilesFilter> reference = project.getUserData(ourProjectFilesSetKey);
ProjectIndexableFilesFilter data = reference != null ? reference.get() : null;
if (data != null && data.myModificationCount == myFilesModCount) return data;
final TIntHashSet filesSet = new TIntHashSet();
iterateIndexableFiles(new ContentIterator() {
@Override
public boolean processFile(VirtualFile fileOrDir) {
filesSet.add(((VirtualFileWithId)fileOrDir).getId());
return true;
}
}, project, ProgressManager.getInstance().getProgressIndicator());
ProjectIndexableFilesFilter files = new ProjectIndexableFilesFilter(filesSet, myFilesModCount);
project.putUserData(ourProjectFilesSetKey, new SoftReference<ProjectIndexableFilesFilter>(files));
return files;
}
@Nullable
private <K, V> TIntHashSet collectFileIdsContainingAllKeys(final ID<K, V> indexId,
final Collection<K> dataKeys,
final GlobalSearchScope filter,
@Nullable final Condition<V> valueChecker) {
@Nullable final Condition<V> valueChecker,
@Nullable final ProjectIndexableFilesFilter projectFilesFilter
) {
final ThrowableConvertor<UpdatableIndex<K, V, FileContent>, TIntHashSet, StorageException> convertor =
new ThrowableConvertor<UpdatableIndex<K, V, FileContent>, TIntHashSet, StorageException>() {
@Nullable
@@ -981,7 +1031,8 @@ public class FileBasedIndex implements ApplicationComponent {
}
for (final ValueContainer.IntIterator inputIdsIterator = container.getInputIdsIterator(value); inputIdsIterator.hasNext(); ) {
final int id = inputIdsIterator.next();
if (mainIntersection == null || mainIntersection.contains(id)) {
if ((mainIntersection == null || mainIntersection.contains(id)) &&
(projectFilesFilter == null || projectFilesFilter.contains(id))) {
copy.add(id);
}
}
@@ -1063,8 +1114,10 @@ public class FileBasedIndex implements ApplicationComponent {
final PersistentFS fs = (PersistentFS)ManagingFS.getInstance();
TIntIterator ids = join(locals).iterator();
ProjectIndexableFilesFilter projectIndexableFilesFilter = projectIndexableFiles(project);
while (ids.hasNext()) {
int id = ids.next();
if (projectIndexableFilesFilter != null && !projectIndexableFilesFilter.contains(id)) continue;
//VirtualFile file = IndexInfrastructure.findFileById(fs, id);
VirtualFile file = IndexInfrastructure.findFileByIdIfCached(fs, id);
if (file != null && filter.accept(file)) {
@@ -1611,7 +1664,7 @@ public class FileBasedIndex implements ApplicationComponent {
@Override
public void fileCreated(final VirtualFileEvent event) {
markDirty(event);
markDirty(event, false);
}
@Override
@@ -1621,7 +1674,7 @@ public class FileBasedIndex implements ApplicationComponent {
@Override
public void fileCopied(final VirtualFileCopyEvent event) {
markDirty(event);
markDirty(event, false);
}
@Override
@@ -1636,7 +1689,7 @@ public class FileBasedIndex implements ApplicationComponent {
@Override
public void contentsChanged(final VirtualFileEvent event) {
markDirty(event);
markDirty(event, true);
}
@Override
@@ -1657,17 +1710,18 @@ public class FileBasedIndex implements ApplicationComponent {
if (event.getPropertyName().equals(VirtualFile.PROP_NAME)) {
// indexes may depend on file name
if (!event.getFile().isDirectory()) {
markDirty(event);
markDirty(event, false);
}
}
}
private void markDirty(final VirtualFileEvent event) {
private void markDirty(final VirtualFileEvent event, final boolean contentChange) {
final VirtualFile eventFile = event.getFile();
cleanProcessedFlag(eventFile);
iterateIndexableFiles(eventFile, new Processor<VirtualFile>() {
@Override
public boolean process(final VirtualFile file) {
if (!contentChange) ++myFilesModCount;
FileContent fileContent = null;
// handle 'content-less' indices separately
for (ID<?, ?> indexId : myNotRequiringContentIndices) {
@@ -2041,6 +2095,7 @@ public class FileBasedIndex implements ApplicationComponent {
}
public CollectingContentIterator createContentIterator() {
++myFilesModCount;
return new UnindexedFilesFinder();
}
platform/util/src/com/intellij/openapi/util/text/StringUtil.java
+12
View File
@@ -54,6 +54,18 @@ public class StringUtil {
}
};
public static List<String> getWordsInStringLongestFirst(String find) {
List<String> words = getWordsIn(find);
// hope long words are rare
Collections.sort(words, new Comparator<String>() {
@Override
public int compare(final String o1, final String o2) {
return o2.length() - o1.length();
}
});
return words;
}
@NotNull
public static String escapePattern(final @NotNull String text) {
return replace(replace(text, "'", "''"), "{", "'{'");
platform/util/src/com/intellij/util/BloomFilterBase.java
+60
View File
@@ -0,0 +1,60 @@
/*
* Copyright 2000-2012 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.util;
public class BloomFilterBase {
private final int myHashFunctionCount;
private final int myBitsCount;
private final long[] myElementsSet;
private static final int BITS_PER_ELEMENT = 6;
protected BloomFilterBase(int _maxElementCount, double probability) {
int bitsPerElementFactor = (int)Math.ceil(-Math.log(probability) / (Math.log(2) * Math.log(2)));
myHashFunctionCount = (int)Math.ceil(bitsPerElementFactor * Math.log(2));
int bitsCount = _maxElementCount * bitsPerElementFactor;
if ((bitsCount & 1) == 0) ++bitsCount;
while(!isPrime(bitsCount)) bitsCount += 2;
myBitsCount = bitsCount;
myElementsSet = new long[(bitsCount >> BITS_PER_ELEMENT) + 1];
}
private static boolean isPrime(int bits) {
if ((bits & 1) == 0 || bits % 3 == 0) return false;
int sqrt = (int)Math.sqrt(bits);
for(int i = 6; i <= sqrt; i += 6) {
if (bits % (i - 1) == 0 || bits % (i + 1) == 0) return false;
}
return true;
}
protected final void addIt(int prime, int prime2) {
for(int i = 0; i < myHashFunctionCount; ++i) {
int abs = Math.abs(i * prime + prime2 * (myHashFunctionCount - i)) % myBitsCount;
myElementsSet[abs >> BITS_PER_ELEMENT] |= (1L << abs);
}
}
protected final boolean maybeContains(int prime, int prime2) {
for(int i = 0; i < myHashFunctionCount; ++i) {
int abs = Math.abs(i * prime + prime2 * (myHashFunctionCount - i)) % myBitsCount;
if ((myElementsSet[abs >> BITS_PER_ELEMENT] & (1L << abs)) == 0) return false;
}
return true;
}
}
platform/util/src/com/intellij/util/lang/ClasspathCache.java
+11 -36
View File
@@ -21,6 +21,7 @@ package com.intellij.util.lang;
import com.intellij.openapi.util.text.StringHash;
import com.intellij.util.ArrayUtil;
import com.intellij.util.BloomFilterBase;
import com.intellij.util.SmartList;
import com.intellij.util.containers.HashMap;
import gnu.trove.THashMap;
@@ -30,7 +31,6 @@ import gnu.trove.TIntObjectHashMap;
import org.jetbrains.annotations.Nullable;
import sun.misc.Resource;
import java.util.BitSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -44,7 +44,7 @@ public class ClasspathCache {
private THashMap<String, Set<Loader>> myResources2LoadersTempMap = new THashMap<String, Set<Loader>>();
private static final double PROBABILITY = 0.005d;
private BloomFilter myNameFilter;
private Name2LoaderFilter myNameFilter;
private boolean myTempMapMode = true;
public ClasspathCache() {
@@ -232,7 +232,7 @@ public class ClasspathCache {
nBits += (int)(nBits * 0.03d); // allow some growth for Idea main loader
}
myNameFilter = new BloomFilter(nBits, PROBABILITY);
myNameFilter = new Name2LoaderFilter(nBits, PROBABILITY);
for(Map.Entry<String, Set<Loader>> e:myResources2LoadersTempMap.entrySet()) {
final String name = e.getKey();
@@ -246,50 +246,25 @@ public class ClasspathCache {
}
}
static class BloomFilter {
private final int myHashFunctionCount;
private final int NBITS;
private final BitSet myResourceMap;
private static class Name2LoaderFilter extends BloomFilterBase {
private static final int SEED = 31;
BloomFilter(int nBits, double probability) {
int bitsPerNameFactor = (int)Math.ceil(-Math.log(probability) / (Math.log(2) * Math.log(2)));
myHashFunctionCount = (int)Math.ceil(bitsPerNameFactor * Math.log(2));
nBits = nBits * bitsPerNameFactor;
if ((nBits & 1) == 0) ++nBits;
while(!isPrime(nBits)) nBits += 2;
NBITS = nBits;
myResourceMap = new BitSet(NBITS);
}
private static boolean isPrime(int bits) {
if ((bits & 1) == 0) return false;
int sqrt = (int)Math.sqrt(bits);
for(int i = 3; i <= sqrt; i+=2) {
if (bits % i == 0) return false;
}
return true;
Name2LoaderFilter(int nBits, double probability) {
super(nBits, probability);
}
private boolean maybeContains(String name, Loader loader) {
int hash = hashFromNameAndLoader(name, loader, StringHash.murmur(name, SEED));
int hash2 = hashFromNameAndLoader(name, loader, hash);
for (int i = 0; i < myHashFunctionCount; ++i) {
if (!myResourceMap.get(Math.abs((hash + i * hash2) % NBITS))) return false;
}
return true;
return maybeContains(hash, hash2);
}
public void add(String name, Loader loader) {
int hash1 = hashFromNameAndLoader(name, loader, StringHash.murmur(name, SEED));
int hash2 = hashFromNameAndLoader(name, loader, hash1);
void add(String name, Loader loader) {
int hash = hashFromNameAndLoader(name, loader, StringHash.murmur(name, SEED));
int hash2 = hashFromNameAndLoader(name, loader, hash);
for (int i = 0; i < myHashFunctionCount; ++i) {
myResourceMap.set(Math.abs((hash1 + i * hash2) % NBITS));
}
addIt(hash, hash2);
}
private int hashFromNameAndLoader(String name, Loader loader, int n) {