diff --git a/platform/lang-impl/src/com/intellij/find/impl/FindInProjectUtil.java b/platform/lang-impl/src/com/intellij/find/impl/FindInProjectUtil.java index 1e5bdc1ddfd6..af462f597adc 100644 --- a/platform/lang-impl/src/com/intellij/find/impl/FindInProjectUtil.java +++ b/platform/lang-impl/src/com/intellij/find/impl/FindInProjectUtil.java @@ -509,15 +509,7 @@ public class FindInProjectUtil { fast |= findModel.isWholeWordsOnly() && findModel.getStringToFind().indexOf('$') < 0; - List words = StringUtil.getWordsIn(findModel.getStringToFind()); - - // hope long words are rare - Collections.sort(words, new Comparator() { - @Override - public int compare(final String o1, final String o2) { - return o2.length() - o1.length(); - } - }); + List words = StringUtil.getWordsInStringLongestFirst(findModel.getStringToFind()); for (int i = 0; i < words.size(); i++) { String word = words.get(i); diff --git a/platform/lang-impl/src/com/intellij/psi/impl/search/PsiSearchHelperImpl.java b/platform/lang-impl/src/com/intellij/psi/impl/search/PsiSearchHelperImpl.java index addd3ad3365d..04449a9d5d78 100644 --- a/platform/lang-impl/src/com/intellij/psi/impl/search/PsiSearchHelperImpl.java +++ b/platform/lang-impl/src/com/intellij/psi/impl/search/PsiSearchHelperImpl.java @@ -781,7 +781,7 @@ public class PsiSearchHelperImpl implements PsiSearchHelper { if (scope instanceof LocalSearchScope) { registerRequest(locals, primitive, processor); } else { - final List words = StringUtil.getWordsIn(primitive.word); + final List words = StringUtil.getWordsInStringLongestFirst(primitive.word); final Set key = new HashSet(words.size() * 2); for (String word : words) { key.add(new IdIndexEntry(word, primitive.caseSensitive)); @@ -857,7 +857,7 @@ public class PsiSearchHelperImpl implements PsiSearchHelper { } private static ArrayList getWordEntries(String name, boolean caseSensitively) { - List words = StringUtil.getWordsIn(name); + List words = StringUtil.getWordsInStringLongestFirst(name); final ArrayList keys = new ArrayList(); for (String word : words) { keys.add(new IdIndexEntry(word, caseSensitively)); diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java index 44234f494e30..05d68cf0016e 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java @@ -199,10 +199,13 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe index.getReadLock().lock(); final ValueContainer container = index.getData(key); + final FileBasedIndex.ProjectIndexableFilesFilter projectFilesFilter = FileBasedIndex.getInstance().projectIndexableFiles(project); + container.forEach(new ValueContainer.ContainerAction() { @Override public void perform(final int id, final TIntArrayList value) { ProgressManager.checkCanceled(); + if (projectFilesFilter != null && !projectFilesFilter.contains(id)) return; final VirtualFile file = IndexInfrastructure.findFileByIdIfCached(fs, id); if (file == null || scope != null && !scope.contains(file)) { return; diff --git a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndex.java b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndex.java index f8ce88ae536d..ded673fee44d 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndex.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndex.java @@ -77,6 +77,7 @@ import org.jetbrains.annotations.Nullable; import javax.swing.*; import java.io.*; +import java.lang.ref.SoftReference; import java.util.*; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ScheduledFuture; @@ -122,6 +123,7 @@ public class FileBasedIndex implements ApplicationComponent { private final boolean myIsUnitTestMode; private ScheduledFuture myFlushingFuture; private volatile int myLocalModCount; + private volatile int myFilesModCount; public void requestReindex(final VirtualFile file) { myChangedFilesCollector.invalidateIndices(file, true); @@ -860,7 +862,7 @@ public class FileBasedIndex implements ApplicationComponent { private R processExceptions(final ID indexId, - @Nullable final VirtualFile restrictToFile, + @Nullable final VirtualFile restrictToFile, final GlobalSearchScope filter, ThrowableConvertor, R, StorageException> computable) { try { @@ -921,10 +923,12 @@ public class FileBasedIndex implements ApplicationComponent { } else { final PersistentFS fs = (PersistentFS)ManagingFS.getInstance(); + ProjectIndexableFilesFilter projectFilesSet = projectIndexableFiles(filter.getProject()); VALUES_LOOP: for (final Iterator valueIt = container.getValueIterator(); valueIt.hasNext();) { final V value = valueIt.next(); for (final ValueContainer.IntIterator inputIdsIterator = container.getInputIdsIterator(value); inputIdsIterator.hasNext();) { final int id = inputIdsIterator.next(); + if (projectFilesSet != null && !projectFilesSet.contains(id)) continue; VirtualFile file = IndexInfrastructure.findFileByIdIfCached(fs, id); if (file != null && filter.accept(file)) { shouldContinue = processor.process(file, value); @@ -944,24 +948,70 @@ public class FileBasedIndex implements ApplicationComponent { final Boolean result = processExceptions(indexId, restrictToFile, filter, keyProcessor); return result == null || result.booleanValue(); } - + public boolean processFilesContainingAllKeys(final ID indexId, final Collection dataKeys, final GlobalSearchScope filter, @Nullable Condition valueChecker, final Processor processor) { - final TIntHashSet set = collectFileIdsContainingAllKeys(indexId, dataKeys, filter, valueChecker); + ProjectIndexableFilesFilter filesSet = projectIndexableFiles(filter.getProject()); + final TIntHashSet set = collectFileIdsContainingAllKeys(indexId, dataKeys, filter, valueChecker, filesSet); if (set == null) { return false; } return processVirtualFiles(set, filter, processor); } + private static final Key> ourProjectFilesSetKey = Key.create("projectFiles"); + + public static final class ProjectIndexableFilesFilter extends BloomFilterBase { + private static final int MAGIC = 0x278DDE6D; + private final int myModificationCount; + + private ProjectIndexableFilesFilter(TIntHashSet set, int modificationCount) { + super(set.size(), 0.005d); + myModificationCount = modificationCount; + set.forEach(new TIntProcedure() { + @Override + public boolean execute(int value) { + addIt(value, value * MAGIC); + return true; + } + }); + } + + public boolean contains(int id) { + return maybeContains(id, id * MAGIC); + } + } + + public @Nullable ProjectIndexableFilesFilter projectIndexableFiles(Project project) { + if (project == null || HeavyProcessLatch.INSTANCE.isRunning()) return null; + + SoftReference reference = project.getUserData(ourProjectFilesSetKey); + ProjectIndexableFilesFilter data = reference != null ? reference.get() : null; + if (data != null && data.myModificationCount == myFilesModCount) return data; + + final TIntHashSet filesSet = new TIntHashSet(); + iterateIndexableFiles(new ContentIterator() { + @Override + public boolean processFile(VirtualFile fileOrDir) { + filesSet.add(((VirtualFileWithId)fileOrDir).getId()); + return true; + } + }, project, ProgressManager.getInstance().getProgressIndicator()); + ProjectIndexableFilesFilter files = new ProjectIndexableFilesFilter(filesSet, myFilesModCount); + project.putUserData(ourProjectFilesSetKey, new SoftReference(files)); + return files; + } + @Nullable private TIntHashSet collectFileIdsContainingAllKeys(final ID indexId, final Collection dataKeys, final GlobalSearchScope filter, - @Nullable final Condition valueChecker) { + @Nullable final Condition valueChecker, + @Nullable final ProjectIndexableFilesFilter projectFilesFilter + ) { final ThrowableConvertor, TIntHashSet, StorageException> convertor = new ThrowableConvertor, TIntHashSet, StorageException>() { @Nullable @@ -981,7 +1031,8 @@ public class FileBasedIndex implements ApplicationComponent { } for (final ValueContainer.IntIterator inputIdsIterator = container.getInputIdsIterator(value); inputIdsIterator.hasNext(); ) { final int id = inputIdsIterator.next(); - if (mainIntersection == null || mainIntersection.contains(id)) { + if ((mainIntersection == null || mainIntersection.contains(id)) && + (projectFilesFilter == null || projectFilesFilter.contains(id))) { copy.add(id); } } @@ -1063,8 +1114,10 @@ public class FileBasedIndex implements ApplicationComponent { final PersistentFS fs = (PersistentFS)ManagingFS.getInstance(); TIntIterator ids = join(locals).iterator(); + ProjectIndexableFilesFilter projectIndexableFilesFilter = projectIndexableFiles(project); while (ids.hasNext()) { int id = ids.next(); + if (projectIndexableFilesFilter != null && !projectIndexableFilesFilter.contains(id)) continue; //VirtualFile file = IndexInfrastructure.findFileById(fs, id); VirtualFile file = IndexInfrastructure.findFileByIdIfCached(fs, id); if (file != null && filter.accept(file)) { @@ -1611,7 +1664,7 @@ public class FileBasedIndex implements ApplicationComponent { @Override public void fileCreated(final VirtualFileEvent event) { - markDirty(event); + markDirty(event, false); } @Override @@ -1621,7 +1674,7 @@ public class FileBasedIndex implements ApplicationComponent { @Override public void fileCopied(final VirtualFileCopyEvent event) { - markDirty(event); + markDirty(event, false); } @Override @@ -1636,7 +1689,7 @@ public class FileBasedIndex implements ApplicationComponent { @Override public void contentsChanged(final VirtualFileEvent event) { - markDirty(event); + markDirty(event, true); } @Override @@ -1657,17 +1710,18 @@ public class FileBasedIndex implements ApplicationComponent { if (event.getPropertyName().equals(VirtualFile.PROP_NAME)) { // indexes may depend on file name if (!event.getFile().isDirectory()) { - markDirty(event); + markDirty(event, false); } } } - private void markDirty(final VirtualFileEvent event) { + private void markDirty(final VirtualFileEvent event, final boolean contentChange) { final VirtualFile eventFile = event.getFile(); cleanProcessedFlag(eventFile); iterateIndexableFiles(eventFile, new Processor() { @Override public boolean process(final VirtualFile file) { + if (!contentChange) ++myFilesModCount; FileContent fileContent = null; // handle 'content-less' indices separately for (ID indexId : myNotRequiringContentIndices) { @@ -2041,6 +2095,7 @@ public class FileBasedIndex implements ApplicationComponent { } public CollectingContentIterator createContentIterator() { + ++myFilesModCount; return new UnindexedFilesFinder(); } diff --git a/platform/util/src/com/intellij/openapi/util/text/StringUtil.java b/platform/util/src/com/intellij/openapi/util/text/StringUtil.java index 09fd71b0eee9..489012d0f1c6 100644 --- a/platform/util/src/com/intellij/openapi/util/text/StringUtil.java +++ b/platform/util/src/com/intellij/openapi/util/text/StringUtil.java @@ -54,6 +54,18 @@ public class StringUtil { } }; + public static List getWordsInStringLongestFirst(String find) { + List words = getWordsIn(find); + // hope long words are rare + Collections.sort(words, new Comparator() { + @Override + public int compare(final String o1, final String o2) { + return o2.length() - o1.length(); + } + }); + return words; + } + @NotNull public static String escapePattern(final @NotNull String text) { return replace(replace(text, "'", "''"), "{", "'{'"); diff --git a/platform/util/src/com/intellij/util/BloomFilterBase.java b/platform/util/src/com/intellij/util/BloomFilterBase.java new file mode 100644 index 000000000000..6779ae273d2b --- /dev/null +++ b/platform/util/src/com/intellij/util/BloomFilterBase.java @@ -0,0 +1,60 @@ +/* + * Copyright 2000-2012 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util; + +public class BloomFilterBase { + private final int myHashFunctionCount; + private final int myBitsCount; + private final long[] myElementsSet; + private static final int BITS_PER_ELEMENT = 6; + + protected BloomFilterBase(int _maxElementCount, double probability) { + int bitsPerElementFactor = (int)Math.ceil(-Math.log(probability) / (Math.log(2) * Math.log(2))); + myHashFunctionCount = (int)Math.ceil(bitsPerElementFactor * Math.log(2)); + + int bitsCount = _maxElementCount * bitsPerElementFactor; + + if ((bitsCount & 1) == 0) ++bitsCount; + while(!isPrime(bitsCount)) bitsCount += 2; + myBitsCount = bitsCount; + myElementsSet = new long[(bitsCount >> BITS_PER_ELEMENT) + 1]; + } + + private static boolean isPrime(int bits) { + if ((bits & 1) == 0 || bits % 3 == 0) return false; + int sqrt = (int)Math.sqrt(bits); + for(int i = 6; i <= sqrt; i += 6) { + if (bits % (i - 1) == 0 || bits % (i + 1) == 0) return false; + } + return true; + } + + protected final void addIt(int prime, int prime2) { + for(int i = 0; i < myHashFunctionCount; ++i) { + int abs = Math.abs(i * prime + prime2 * (myHashFunctionCount - i)) % myBitsCount; + myElementsSet[abs >> BITS_PER_ELEMENT] |= (1L << abs); + } + } + + protected final boolean maybeContains(int prime, int prime2) { + for(int i = 0; i < myHashFunctionCount; ++i) { + int abs = Math.abs(i * prime + prime2 * (myHashFunctionCount - i)) % myBitsCount; + if ((myElementsSet[abs >> BITS_PER_ELEMENT] & (1L << abs)) == 0) return false; + } + + return true; + } +} diff --git a/platform/util/src/com/intellij/util/lang/ClasspathCache.java b/platform/util/src/com/intellij/util/lang/ClasspathCache.java index 658cda05f986..9acbbe87c104 100644 --- a/platform/util/src/com/intellij/util/lang/ClasspathCache.java +++ b/platform/util/src/com/intellij/util/lang/ClasspathCache.java @@ -21,6 +21,7 @@ package com.intellij.util.lang; import com.intellij.openapi.util.text.StringHash; import com.intellij.util.ArrayUtil; +import com.intellij.util.BloomFilterBase; import com.intellij.util.SmartList; import com.intellij.util.containers.HashMap; import gnu.trove.THashMap; @@ -30,7 +31,6 @@ import gnu.trove.TIntObjectHashMap; import org.jetbrains.annotations.Nullable; import sun.misc.Resource; -import java.util.BitSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -44,7 +44,7 @@ public class ClasspathCache { private THashMap> myResources2LoadersTempMap = new THashMap>(); private static final double PROBABILITY = 0.005d; - private BloomFilter myNameFilter; + private Name2LoaderFilter myNameFilter; private boolean myTempMapMode = true; public ClasspathCache() { @@ -232,7 +232,7 @@ public class ClasspathCache { nBits += (int)(nBits * 0.03d); // allow some growth for Idea main loader } - myNameFilter = new BloomFilter(nBits, PROBABILITY); + myNameFilter = new Name2LoaderFilter(nBits, PROBABILITY); for(Map.Entry> e:myResources2LoadersTempMap.entrySet()) { final String name = e.getKey(); @@ -246,50 +246,25 @@ public class ClasspathCache { } } - static class BloomFilter { - private final int myHashFunctionCount; - private final int NBITS; - private final BitSet myResourceMap; + private static class Name2LoaderFilter extends BloomFilterBase { private static final int SEED = 31; - BloomFilter(int nBits, double probability) { - int bitsPerNameFactor = (int)Math.ceil(-Math.log(probability) / (Math.log(2) * Math.log(2))); - myHashFunctionCount = (int)Math.ceil(bitsPerNameFactor * Math.log(2)); - - nBits = nBits * bitsPerNameFactor; - - if ((nBits & 1) == 0) ++nBits; - while(!isPrime(nBits)) nBits += 2; - NBITS = nBits; - myResourceMap = new BitSet(NBITS); - } - - private static boolean isPrime(int bits) { - if ((bits & 1) == 0) return false; - int sqrt = (int)Math.sqrt(bits); - for(int i = 3; i <= sqrt; i+=2) { - if (bits % i == 0) return false; - } - return true; + Name2LoaderFilter(int nBits, double probability) { + super(nBits, probability); } private boolean maybeContains(String name, Loader loader) { int hash = hashFromNameAndLoader(name, loader, StringHash.murmur(name, SEED)); int hash2 = hashFromNameAndLoader(name, loader, hash); - for (int i = 0; i < myHashFunctionCount; ++i) { - if (!myResourceMap.get(Math.abs((hash + i * hash2) % NBITS))) return false; - } - return true; + return maybeContains(hash, hash2); } - public void add(String name, Loader loader) { - int hash1 = hashFromNameAndLoader(name, loader, StringHash.murmur(name, SEED)); - int hash2 = hashFromNameAndLoader(name, loader, hash1); + void add(String name, Loader loader) { + int hash = hashFromNameAndLoader(name, loader, StringHash.murmur(name, SEED)); + int hash2 = hashFromNameAndLoader(name, loader, hash); - for (int i = 0; i < myHashFunctionCount; ++i) { - myResourceMap.set(Math.abs((hash1 + i * hash2) % NBITS)); - } + addIt(hash, hash2); } private int hashFromNameAndLoader(String name, Loader loader, int n) {