From 8f3acdeb936a18bfc11bd217bb4918f2305c6a3d Mon Sep 17 00:00:00 2001 From: "Maxim.Mossienko" Date: Fri, 25 Apr 2014 23:12:29 +0200 Subject: [PATCH] use isjavaidentifierpart + enable trigramindex for tests because test appeared --- .../intellij/find/impl/FindInProjectTask.java | 25 ++- .../intellij/find/ngrams/TrigramIndex.java | 35 ++-- .../openapi/util/text/TrigramBuilderTest.java | 157 ++++-------------- .../openapi/util/text/TrigramBuilder.java | 110 ++++++++++-- 4 files changed, 166 insertions(+), 161 deletions(-) diff --git a/platform/lang-impl/src/com/intellij/find/impl/FindInProjectTask.java b/platform/lang-impl/src/com/intellij/find/impl/FindInProjectTask.java index 90d2cc6385bd..7edb9a9e9292 100644 --- a/platform/lang-impl/src/com/intellij/find/impl/FindInProjectTask.java +++ b/platform/lang-impl/src/com/intellij/find/impl/FindInProjectTask.java @@ -57,8 +57,6 @@ import com.intellij.util.Processor; import com.intellij.util.containers.ContainerUtil; import com.intellij.util.indexing.FileBasedIndex; import com.intellij.util.indexing.FileBasedIndexImpl; -import gnu.trove.TIntHashSet; -import gnu.trove.TIntIterator; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -363,7 +361,14 @@ class FindInProjectTask { String text = myFindModel.getStringToFind(); if (StringUtil.isEmptyOrSpaces(text)) return false; - if (TrigramIndex.ENABLED) return !TrigramBuilder.buildTrigram(text).isEmpty(); + if (TrigramIndex.ENABLED) { + return !TrigramBuilder.processTrigrams(text, new TrigramBuilder.TrigramProcessor() { + @Override + public boolean execute(int value) { + return false; + } + }); + } // $ is used to separate words when indexing plain-text files but not when indexing // Java identifiers, so we can't consistently break a string containing $ characters into words @@ -394,12 +399,14 @@ class FindInProjectTask { final Set resultFiles = new LinkedHashSet(); if (TrigramIndex.ENABLED) { - Set keys = ContainerUtil.newTroveSet(); - TIntHashSet trigrams = TrigramBuilder.buildTrigram(stringToFind); - TIntIterator it = trigrams.iterator(); - while (it.hasNext()) { - keys.add(it.next()); - } + final Set keys = ContainerUtil.newTroveSet(); + TrigramBuilder.processTrigrams(stringToFind, new TrigramBuilder.TrigramProcessor() { + @Override + public boolean execute(int value) { + keys.add(value); + return true; + } + }); if (!keys.isEmpty()) { List hits = new ArrayList(); diff --git a/platform/lang-impl/src/com/intellij/find/ngrams/TrigramIndex.java b/platform/lang-impl/src/com/intellij/find/ngrams/TrigramIndex.java index cd9d88a618fb..72f5f947d0e7 100644 --- a/platform/lang-impl/src/com/intellij/find/ngrams/TrigramIndex.java +++ b/platform/lang-impl/src/com/intellij/find/ngrams/TrigramIndex.java @@ -19,7 +19,6 @@ */ package com.intellij.find.ngrams; -import com.intellij.openapi.application.ApplicationManager; import com.intellij.openapi.util.ThreadLocalCachedIntArray; import com.intellij.openapi.util.text.TrigramBuilder; import com.intellij.openapi.vfs.VirtualFile; @@ -31,8 +30,6 @@ import com.intellij.util.io.DataInputOutputUtil; import com.intellij.util.io.EnumeratorIntegerDescriptor; import com.intellij.util.io.KeyDescriptor; import gnu.trove.THashMap; -import gnu.trove.TIntHashSet; -import gnu.trove.TIntProcedure; import org.jetbrains.annotations.NotNull; import java.io.DataInput; @@ -44,8 +41,7 @@ import java.util.Collection; import java.util.Map; public class TrigramIndex extends ScalarIndexExtension implements CustomInputsIndexFileBasedIndexExtension { - public static final boolean ENABLED = SystemProperties.getBooleanProperty("idea.internal.trigramindex.enabled", - !ApplicationManager.getApplication().isUnitTestMode()); + public static final boolean ENABLED = SystemProperties.getBooleanProperty("idea.internal.trigramindex.enabled", true); public static final ID INDEX_ID = ID.create("Trigram.Index"); @@ -75,16 +71,10 @@ public class TrigramIndex extends ScalarIndexExtension implements Custo @Override @NotNull public Map map(@NotNull FileContent inputData) { - TIntHashSet built = TrigramBuilder.buildTrigram(inputData.getContentAsText()); - final Map result = new THashMap(built.size()); - built.forEach(new TIntProcedure() { - @Override - public boolean execute(int value) { - result.put(value, null); - return true; - } - }); - return result; + MyTrigramProcessor trigramProcessor = new MyTrigramProcessor(); + TrigramBuilder.processTrigrams(inputData.getContentAsText(), trigramProcessor); + + return trigramProcessor.map; } }; } @@ -160,4 +150,19 @@ public class TrigramIndex extends ScalarIndexExtension implements Custo } }; } + + private static class MyTrigramProcessor extends TrigramBuilder.TrigramProcessor { + Map map; + @Override + public boolean consumeTrigramsCount(int count) { + map = new THashMap(count); + return true; + } + + @Override + public boolean execute(int value) { + map.put(value, null); + return true; + } + } } diff --git a/platform/platform-tests/testSrc/com/intellij/openapi/util/text/TrigramBuilderTest.java b/platform/platform-tests/testSrc/com/intellij/openapi/util/text/TrigramBuilderTest.java index 5e5b72d7aaac..26b0920c473a 100644 --- a/platform/platform-tests/testSrc/com/intellij/openapi/util/text/TrigramBuilderTest.java +++ b/platform/platform-tests/testSrc/com/intellij/openapi/util/text/TrigramBuilderTest.java @@ -15,141 +15,44 @@ */ package com.intellij.openapi.util.text; -import com.intellij.openapi.util.io.FileUtil; -import com.intellij.openapi.util.io.FileUtilRt; -import gnu.trove.THashSet; -import gnu.trove.TIntHashSet; -import gnu.trove.TIntObjectHashMap; -import gnu.trove.TIntObjectProcedure; +import com.intellij.openapi.util.Ref; +import gnu.trove.TIntArrayList; +import junit.framework.TestCase; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.util.*; +public class TrigramBuilderTest extends TestCase { + public void testBuilder() { + final Ref trigramCountRef = new Ref(); + final TIntArrayList list = new TIntArrayList(); -public class TrigramBuilderTest { - public static void main(String[] args) throws IOException { - File root = new File(args[0]); + TrigramBuilder.processTrigrams("String$CharData", new TrigramBuilder.TrigramProcessor() { + @Override + public boolean execute(int value) { + list.add(value); + return true; + } - Stats stats = new Stats(); - walk(root, stats); - - System.out.println("Scanned " + stats.files + " files, total of " + stats.lines + " lines in " + (stats.time / 1000000) + " ms."); - System.out.println("Size:" + stats.bytes); - System.out.println("Total trigrams: " + stats.allTrigrams.size()); - System.out.println("Max per file: " + stats.maxtrigrams); - - System.out.println("Sample query 1: " + lookup(stats.filesMap, "trigram")); - System.out.println("Sample query 2: " + lookup(stats.filesMap, "some text that most probably doesn't exist")); - System.out.println("Sample query 3: " + lookup(stats.filesMap, "ProfilingUtil.captureCPUSnapshot();")); - - System.out.println("Stop words:"); - - listWithBarier(stats, stats.files * 2 / 4); - listWithBarier(stats, stats.files * 3 / 4); - listWithBarier(stats, stats.files * 4 / 5); - } - - private static void listWithBarier(Stats stats, final int barrier) { - final int[] stopCount = {0}; - stats.filesMap.forEachEntry(new TIntObjectProcedure>() { - public boolean execute(int a, List b) { - if (b.size() > barrier) { - System.out.println(a); - stopCount[0]++; - } + @Override + public boolean consumeTrigramsCount(int count) { + trigramCountRef.set(count); return true; } }); - System.out.println("Total of " + stopCount[0]); + list.sort(); + Integer trigramCount = trigramCountRef.get(); + assertNotNull(trigramCount); + + int expectedTrigramCount = 13; + assertEquals(expectedTrigramCount, (int)trigramCount); + assertEquals(expectedTrigramCount, list.size()); + + int[] expected = {buildTrigram("$Ch"), buildTrigram("arD"), buildTrigram("ata"), 6514785, 6578548, 6759523, 6840690, 6909543, 7235364, 7496801, 7498094, 7566450, 7631465, }; + for(int i = 0; i < expectedTrigramCount; ++i) assertEquals(expected[i], list.getQuick(i)); } - private static void walk(File root, Stats stats) throws IOException { - String name = root.getName(); - if (root.isDirectory()) { - if (name.startsWith(".") || name.equals("out")) return; - System.out.println("Lexing in " + root.getPath()); - for (File file : root.listFiles()) { - walk(file, stats); - } - } - else { - String ext = FileUtilRt.getExtension(name); - if (!allowedExtension.contains(ext)) return; - if (root.length() > 100 * 1024) return; - - stats.extensions.add(ext); - lex(root, stats); - } + private static int buildTrigram(String s) { + int tc1 = StringUtil.toLowerCase(s.charAt(0)); + int tc2 = (tc1 << 8) + StringUtil.toLowerCase(s.charAt(1)); + return (tc2 << 8) + StringUtil.toLowerCase(s.charAt(2)); } - - private static void lex(File root, Stats stats) throws IOException { - stats.files++; - BufferedReader reader = new BufferedReader(new FileReader(root)); - String s; - StringBuilder buf = new StringBuilder(); - while ((s = reader.readLine()) != null) { - stats.lines++; - buf.append(s).append("\n"); - } - - stats.bytes += buf.length(); - - long start = System.nanoTime(); - TIntHashSet localTrigrams = lexText(buf); - stats.time += System.nanoTime() - start; - stats.maxtrigrams = Math.max(stats.maxtrigrams, localTrigrams.size()); - int[] graphs = localTrigrams.toArray(); - stats.allTrigrams.addAll(graphs); - for (int graph : graphs) { - List list = stats.filesMap.get(graph); - if (list == null) { - list = new ArrayList(); - stats.filesMap.put(graph, list); - } - list.add(root); - } - } - - private static final Set allowedExtension = new THashSet( - Arrays.asList("iml", "xml", "java", "html", "bat", "policy", "properties", "sh", "dtd", "ipr", "txt", "plist", "form", "xsl", "css", - "jsp", "jspx", "xhtml", "tld", "htm", "tag", "jspf", "js", "ft", "xsd", "xls", "rb", "php", "ftl", "c", "y", "erb", "rjs", - "rhtml", "sql", "cfml", "groovy", "text", "gsp", "h", "cc", "cpp", "wsdl"), FileUtil.PATH_HASHING_STRATEGY); - - private static Collection lookup(TIntObjectHashMap> trigramsDatabase, String query) { - final Set result = new HashSet(); - int[] graphs = TrigramBuilder.buildTrigram(query).toArray(); - boolean first = true; - - for (int graph : graphs) { - if (first) { - result.addAll(trigramsDatabase.get(graph)); - first = false; - } - else { - result.retainAll(trigramsDatabase.get(graph)); - } - } - - return result; - } - - - private static TIntHashSet lexText(StringBuilder buf) { - return TrigramBuilder.buildTrigram(buf); - } - - private static class Stats { - public int files; - public int lines; - public int maxtrigrams; - public long time; - public long bytes; - public final TIntHashSet allTrigrams = new TIntHashSet(); - public final TIntObjectHashMap> filesMap = new TIntObjectHashMap>(); - public final Set extensions = new HashSet(); - } - } diff --git a/platform/util/src/com/intellij/openapi/util/text/TrigramBuilder.java b/platform/util/src/com/intellij/openapi/util/text/TrigramBuilder.java index b43a3aaabe2b..1d39ccabe5a5 100644 --- a/platform/util/src/com/intellij/openapi/util/text/TrigramBuilder.java +++ b/platform/util/src/com/intellij/openapi/util/text/TrigramBuilder.java @@ -20,15 +20,14 @@ package com.intellij.openapi.util.text; import com.intellij.util.text.CharArrayUtil; -import gnu.trove.TIntHashSet; +import gnu.trove.TIntProcedure; public class TrigramBuilder { private TrigramBuilder() { } - public static TIntHashSet buildTrigram(CharSequence text) { - TIntHashSet result = new TIntHashSet(); - + public static boolean processTrigrams(CharSequence text, TrigramProcessor consumer) { + final AddonlyIntSet set = new AddonlyIntSet(); int index = 0; final char[] fileTextArray = CharArrayUtil.fromSequenceWithoutCopying(text); @@ -43,32 +42,123 @@ public class TrigramBuilder { } index++; } - int index1 = index; + int identifierStart = index; while (true) { index++; if (index == text.length()) break; final char c = fileTextArray != null ? fileTextArray[index]:text.charAt(index); if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')) continue; - if (!Character.isUnicodeIdentifierPart(c)) break; + if (!Character.isJavaIdentifierPart(c)) break; } - if (index - index1 > 100) continue; // Strange limit but we should have some! int tc1 = 0; int tc2 = 0; int tc3; - for (int i = index1, iters = 0; i < index; ++i, ++iters) { + for (int i = identifierStart, iters = 0; i < index; ++i, ++iters) { char c = StringUtil.toLowerCase(fileTextArray != null ? fileTextArray[i]:text.charAt(i)); tc3 = (tc2 << 8) + c; tc2 = (tc1 << 8) + c; tc1 = c; if (iters >= 2) { - result.add(tc3); + set.add(tc3); } } } - return result; + return consumer.consumeTrigramsCount(set.size()) && set.forEach(consumer); + } + + public static abstract class TrigramProcessor implements TIntProcedure { + public boolean consumeTrigramsCount(int count) { return true; } + } +} + +class AddonlyIntSet { + //private static final int MAGIC = 0x9E3779B9; + private int size; + private int[] data; + private int shift; + private int mask; + private boolean hasZeroKey; + + public AddonlyIntSet() { + this(21); + } + + public AddonlyIntSet(int expectedSize) { + int powerOfTwo = Integer.highestOneBit((3 * expectedSize) / 2) << 1; + shift = Integer.numberOfLeadingZeros(powerOfTwo) + 1; + mask = powerOfTwo - 1; + data = new int[powerOfTwo]; + } + + public int size() { + return size; + } + + private int hash(int h, int[] a) { + h ^= (h >>> 20) ^ (h >>> 12); + return (h ^ (h >>> 7) ^ (h >>> 4)) & mask; + //int idx = (id * MAGIC) >>> shift; + //if (idx >= a.length) { + // idx %= a.length; + //} + //return idx; + } + + public void add(int key) { + if (key == 0) { + if (!hasZeroKey) ++size; + hasZeroKey = true; + return; + } + if (size >= (2 * data.length) / 3) rehash(); + if (doPut(data, key)) size++; + } + + private boolean doPut(int[] a, int o) { + int index = hash(o, a); + int obj; + while ((obj = a[index]) != 0) { + if (obj == o) break; + if (index == 0) index = a.length; + index--; + } + a[index] = o; + return obj == 0; + } + + private void rehash() { + --shift; + int[] b = new int[data.length << 1]; + mask = b.length - 1; + for (int i = data.length; --i >= 0;) { + int ns = data[i]; + if (ns != 0) doPut(b, ns); + } + data = b; + } + + public boolean contains(int key) { + if (key == 0) return hasZeroKey; + int index = hash(key, data); + int v; + while ((v = data[index]) != 0) { + if (v == key) return true; + if (index == 0) index = data.length; + index--; + } + return false; + } + + public boolean forEach(TIntProcedure consumer) { + if (hasZeroKey && !consumer.execute(0)) return false; + for(int o:data) { + if (o == 0) continue; + if(!consumer.execute(o)) return false; + } + return true; } }