[vcs-log] trigram index for commit messages

platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java

@@ -72,7 +72,7 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
 
   private PersistentHashMap<Integer, ByteSequence> myContents;
   private PersistentHashMap<Integer, Integer> myInputsSnapshotMapping;
-  private PersistentHashMap<Integer, Collection<Key>> myInputsIndex;
+  protected PersistentHashMap<Integer, Collection<Key>> myInputsIndex;
   private PersistentHashMap<Integer, String> myIndexingTrace;
 
   private final ReentrantReadWriteLock myLock = new ReentrantReadWriteLock();

platform/vcs-log/impl/src/com/intellij/vcs/log/data/VcsLogData.java

@@ -32,6 +32,8 @@ import com.intellij.util.ThrowableConsumer;
 import com.intellij.util.containers.ConcurrentIntObjectMap;
 import com.intellij.util.containers.ContainerUtil;
 import com.intellij.vcs.log.*;
+import com.intellij.vcs.log.data.index.VcsLogIndex;
+import com.intellij.vcs.log.data.index.VcsLogPersistentIndex;
 import com.intellij.vcs.log.util.StopWatch;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;

platform/vcs-log/impl/src/com/intellij/vcs/log/data/VcsLogRefresherImpl.java

@@ -31,6 +31,7 @@ import com.intellij.util.NotNullFunction;
 import com.intellij.util.containers.ContainerUtil;
 import com.intellij.util.ui.UIUtil;
 import com.intellij.vcs.log.*;
+import com.intellij.vcs.log.data.index.VcsLogIndex;
 import com.intellij.vcs.log.graph.GraphCommit;
 import com.intellij.vcs.log.graph.GraphCommitImpl;
 import com.intellij.vcs.log.graph.PermanentGraph;

platform/vcs-log/impl/src/com/intellij/vcs/log/data/VisiblePackBuilder.java

@@ -25,6 +25,7 @@ import com.intellij.util.containers.ContainerUtil;
 import com.intellij.util.containers.HashSet;
 import com.intellij.util.ui.UIUtil;
 import com.intellij.vcs.log.*;
+import com.intellij.vcs.log.data.index.VcsLogIndex;
 import com.intellij.vcs.log.graph.GraphCommit;
 import com.intellij.vcs.log.graph.PermanentGraph;
 import com.intellij.vcs.log.graph.VisibleGraph;

platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogFullDetailsIndex.java

@@ -0,0 +1,168 @@
+/*
+ * Copyright 2000-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.vcs.log.data.index;
+
+import com.intellij.openapi.Disposable;
+import com.intellij.openapi.util.Disposer;
+import com.intellij.util.PathUtilRt;
+import com.intellij.util.indexing.*;
+import com.intellij.util.io.DataExternalizer;
+import com.intellij.util.io.EnumeratorIntegerDescriptor;
+import com.intellij.util.io.KeyDescriptor;
+import com.intellij.util.io.PersistentHashMap;
+import com.intellij.vcs.log.VcsFullCommitDetails;
+import com.intellij.vcs.log.util.PersistentUtil;
+import gnu.trove.TIntHashSet;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Set;
+
+public class VcsLogFullDetailsIndex implements Disposable {
+  @NotNull private final MyMapReduceIndex myMapReduceIndex;
+  @NotNull private final ID<Integer, Void> myID;
+  private final String myLogId;
+  private final String myName;
+
+  public VcsLogFullDetailsIndex(@NotNull String logId,
+                                @NotNull String name,
+                                int version,
+                                @NotNull DataIndexer<Integer, Void, VcsFullCommitDetails> indexer,
+                                @NotNull Disposable disposableParent)
+    throws IOException {
+    myID = ID.create(name);
+    myName = name;
+    myLogId = logId;
+
+    myMapReduceIndex = new MyMapReduceIndex(indexer, version);
+
+    Disposer.register(disposableParent, this);
+  }
+
+  @NotNull
+  public TIntHashSet getCommitsWithAllKeys(@NotNull Set<Integer> keys) throws StorageException {
+    TIntHashSet result = null;
+
+    for (Integer key : keys) {
+      TIntHashSet newResult = new TIntHashSet();
+      ValueContainer<Void> data = myMapReduceIndex.getData(key);
+
+      ValueContainer.ValueIterator<Void> valueIt = data.getValueIterator();
+      while (valueIt.hasNext()) {
+        valueIt.next();
+        ValueContainer.IntIterator inputIt = valueIt.getInputIdsIterator();
+        while (inputIt.hasNext()) {
+          int id = inputIt.next();
+          if (result == null || result.contains(id)) {
+            newResult.add(id);
+          }
+        }
+      }
+
+      result = newResult;
+    }
+
+    if (result == null) return new TIntHashSet();
+    return result;
+  }
+
+  public void update(int commitId, @NotNull VcsFullCommitDetails details) throws IOException {
+    myMapReduceIndex.update(commitId, details).compute();
+  }
+
+  public void flush() throws StorageException {
+    myMapReduceIndex.flush();
+  }
+
+  public boolean isIndexed(int commit) throws IOException {
+    return myMapReduceIndex.isIndexed(commit);
+  }
+
+  @Override
+  public void dispose() {
+    myMapReduceIndex.dispose();
+  }
+
+  @NotNull
+  public static File getStorageFile(@NotNull String kind, @NotNull String id, int version) {
+    File subdir = new File(PersistentUtil.LOG_CACHE, kind);
+    String safeLogId = PathUtilRt.suggestFileName(id, true, true);
+    return new File(subdir, safeLogId + "." + version);
+  }
+
+  private class MyMapReduceIndex extends MapReduceIndex<Integer, Void, VcsFullCommitDetails> {
+
+    public MyMapReduceIndex(@NotNull DataIndexer<Integer, Void, VcsFullCommitDetails> indexer, int version) throws IOException {
+      super(new MyIndexExtension(indexer, version),
+            new MapIndexStorage<>(getStorageFile("index-" + myName, VcsLogFullDetailsIndex.this.myLogId, version),
+                                  EnumeratorIntegerDescriptor.INSTANCE,
+                                  ScalarIndexExtension.VOID_DATA_EXTERNALIZER, 5000));
+    }
+
+    public boolean isIndexed(int commitId) throws IOException {
+      return myInputsIndex.containsMapping(commitId);
+    }
+
+    @Override
+    protected PersistentHashMap<Integer, Collection<Integer>> createInputsIndex() throws IOException {
+      IndexExtension<Integer, Void, VcsFullCommitDetails> extension = getExtension();
+      return new PersistentHashMap<>(PersistentUtil.getStorageFile("index-inputs-" + myName, myLogId, extension.getVersion()),
+                                     EnumeratorIntegerDescriptor.INSTANCE,
+                                     new InputIndexDataExternalizer<>(extension.getKeyDescriptor(), myID));
+    }
+  }
+
+  private class MyIndexExtension extends IndexExtension<Integer, Void, VcsFullCommitDetails> {
+    @NotNull private final DataIndexer<Integer, Void, VcsFullCommitDetails> myIndexer;
+    private final int myVersion;
+
+    public MyIndexExtension(@NotNull DataIndexer<Integer, Void, VcsFullCommitDetails> indexer, int version) {
+      myIndexer = indexer;
+      myVersion = version;
+    }
+
+    @NotNull
+    @Override
+    public ID<Integer, Void> getName() {
+      return myID;
+    }
+
+    @NotNull
+    @Override
+    public DataIndexer<Integer, Void, VcsFullCommitDetails> getIndexer() {
+      return myIndexer;
+    }
+
+    @NotNull
+    @Override
+    public KeyDescriptor<Integer> getKeyDescriptor() {
+      return EnumeratorIntegerDescriptor.INSTANCE;
+    }
+
+    @NotNull
+    @Override
+    public DataExternalizer<Void> getValueExternalizer() {
+      return ScalarIndexExtension.VOID_DATA_EXTERNALIZER;
+    }
+
+    @Override
+    public int getVersion() {
+      return myVersion;
+    }
+  }
+}

platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogIndex.java

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.intellij.vcs.log.data;
+package com.intellij.vcs.log.data.index;
 
 import com.intellij.vcs.log.VcsLogDetailsFilter;
 import org.jetbrains.annotations.NotNull;

platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogMessagesTrigramIndex.java

@@ -0,0 +1,77 @@
+/*
+ * Copyright 2000-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.vcs.log.data.index;
+
+import com.intellij.openapi.Disposable;
+import com.intellij.openapi.util.text.TrigramBuilder;
+import com.intellij.util.containers.ContainerUtil;
+import com.intellij.util.indexing.DataIndexer;
+import com.intellij.util.indexing.StorageException;
+import com.intellij.vcs.log.VcsFullCommitDetails;
+import gnu.trove.THashMap;
+import gnu.trove.TIntHashSet;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+
+public class VcsLogMessagesTrigramIndex extends VcsLogFullDetailsIndex {
+  private static final int VERSION = 0;
+  private static final String TRIGRAMS = "trigrams";
+
+  public VcsLogMessagesTrigramIndex(@NotNull String logId, @NotNull Disposable disposableParent) throws IOException {
+    super(logId, TRIGRAMS, VERSION, new TrigramMessageIndexer(), disposableParent);
+  }
+
+  @Nullable
+  public TIntHashSet getCommitsForSubstring(@NotNull String string) throws StorageException {
+    MyTrigramProcessor trigramProcessor = new MyTrigramProcessor();
+    TrigramBuilder.processTrigrams(string, trigramProcessor);
+
+    if (trigramProcessor.map.isEmpty()) return null;
+
+    return getCommitsWithAllKeys(trigramProcessor.map.keySet());
+  }
+
+  public static class TrigramMessageIndexer implements DataIndexer<Integer, Void, VcsFullCommitDetails> {
+    @NotNull
+    @Override
+    public Map<Integer, Void> map(@NotNull VcsFullCommitDetails inputData) {
+      MyTrigramProcessor trigramProcessor = new MyTrigramProcessor();
+      TrigramBuilder.processTrigrams(inputData.getFullMessage(), trigramProcessor);
+
+      return trigramProcessor.map;
+    }
+  }
+
+  private static class MyTrigramProcessor extends TrigramBuilder.TrigramProcessor {
+    Map<Integer, Void> map;
+
+    @Override
+    public boolean consumeTrigramsCount(int count) {
+      map = new THashMap<>(count);
+      return true;
+    }
+
+    @Override
+    public boolean execute(int value) {
+      map.put(value, null);
+      return true;
+    }
+  }
+}

platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogPersistentIndex.java

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.intellij.vcs.log.data;
+package com.intellij.vcs.log.data.index;
 
 import com.intellij.openapi.Disposable;
 import com.intellij.openapi.application.ApplicationManager;
@@ -33,13 +33,16 @@ import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.util.Consumer;
 import com.intellij.util.Processor;
 import com.intellij.util.containers.ContainerUtil;
-import com.intellij.util.io.EnumeratorStringDescriptor;
-import com.intellij.util.io.PersistentHashMap;
-import com.intellij.util.io.PersistentMap;
+import com.intellij.util.indexing.StorageException;
+import com.intellij.util.io.*;
 import com.intellij.vcs.log.*;
+import com.intellij.vcs.log.data.CommitDetailsGetter;
+import com.intellij.vcs.log.data.InMemoryMap;
 import com.intellij.vcs.log.util.PersistentUtil;
 import gnu.trove.TIntHashSet;
+import gnu.trove.TIntProcedure;
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
 
 import java.io.IOException;
 import java.util.Collection;
@@ -58,6 +61,7 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
   @NotNull private final VcsLogStorage myHashMap;
 
   @NotNull private final PersistentMap<Integer, String> myMessagesIndex;
+  @Nullable private final VcsLogMessagesTrigramIndex myTrigramIndex;
 
   @NotNull private TIntHashSet myCommitsToIndex = new TIntHashSet();
 
@@ -72,16 +76,29 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
     myDetailsGetter = detailsGetter;
     myFatalErrorsConsumer = fatalErrorsConsumer;
 
-    myMessagesIndex = createMessagesIndex(providers);
+    String logId = PersistentUtil.calcLogId(myProject, providers);
+
+    myMessagesIndex = createIndex(EnumeratorStringDescriptor.INSTANCE, "messages", logId, 0);
+
+    VcsLogMessagesTrigramIndex trigramIndex;
+    try {
+      trigramIndex = new VcsLogMessagesTrigramIndex(logId, disposableParent);
+    }
+    catch (IOException e) {
+      myFatalErrorsConsumer.consume(e);
+      trigramIndex = null;
+    }
+    myTrigramIndex = trigramIndex;
 
     Disposer.register(disposableParent, this);
   }
 
   @NotNull
-  public PersistentMap<Integer, String> createMessagesIndex(@NotNull Map<VirtualFile, VcsLogProvider> providers) {
+  private <V> PersistentMap<Integer, V> createIndex(@NotNull KeyDescriptor<V> descriptor,
+                                                    @NotNull String kind,
+                                                    @NotNull String logId, int version) {
     try {
-      return PersistentUtil
-        .createPersistentHashMap(EnumeratorStringDescriptor.INSTANCE, "messages", PersistentUtil.calcLogId(myProject, providers), 0);
+      return PersistentUtil.createPersistentHashMap(descriptor, kind, logId, version);
     }
     catch (IOException e) {
       myFatalErrorsConsumer.consume(e);
@@ -118,20 +135,24 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
   }
 
   private void storeDetails(@NotNull Collection<VcsFullCommitDetails> details) {
-    for (VcsFullCommitDetails detail : details) {
-      try {
-        myMessagesIndex.put(myHashMap.getCommitIndex(detail.getId(), detail.getRoot()), detail.getFullMessage());
-      }
-      catch (IOException e) {
-        myFatalErrorsConsumer.consume(e);
+    try {
+      for (VcsFullCommitDetails detail : details) {
+        int index = myHashMap.getCommitIndex(detail.getId(), detail.getRoot());
+
+        myMessagesIndex.put(index, detail.getFullMessage());
+        if (myTrigramIndex != null) myTrigramIndex.update(index, detail);
       }
+      myMessagesIndex.force();
+      if (myTrigramIndex != null) myTrigramIndex.flush();
+    } catch (IOException | StorageException e) {
+      myFatalErrorsConsumer.consume(e);
     }
-    myMessagesIndex.force();
   }
 
   public boolean isIndexed(int commit) {
     try {
-      return myMessagesIndex.get(commit) != null;
+      return myMessagesIndex.get(commit) != null &&
+             (myTrigramIndex == null || myTrigramIndex.isIndexed(commit));
     }
     catch (IOException e) {
       myFatalErrorsConsumer.consume(e);
@@ -179,6 +200,38 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
 
   @NotNull
   public TIntHashSet filterMessages(@NotNull String text) {
+    if (myTrigramIndex != null) {
+      try {
+        TIntHashSet commitsForSearch = myTrigramIndex.getCommitsForSubstring(text);
+        if (commitsForSearch != null) {
+          TIntHashSet result = new TIntHashSet();
+          commitsForSearch.forEach(new TIntProcedure() {
+            @Override
+            public boolean execute(int commit) {
+              try {
+                String value = myMessagesIndex.get(commit);
+                if (value != null) {
+                  if (StringUtil.containsIgnoreCase(value, text)) {
+                    result.add(commit);
+                  }
+                }
+              }
+              catch (IOException e) {
+                myFatalErrorsConsumer.consume(e);
+                return false;
+              }
+
+              return true;
+            }
+          });
+          return result;
+        }
+      }
+      catch (StorageException e) {
+        myFatalErrorsConsumer.consume(e);
+      }
+    }
+
     return filter(myMessagesIndex, message -> StringUtil.containsIgnoreCase(message, text));
   }
 

platform/vcs-log/impl/src/com/intellij/vcs/log/util/PersistentUtil.java

@@ -45,7 +45,7 @@ public class PersistentUtil {
   }
 
   @NotNull
-  private static File getStorageFile(@NotNull String storageKind, @NotNull String logId, int version) {
+  public static File getStorageFile(@NotNull String storageKind, @NotNull String logId, int version) {
     File subdir = new File(LOG_CACHE, storageKind);
     String safeLogId = PathUtilRt.suggestFileName(logId, true, true);
     final File mapFile = new File(subdir, safeLogId + "." + version);

platform/vcs-log/impl/test/com/intellij/vcs/log/data/VisiblePackBuilderTest.kt

@@ -22,6 +22,7 @@ import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.util.Consumer
 import com.intellij.util.Function
 import com.intellij.vcs.log.*
+import com.intellij.vcs.log.data.index.VcsLogIndex
 import com.intellij.vcs.log.graph.GraphCommit
 import com.intellij.vcs.log.graph.GraphCommitImpl
 import com.intellij.vcs.log.graph.PermanentGraph