From 8e511fb338a8efb56723ae7a9c1131234e4c4e7d Mon Sep 17 00:00:00 2001 From: Julia Beliaeva Date: Tue, 5 Jul 2016 01:57:40 +0300 Subject: [PATCH] [vcs-log] trigram index for commit messages --- .../util/indexing/MapReduceIndex.java | 2 +- .../com/intellij/vcs/log/data/VcsLogData.java | 2 + .../vcs/log/data/VcsLogRefresherImpl.java | 1 + .../vcs/log/data/VisiblePackBuilder.java | 1 + .../data/index/VcsLogFullDetailsIndex.java | 168 ++++++++++++++++++ .../vcs/log/data/{ => index}/VcsLogIndex.java | 2 +- .../index/VcsLogMessagesTrigramIndex.java | 77 ++++++++ .../{ => index}/VcsLogPersistentIndex.java | 85 +++++++-- .../intellij/vcs/log/util/PersistentUtil.java | 2 +- .../vcs/log/data/VisiblePackBuilderTest.kt | 1 + 10 files changed, 322 insertions(+), 19 deletions(-) create mode 100644 platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogFullDetailsIndex.java rename platform/vcs-log/impl/src/com/intellij/vcs/log/data/{ => index}/VcsLogIndex.java (96%) create mode 100644 platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogMessagesTrigramIndex.java rename platform/vcs-log/impl/src/com/intellij/vcs/log/data/{ => index}/VcsLogPersistentIndex.java (75%) diff --git a/platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java b/platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java index dad726119b41..81b13eef9bba 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java @@ -72,7 +72,7 @@ public class MapReduceIndex implements UpdatableIndex myContents; private PersistentHashMap myInputsSnapshotMapping; - private PersistentHashMap> myInputsIndex; + protected PersistentHashMap> myInputsIndex; private PersistentHashMap myIndexingTrace; private final ReentrantReadWriteLock myLock = new ReentrantReadWriteLock(); diff --git a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/VcsLogData.java b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/VcsLogData.java index f0bcddfd7427..0c8cff65481f 100644 --- a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/VcsLogData.java +++ b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/VcsLogData.java @@ -32,6 +32,8 @@ import com.intellij.util.ThrowableConsumer; import com.intellij.util.containers.ConcurrentIntObjectMap; import com.intellij.util.containers.ContainerUtil; import com.intellij.vcs.log.*; +import com.intellij.vcs.log.data.index.VcsLogIndex; +import com.intellij.vcs.log.data.index.VcsLogPersistentIndex; import com.intellij.vcs.log.util.StopWatch; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; diff --git a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/VcsLogRefresherImpl.java b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/VcsLogRefresherImpl.java index db19e4bf7367..121716ffeeef 100644 --- a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/VcsLogRefresherImpl.java +++ b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/VcsLogRefresherImpl.java @@ -31,6 +31,7 @@ import com.intellij.util.NotNullFunction; import com.intellij.util.containers.ContainerUtil; import com.intellij.util.ui.UIUtil; import com.intellij.vcs.log.*; +import com.intellij.vcs.log.data.index.VcsLogIndex; import com.intellij.vcs.log.graph.GraphCommit; import com.intellij.vcs.log.graph.GraphCommitImpl; import com.intellij.vcs.log.graph.PermanentGraph; diff --git a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/VisiblePackBuilder.java b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/VisiblePackBuilder.java index aa348fff2919..6c86771c1ee2 100644 --- a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/VisiblePackBuilder.java +++ b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/VisiblePackBuilder.java @@ -25,6 +25,7 @@ import com.intellij.util.containers.ContainerUtil; import com.intellij.util.containers.HashSet; import com.intellij.util.ui.UIUtil; import com.intellij.vcs.log.*; +import com.intellij.vcs.log.data.index.VcsLogIndex; import com.intellij.vcs.log.graph.GraphCommit; import com.intellij.vcs.log.graph.PermanentGraph; import com.intellij.vcs.log.graph.VisibleGraph; diff --git a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogFullDetailsIndex.java b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogFullDetailsIndex.java new file mode 100644 index 000000000000..ebfb120880ba --- /dev/null +++ b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogFullDetailsIndex.java @@ -0,0 +1,168 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.vcs.log.data.index; + +import com.intellij.openapi.Disposable; +import com.intellij.openapi.util.Disposer; +import com.intellij.util.PathUtilRt; +import com.intellij.util.indexing.*; +import com.intellij.util.io.DataExternalizer; +import com.intellij.util.io.EnumeratorIntegerDescriptor; +import com.intellij.util.io.KeyDescriptor; +import com.intellij.util.io.PersistentHashMap; +import com.intellij.vcs.log.VcsFullCommitDetails; +import com.intellij.vcs.log.util.PersistentUtil; +import gnu.trove.TIntHashSet; +import org.jetbrains.annotations.NotNull; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.Set; + +public class VcsLogFullDetailsIndex implements Disposable { + @NotNull private final MyMapReduceIndex myMapReduceIndex; + @NotNull private final ID myID; + private final String myLogId; + private final String myName; + + public VcsLogFullDetailsIndex(@NotNull String logId, + @NotNull String name, + int version, + @NotNull DataIndexer indexer, + @NotNull Disposable disposableParent) + throws IOException { + myID = ID.create(name); + myName = name; + myLogId = logId; + + myMapReduceIndex = new MyMapReduceIndex(indexer, version); + + Disposer.register(disposableParent, this); + } + + @NotNull + public TIntHashSet getCommitsWithAllKeys(@NotNull Set keys) throws StorageException { + TIntHashSet result = null; + + for (Integer key : keys) { + TIntHashSet newResult = new TIntHashSet(); + ValueContainer data = myMapReduceIndex.getData(key); + + ValueContainer.ValueIterator valueIt = data.getValueIterator(); + while (valueIt.hasNext()) { + valueIt.next(); + ValueContainer.IntIterator inputIt = valueIt.getInputIdsIterator(); + while (inputIt.hasNext()) { + int id = inputIt.next(); + if (result == null || result.contains(id)) { + newResult.add(id); + } + } + } + + result = newResult; + } + + if (result == null) return new TIntHashSet(); + return result; + } + + public void update(int commitId, @NotNull VcsFullCommitDetails details) throws IOException { + myMapReduceIndex.update(commitId, details).compute(); + } + + public void flush() throws StorageException { + myMapReduceIndex.flush(); + } + + public boolean isIndexed(int commit) throws IOException { + return myMapReduceIndex.isIndexed(commit); + } + + @Override + public void dispose() { + myMapReduceIndex.dispose(); + } + + @NotNull + public static File getStorageFile(@NotNull String kind, @NotNull String id, int version) { + File subdir = new File(PersistentUtil.LOG_CACHE, kind); + String safeLogId = PathUtilRt.suggestFileName(id, true, true); + return new File(subdir, safeLogId + "." + version); + } + + private class MyMapReduceIndex extends MapReduceIndex { + + public MyMapReduceIndex(@NotNull DataIndexer indexer, int version) throws IOException { + super(new MyIndexExtension(indexer, version), + new MapIndexStorage<>(getStorageFile("index-" + myName, VcsLogFullDetailsIndex.this.myLogId, version), + EnumeratorIntegerDescriptor.INSTANCE, + ScalarIndexExtension.VOID_DATA_EXTERNALIZER, 5000)); + } + + public boolean isIndexed(int commitId) throws IOException { + return myInputsIndex.containsMapping(commitId); + } + + @Override + protected PersistentHashMap> createInputsIndex() throws IOException { + IndexExtension extension = getExtension(); + return new PersistentHashMap<>(PersistentUtil.getStorageFile("index-inputs-" + myName, myLogId, extension.getVersion()), + EnumeratorIntegerDescriptor.INSTANCE, + new InputIndexDataExternalizer<>(extension.getKeyDescriptor(), myID)); + } + } + + private class MyIndexExtension extends IndexExtension { + @NotNull private final DataIndexer myIndexer; + private final int myVersion; + + public MyIndexExtension(@NotNull DataIndexer indexer, int version) { + myIndexer = indexer; + myVersion = version; + } + + @NotNull + @Override + public ID getName() { + return myID; + } + + @NotNull + @Override + public DataIndexer getIndexer() { + return myIndexer; + } + + @NotNull + @Override + public KeyDescriptor getKeyDescriptor() { + return EnumeratorIntegerDescriptor.INSTANCE; + } + + @NotNull + @Override + public DataExternalizer getValueExternalizer() { + return ScalarIndexExtension.VOID_DATA_EXTERNALIZER; + } + + @Override + public int getVersion() { + return myVersion; + } + } +} diff --git a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/VcsLogIndex.java b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogIndex.java similarity index 96% rename from platform/vcs-log/impl/src/com/intellij/vcs/log/data/VcsLogIndex.java rename to platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogIndex.java index 15272cb97f1a..c4b7c3454b86 100644 --- a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/VcsLogIndex.java +++ b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogIndex.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intellij.vcs.log.data; +package com.intellij.vcs.log.data.index; import com.intellij.vcs.log.VcsLogDetailsFilter; import org.jetbrains.annotations.NotNull; diff --git a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogMessagesTrigramIndex.java b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogMessagesTrigramIndex.java new file mode 100644 index 000000000000..6c34b88bbf7b --- /dev/null +++ b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogMessagesTrigramIndex.java @@ -0,0 +1,77 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.vcs.log.data.index; + +import com.intellij.openapi.Disposable; +import com.intellij.openapi.util.text.TrigramBuilder; +import com.intellij.util.containers.ContainerUtil; +import com.intellij.util.indexing.DataIndexer; +import com.intellij.util.indexing.StorageException; +import com.intellij.vcs.log.VcsFullCommitDetails; +import gnu.trove.THashMap; +import gnu.trove.TIntHashSet; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.IOException; +import java.util.Map; +import java.util.Set; + +public class VcsLogMessagesTrigramIndex extends VcsLogFullDetailsIndex { + private static final int VERSION = 0; + private static final String TRIGRAMS = "trigrams"; + + public VcsLogMessagesTrigramIndex(@NotNull String logId, @NotNull Disposable disposableParent) throws IOException { + super(logId, TRIGRAMS, VERSION, new TrigramMessageIndexer(), disposableParent); + } + + @Nullable + public TIntHashSet getCommitsForSubstring(@NotNull String string) throws StorageException { + MyTrigramProcessor trigramProcessor = new MyTrigramProcessor(); + TrigramBuilder.processTrigrams(string, trigramProcessor); + + if (trigramProcessor.map.isEmpty()) return null; + + return getCommitsWithAllKeys(trigramProcessor.map.keySet()); + } + + public static class TrigramMessageIndexer implements DataIndexer { + @NotNull + @Override + public Map map(@NotNull VcsFullCommitDetails inputData) { + MyTrigramProcessor trigramProcessor = new MyTrigramProcessor(); + TrigramBuilder.processTrigrams(inputData.getFullMessage(), trigramProcessor); + + return trigramProcessor.map; + } + } + + private static class MyTrigramProcessor extends TrigramBuilder.TrigramProcessor { + Map map; + + @Override + public boolean consumeTrigramsCount(int count) { + map = new THashMap<>(count); + return true; + } + + @Override + public boolean execute(int value) { + map.put(value, null); + return true; + } + } +} diff --git a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/VcsLogPersistentIndex.java b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogPersistentIndex.java similarity index 75% rename from platform/vcs-log/impl/src/com/intellij/vcs/log/data/VcsLogPersistentIndex.java rename to platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogPersistentIndex.java index 10f1be5c6dbe..e4313234f9ba 100644 --- a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/VcsLogPersistentIndex.java +++ b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogPersistentIndex.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intellij.vcs.log.data; +package com.intellij.vcs.log.data.index; import com.intellij.openapi.Disposable; import com.intellij.openapi.application.ApplicationManager; @@ -33,13 +33,16 @@ import com.intellij.openapi.vfs.VirtualFile; import com.intellij.util.Consumer; import com.intellij.util.Processor; import com.intellij.util.containers.ContainerUtil; -import com.intellij.util.io.EnumeratorStringDescriptor; -import com.intellij.util.io.PersistentHashMap; -import com.intellij.util.io.PersistentMap; +import com.intellij.util.indexing.StorageException; +import com.intellij.util.io.*; import com.intellij.vcs.log.*; +import com.intellij.vcs.log.data.CommitDetailsGetter; +import com.intellij.vcs.log.data.InMemoryMap; import com.intellij.vcs.log.util.PersistentUtil; import gnu.trove.TIntHashSet; +import gnu.trove.TIntProcedure; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import java.io.IOException; import java.util.Collection; @@ -58,6 +61,7 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable { @NotNull private final VcsLogStorage myHashMap; @NotNull private final PersistentMap myMessagesIndex; + @Nullable private final VcsLogMessagesTrigramIndex myTrigramIndex; @NotNull private TIntHashSet myCommitsToIndex = new TIntHashSet(); @@ -72,16 +76,29 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable { myDetailsGetter = detailsGetter; myFatalErrorsConsumer = fatalErrorsConsumer; - myMessagesIndex = createMessagesIndex(providers); + String logId = PersistentUtil.calcLogId(myProject, providers); + + myMessagesIndex = createIndex(EnumeratorStringDescriptor.INSTANCE, "messages", logId, 0); + + VcsLogMessagesTrigramIndex trigramIndex; + try { + trigramIndex = new VcsLogMessagesTrigramIndex(logId, disposableParent); + } + catch (IOException e) { + myFatalErrorsConsumer.consume(e); + trigramIndex = null; + } + myTrigramIndex = trigramIndex; Disposer.register(disposableParent, this); } @NotNull - public PersistentMap createMessagesIndex(@NotNull Map providers) { + private PersistentMap createIndex(@NotNull KeyDescriptor descriptor, + @NotNull String kind, + @NotNull String logId, int version) { try { - return PersistentUtil - .createPersistentHashMap(EnumeratorStringDescriptor.INSTANCE, "messages", PersistentUtil.calcLogId(myProject, providers), 0); + return PersistentUtil.createPersistentHashMap(descriptor, kind, logId, version); } catch (IOException e) { myFatalErrorsConsumer.consume(e); @@ -118,20 +135,24 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable { } private void storeDetails(@NotNull Collection details) { - for (VcsFullCommitDetails detail : details) { - try { - myMessagesIndex.put(myHashMap.getCommitIndex(detail.getId(), detail.getRoot()), detail.getFullMessage()); - } - catch (IOException e) { - myFatalErrorsConsumer.consume(e); + try { + for (VcsFullCommitDetails detail : details) { + int index = myHashMap.getCommitIndex(detail.getId(), detail.getRoot()); + + myMessagesIndex.put(index, detail.getFullMessage()); + if (myTrigramIndex != null) myTrigramIndex.update(index, detail); } + myMessagesIndex.force(); + if (myTrigramIndex != null) myTrigramIndex.flush(); + } catch (IOException | StorageException e) { + myFatalErrorsConsumer.consume(e); } - myMessagesIndex.force(); } public boolean isIndexed(int commit) { try { - return myMessagesIndex.get(commit) != null; + return myMessagesIndex.get(commit) != null && + (myTrigramIndex == null || myTrigramIndex.isIndexed(commit)); } catch (IOException e) { myFatalErrorsConsumer.consume(e); @@ -179,6 +200,38 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable { @NotNull public TIntHashSet filterMessages(@NotNull String text) { + if (myTrigramIndex != null) { + try { + TIntHashSet commitsForSearch = myTrigramIndex.getCommitsForSubstring(text); + if (commitsForSearch != null) { + TIntHashSet result = new TIntHashSet(); + commitsForSearch.forEach(new TIntProcedure() { + @Override + public boolean execute(int commit) { + try { + String value = myMessagesIndex.get(commit); + if (value != null) { + if (StringUtil.containsIgnoreCase(value, text)) { + result.add(commit); + } + } + } + catch (IOException e) { + myFatalErrorsConsumer.consume(e); + return false; + } + + return true; + } + }); + return result; + } + } + catch (StorageException e) { + myFatalErrorsConsumer.consume(e); + } + } + return filter(myMessagesIndex, message -> StringUtil.containsIgnoreCase(message, text)); } diff --git a/platform/vcs-log/impl/src/com/intellij/vcs/log/util/PersistentUtil.java b/platform/vcs-log/impl/src/com/intellij/vcs/log/util/PersistentUtil.java index 906366505910..f05072f14b37 100644 --- a/platform/vcs-log/impl/src/com/intellij/vcs/log/util/PersistentUtil.java +++ b/platform/vcs-log/impl/src/com/intellij/vcs/log/util/PersistentUtil.java @@ -45,7 +45,7 @@ public class PersistentUtil { } @NotNull - private static File getStorageFile(@NotNull String storageKind, @NotNull String logId, int version) { + public static File getStorageFile(@NotNull String storageKind, @NotNull String logId, int version) { File subdir = new File(LOG_CACHE, storageKind); String safeLogId = PathUtilRt.suggestFileName(logId, true, true); final File mapFile = new File(subdir, safeLogId + "." + version); diff --git a/platform/vcs-log/impl/test/com/intellij/vcs/log/data/VisiblePackBuilderTest.kt b/platform/vcs-log/impl/test/com/intellij/vcs/log/data/VisiblePackBuilderTest.kt index 92d7d220a7a8..4698b4746473 100644 --- a/platform/vcs-log/impl/test/com/intellij/vcs/log/data/VisiblePackBuilderTest.kt +++ b/platform/vcs-log/impl/test/com/intellij/vcs/log/data/VisiblePackBuilderTest.kt @@ -22,6 +22,7 @@ import com.intellij.openapi.vfs.VirtualFile import com.intellij.util.Consumer import com.intellij.util.Function import com.intellij.vcs.log.* +import com.intellij.vcs.log.data.index.VcsLogIndex import com.intellij.vcs.log.graph.GraphCommit import com.intellij.vcs.log.graph.GraphCommitImpl import com.intellij.vcs.log.graph.PermanentGraph