From d174d83a2dafa09d01aadcc8d6a36c34ea8c3225 Mon Sep 17 00:00:00 2001 From: Julia Beliaeva Date: Sat, 9 Jul 2016 00:01:07 +0300 Subject: [PATCH] [vcs-log] paths index --- .../com/intellij/vcs/log/data/TroveUtil.java | 66 +++++++ .../data/index/VcsLogFullDetailsIndex.java | 10 +- .../vcs/log/data/index/VcsLogPathsIndex.java | 167 ++++++++++++++++++ .../log/data/index/VcsLogPersistentIndex.java | 57 +++--- 4 files changed, 268 insertions(+), 32 deletions(-) create mode 100644 platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogPathsIndex.java diff --git a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/TroveUtil.java b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/TroveUtil.java index d394408cc48d..49f3575faad1 100644 --- a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/TroveUtil.java +++ b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/TroveUtil.java @@ -15,11 +15,16 @@ */ package com.intellij.vcs.log.data; +import com.intellij.util.containers.ContainerUtil; import gnu.trove.TIntArrayList; +import gnu.trove.TIntHashSet; import gnu.trove.TIntObjectHashMap; import gnu.trove.TIntObjectIterator; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import java.util.Arrays; +import java.util.Set; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -47,4 +52,65 @@ public class TroveUtil { if (list.isEmpty()) return IntStream.empty(); return IntStream.range(0, list.size()).map(list::get); } + + @NotNull + public static Set intersect(@NotNull TIntHashSet... sets) { + TIntHashSet result = null; + + Arrays.sort(sets, (set1, set2) -> { + if (set1 == null) return -1; + if (set2 == null) return 1; + return set1.size() - set2.size(); + }); + for (TIntHashSet set : sets) { + result = intersect(result, set); + } + + if (result == null) return ContainerUtil.newHashSet(); + return createJavaSet(result); + } + + @Nullable + private static TIntHashSet intersect(@Nullable TIntHashSet set1, @Nullable TIntHashSet set2) { + if (set1 == null) return set2; + if (set2 == null) return set1; + + TIntHashSet result = new TIntHashSet(); + + if (set1.size() < set2.size()) { + set1.forEach(value -> { + if (set2.contains(value)) { + result.add(value); + } + return true; + }); + } + else { + set2.forEach(value -> { + if (set1.contains(value)) { + result.add(value); + } + return true; + }); + } + + return result; + } + + @NotNull + private static Set createJavaSet(@NotNull TIntHashSet set) { + Set result = ContainerUtil.newHashSet(set.size()); + set.forEach(value -> { + result.add(value); + return true; + }); + return result; + } + + public static void addAll(@NotNull TIntHashSet where, @NotNull TIntHashSet what) { + what.forEach(value -> { + where.add(value); + return true; + }); + } } diff --git a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogFullDetailsIndex.java b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogFullDetailsIndex.java index b7dff148e110..4de9b86ca4fb 100644 --- a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogFullDetailsIndex.java +++ b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogFullDetailsIndex.java @@ -38,8 +38,9 @@ import java.util.Set; public class VcsLogFullDetailsIndex implements Disposable { @NotNull private final MyMapReduceIndex myMapReduceIndex; @NotNull private final ID myID; - private final String myLogId; - private final String myName; + @NotNull private final String myLogId; + @NotNull private final String myName; + @NotNull protected final DataIndexer myIndexer; public VcsLogFullDetailsIndex(@NotNull String logId, @NotNull String name, @@ -50,8 +51,9 @@ public class VcsLogFullDetailsIndex implements Disposable { myID = ID.create(name); myName = name; myLogId = logId; + myIndexer = indexer; - myMapReduceIndex = new MyMapReduceIndex(indexer, version); + myMapReduceIndex = new MyMapReduceIndex(myIndexer, version); Disposer.register(disposableParent, this); } @@ -68,7 +70,7 @@ public class VcsLogFullDetailsIndex implements Disposable { } @NotNull - public TIntHashSet getCommitsWithAllKeys(@NotNull Set keys) throws StorageException { + public TIntHashSet getCommitsWithAllKeys(@NotNull Collection keys) throws StorageException { Ref result = new Ref<>(null); for (Integer key : keys) { diff --git a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogPathsIndex.java b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogPathsIndex.java new file mode 100644 index 000000000000..0d0f903d5c61 --- /dev/null +++ b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogPathsIndex.java @@ -0,0 +1,167 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.vcs.log.data.index; + +import com.intellij.openapi.Disposable; +import com.intellij.openapi.diagnostic.Logger; +import com.intellij.openapi.vcs.FilePath; +import com.intellij.openapi.vcs.changes.Change; +import com.intellij.openapi.vcs.changes.ContentRevision; +import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.util.containers.ContainerUtil; +import com.intellij.util.indexing.DataIndexer; +import com.intellij.util.indexing.StorageException; +import com.intellij.util.io.*; +import com.intellij.vcs.log.VcsFullCommitDetails; +import com.intellij.vcs.log.util.PersistentUtil; +import gnu.trove.THashMap; +import gnu.trove.TIntHashSet; +import org.jetbrains.annotations.NotNull; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class VcsLogPathsIndex extends VcsLogFullDetailsIndex { + private static final Logger LOG = Logger.getInstance(VcsLogPathsIndex.class); + private static final String NAME = "paths"; + private static final int VERSION = 0; + private static final int VALUE = 239; + + @NotNull private final PersistentHashMap myEmptyCommits; + @NotNull private final PathsIndexer myPathsIndexer; + + public VcsLogPathsIndex(@NotNull String logId, + @NotNull Set roots, + @NotNull Disposable disposableParent) throws IOException { + super(logId, NAME, VERSION, new PathsIndexer( + PersistentUtil.createPersistentEnumerator(EnumeratorStringDescriptor.INSTANCE, "index-paths-ids", logId, VERSION), roots), + disposableParent); + + myEmptyCommits = PersistentUtil.createPersistentHashMap(EnumeratorIntegerDescriptor.INSTANCE, "index-no-" + NAME, logId, VERSION); + myPathsIndexer = (PathsIndexer)myIndexer; + } + + @Override + protected void onNotIndexableCommit(int commit) throws StorageException { + try { + myEmptyCommits.put(commit, VALUE); + } + catch (IOException e) { + throw new StorageException(e); + } + } + + @Override + public boolean isIndexed(int commit) throws IOException { + return super.isIndexed(commit) || myEmptyCommits.containsMapping(commit); + } + + @Override + public void flush() throws StorageException { + super.flush(); + myEmptyCommits.force(); + myPathsIndexer.getPathsEnumerator().force(); + } + + public TIntHashSet getCommitsForPaths(@NotNull Collection paths) throws IOException, StorageException { + Set result = ContainerUtil.newHashSet(); + for (FilePath path : paths) { + result.add(myPathsIndexer.getPathId(path)); + } + return getCommitsWithAnyKey(result); + } + + @Override + public void dispose() { + super.dispose(); + try { + myEmptyCommits.close(); + } + catch (IOException e) { + LOG.warn(e); + } + try { + myPathsIndexer.getPathsEnumerator().close(); + } + catch (IOException e) { + LOG.warn(e); + } + } + + private static class PathsIndexer implements DataIndexer { + @NotNull private final PersistentEnumeratorBase myPathsEnumerator; + @NotNull private final Set myRoots; + + private PathsIndexer(@NotNull PersistentEnumeratorBase enumerator, @NotNull Set roots) { + myPathsEnumerator = enumerator; + myRoots = roots; + } + + @NotNull + @Override + public Map map(@NotNull VcsFullCommitDetails inputData) { + Map result = new THashMap<>(); + + Collection changes = inputData.getChanges(); + for (Change change : changes) { + ContentRevision beforeRevision = change.getBeforeRevision(); + if (beforeRevision != null) { + for (Integer pathId : getPathIds(beforeRevision.getFile())) { + result.put(pathId, null); + } + } + + ContentRevision afterRevision = change.getAfterRevision(); + if (afterRevision != null) { + for (Integer pathId : getPathIds(afterRevision.getFile())) { + result.put(pathId, null); + } + } + } + return result; + } + + @NotNull + private Collection getPathIds(@NotNull FilePath path) { + List result = ContainerUtil.newArrayList(); + try { + while (path != null) { + result.add(getPathId(path)); + VirtualFile file = path.getVirtualFile(); + if (file != null && myRoots.contains(file)) break; + + path = path.getParentPath(); + } + } + catch (IOException e) { + e.printStackTrace(); // TODO ? + } + return result; + } + + public int getPathId(@NotNull FilePath path) throws IOException { + return myPathsEnumerator.enumerate(path.getPath()); + } + + @NotNull + public PersistentEnumeratorBase getPathsEnumerator() { + return myPathsEnumerator; + } + } +} diff --git a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogPersistentIndex.java b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogPersistentIndex.java index eeace0c7dad8..d28209177b8a 100644 --- a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogPersistentIndex.java +++ b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogPersistentIndex.java @@ -30,6 +30,7 @@ import com.intellij.openapi.util.Disposer; import com.intellij.openapi.util.Ref; import com.intellij.openapi.util.ThrowableComputable; import com.intellij.openapi.util.text.StringUtil; +import com.intellij.openapi.vcs.FilePath; import com.intellij.openapi.vcs.VcsException; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.util.Consumer; @@ -73,6 +74,7 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable { @NotNull private final PersistentMap myMessagesIndex; @Nullable private final VcsLogMessagesTrigramIndex myTrigramIndex; @Nullable private final VcsLogUserIndex myUserIndex; + @Nullable private final VcsLogPathsIndex myPathsIndex; @NotNull private TIntHashSet myCommitsToIndex = new TIntHashSet(); @@ -95,6 +97,7 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable { myMessagesIndex = createMap(EnumeratorStringDescriptor.INSTANCE, "messages", logId, 0); myTrigramIndex = createIndex(() -> new VcsLogMessagesTrigramIndex(logId, this)); myUserIndex = createIndex(() -> new VcsLogUserIndex(logId, myUserRegistry, this)); + myPathsIndex = createIndex(() -> new VcsLogPathsIndex(logId, myRoots, this)); Disposer.register(disposableParent, this); } @@ -159,10 +162,12 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable { myMessagesIndex.put(index, detail.getFullMessage()); if (myTrigramIndex != null) myTrigramIndex.update(index, detail); if (myUserIndex != null) myUserIndex.update(index, detail); + if (myPathsIndex != null) myPathsIndex.update(index, detail); } myMessagesIndex.force(); if (myTrigramIndex != null) myTrigramIndex.flush(); if (myUserIndex != null) myUserIndex.flush(); + if (myPathsIndex != null) myPathsIndex.flush(); } catch (IOException | StorageException e) { myFatalErrorsConsumer.consume(e); @@ -173,6 +178,7 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable { try { return myMessagesIndex.get(commit) != null && (myUserIndex == null || myUserIndex.isIndexed(commit)) && + (myPathsIndex == null || myPathsIndex.isIndexed(commit)) && (myTrigramIndex == null || myTrigramIndex.isIndexed(commit)); } catch (IOException e) { @@ -232,6 +238,19 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable { return new TIntHashSet(); } + @NotNull + private TIntHashSet filterPaths(@NotNull Collection paths) { + if (myPathsIndex != null) { + try { + return myPathsIndex.getCommitsForPaths(paths); + } + catch (IOException | StorageException e) { + myFatalErrorsConsumer.consume(e); + } + } + return new TIntHashSet(); + } + @NotNull public TIntHashSet filterMessages(@NotNull String text) { if (myTrigramIndex != null) { @@ -274,7 +293,8 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable { if (filters.isEmpty()) return false; for (VcsLogDetailsFilter filter : filters) { if (!((filter instanceof VcsLogTextFilter && myTrigramIndex != null) || - (filter instanceof VcsLogUserFilterImpl && myUserIndex != null))) { + (filter instanceof VcsLogUserFilterImpl && myUserIndex != null) || + (filter instanceof VcsLogStructureFilter && myPathsIndex != null))) { return false; } } @@ -284,20 +304,16 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable { @Override @NotNull public Set filter(@NotNull List detailsFilters) { - Set result = ContainerUtil.newHashSet(); - VcsLogTextFilter textFilter = ContainerUtil.findInstance(detailsFilters, VcsLogTextFilter.class); VcsLogUserFilterImpl userFilter = ContainerUtil.findInstance(detailsFilters, VcsLogUserFilterImpl.class); + VcsLogStructureFilter pathFilter = ContainerUtil.findInstance(detailsFilters, VcsLogStructureFilter.class); - TIntHashSet filteredByMessage; + TIntHashSet filteredByMessage = null; if (textFilter != null) { filteredByMessage = filterMessages(textFilter.getText()); } - else { - filteredByMessage = null; - } - TIntHashSet filteredByUser; + TIntHashSet filteredByUser = null; if (userFilter != null) { Set users = ContainerUtil.newHashSet(); for (VirtualFile root : myRoots) { @@ -306,28 +322,13 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable { filteredByUser = filterUsers(users); } - else { - filteredByUser = null; + + TIntHashSet filteredByPath = null; + if (pathFilter != null) { + filteredByPath = filterPaths(pathFilter.getFiles()); } - if (filteredByMessage == null) { - if (filteredByUser != null) { - filteredByUser.forEach(value -> { - result.add(value); - return true; - }); - } - } - else { - filteredByMessage.forEach(value -> { - if (filteredByUser != null && filteredByUser.contains(value)) { - result.add(value); - } - return true; - }); - } - - return result; + return TroveUtil.intersect(filteredByMessage, filteredByPath, filteredByUser); } @Override