[file-history] refine file history by removing commits that change different files having the same name

How it is done:
1. Collect data from index, containing files for commits and renames for commits.
2. Find the last revision under HEAD that changes the file.
3. Starting from this revision walk the visible graph, keeping track of file names, and remove some commits that do not fit.
This commit is contained in:
Julia Beliaeva
2017-01-31 23:54:52 +03:00
parent c5360d35e3
commit bb53f896a8
9 changed files with 412 additions and 27 deletions
@@ -31,6 +31,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import java.util.function.IntPredicate;
public class ReachableNodes {
@NotNull private final LiteLinearGraph myGraph;
@@ -72,6 +73,13 @@ public class ReachableNodes {
}
private void walk(@NotNull Collection<Integer> startNodes, boolean goDown, @NotNull Consumer<Integer> consumer) {
walk(startNodes, goDown, node -> {
consumer.consume(node);
return true;
});
}
public void walk(@NotNull Collection<Integer> startNodes, boolean goDown, @NotNull IntPredicate consumer) {
synchronized (myTempFlags) {
myTempFlags.setAll(false);
@@ -79,16 +87,16 @@ public class ReachableNodes {
if (start < 0) continue;
if (myTempFlags.get(start)) continue;
myTempFlags.set(start, true);
consumer.consume(start);
if (!consumer.test(start)) return;
DfsUtil.walk(start, currentNode -> {
for (int downNode : myGraph.getNodes(currentNode, goDown ? LiteLinearGraph.NodeFilter.DOWN : LiteLinearGraph.NodeFilter.UP)) {
if (!myTempFlags.get(downNode)) {
myTempFlags.set(downNode, true);
consumer.consume(downNode);
return downNode;
DfsUtil.walk(start, currentNode-> {
for (int downNode : myGraph.getNodes(currentNode, goDown ? LiteLinearGraph.NodeFilter.DOWN : LiteLinearGraph.NodeFilter.UP)) {
if (!myTempFlags.get(downNode)) {
myTempFlags.set(downNode, true);
if (!consumer.test(downNode)) return DfsUtil.NextNode.EXIT;
return downNode;
}
}
}
return DfsUtil.NextNode.NODE_NOT_FOUND;
});
@@ -19,6 +19,7 @@ import com.intellij.vcs.log.graph.*;
import com.intellij.vcs.log.graph.actions.ActionController;
import com.intellij.vcs.log.graph.actions.GraphAction;
import com.intellij.vcs.log.graph.actions.GraphAnswer;
import com.intellij.vcs.log.graph.api.LinearGraph;
import com.intellij.vcs.log.graph.api.elements.GraphEdge;
import com.intellij.vcs.log.graph.api.elements.GraphEdgeType;
import com.intellij.vcs.log.graph.api.elements.GraphElement;
@@ -97,6 +98,10 @@ public class VisibleGraphImpl<CommitId> implements VisibleGraph<CommitId> {
return myPrintElementGenerator.getRecommendedWidth();
}
public LinearGraph getLinearGraph() {
return myGraphController.getCompiledGraph();
}
private class ActionControllerImpl implements ActionController<CommitId> {
@Nullable
@@ -22,6 +22,7 @@ import org.jetbrains.annotations.NotNull;
public class DfsUtil {
public interface NextNode {
int NODE_NOT_FOUND = -1;
int EXIT = -10;
int fun(int currentNode);
}
@@ -32,6 +33,7 @@ public class DfsUtil {
while (!stack.empty()) {
int nextNode = nextNodeFun.fun(stack.peek());
if (nextNode == NextNode.EXIT) return;
if (nextNode != NextNode.NODE_NOT_FOUND) {
stack.push(nextNode);
}
@@ -77,7 +77,7 @@ public class CompressedRefs {
}
@NotNull
Stream<VcsRef> streamBranches() {
public Stream<VcsRef> streamBranches() {
return TroveUtil.streamValues(myBranches).flatMap(Collection::stream);
}
@@ -16,11 +16,15 @@
package com.intellij.vcs.log.data.index;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.util.Couple;
import com.intellij.openapi.util.UnorderedPair;
import com.intellij.openapi.vcs.FilePath;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.indexing.StorageException;
import com.intellij.vcs.log.impl.FatalErrorHandler;
import com.intellij.vcsUtil.VcsUtil;
import gnu.trove.TIntObjectHashMap;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -69,4 +73,103 @@ public class IndexDataGetter {
return Collections.emptySet();
}
@NotNull
public FileNamesData buildFileNamesData(@NotNull FilePath path) {
FileNamesData result = new FileNamesData();
VirtualFile root = VcsUtil.getVcsRootFor(myProject, path);
if (myRoots.contains(root)) {
try {
myIndexStorage.paths.iterateCommits(Collections.singleton(path), (paths, commit) -> result.add(commit, paths));
}
catch (IOException | StorageException e) {
myFatalErrorsConsumer.consume(this, e);
}
}
return result;
}
public static class FileNamesData {
@NotNull private final TIntObjectHashMap<Set<FilePath>> myCommitsToPaths;
@NotNull private final TIntObjectHashMap<Set<UnorderedPair<FilePath>>> myCommitsToRenames;
public FileNamesData() {
myCommitsToPaths = new TIntObjectHashMap<>();
myCommitsToRenames = new TIntObjectHashMap<>();
}
public boolean hasRenames() {
return !myCommitsToRenames.isEmpty();
}
private void addPath(int commit, @NotNull FilePath path) {
Set<FilePath> paths = myCommitsToPaths.get(commit);
if (paths == null) {
paths = ContainerUtil.newHashSet();
myCommitsToPaths.put(commit, paths);
}
paths.add(path);
}
private void addRename(int commit, @NotNull Couple<FilePath> path) {
Set<UnorderedPair<FilePath>> paths = myCommitsToRenames.get(commit);
if (paths == null) {
paths = ContainerUtil.newHashSet();
myCommitsToRenames.put(commit, paths);
}
paths.add(new UnorderedPair<>(path.first, path.second));
}
public void add(int commit, @NotNull Couple<FilePath> paths) {
if (paths.second == null) {
addPath(commit, paths.first);
}
else {
addRename(commit, paths);
}
}
public boolean affects(int commit, @NotNull FilePath path) {
Set<FilePath> paths = myCommitsToPaths.get(commit);
if (paths != null && paths.contains(path)) return true;
return getRenamedPath(commit, path) != null;
}
@Nullable
public FilePath getRenamedPath(int commit, @Nullable FilePath newName) {
Set<UnorderedPair<FilePath>> renames = myCommitsToRenames.get(commit);
if (renames == null) return null;
for (UnorderedPair<FilePath> rename : renames) {
if (rename.first.equals(newName)) return rename.second;
if (rename.second.equals(newName)) return rename.first;
}
return null;
}
@Nullable
public FilePath getPreviousPath(int commit, @Nullable FilePath path) {
Set<FilePath> paths = myCommitsToPaths.get(commit);
if (paths != null && paths.contains(path)) return path;
return getRenamedPath(commit, path);
}
public void remove(int commit) {
myCommitsToPaths.remove(commit);
myCommitsToRenames.remove(commit);
}
public void retain(int commit, @NotNull FilePath path, @NotNull FilePath previousPath) {
if (path.equals(previousPath)) {
myCommitsToPaths.put(commit, ContainerUtil.set(path));
myCommitsToRenames.remove(commit);
}
else {
myCommitsToPaths.remove(commit);
myCommitsToRenames.put(commit, ContainerUtil.set(new UnorderedPair<>(path, previousPath)));
}
}
}
}
@@ -46,6 +46,7 @@ import java.io.IOException;
import java.util.Collection;
import java.util.Map;
import java.util.Set;
import java.util.function.ObjIntConsumer;
import static com.intellij.util.containers.ContainerUtil.newTroveSet;
import static com.intellij.vcs.log.data.index.VcsLogPersistentIndex.getVersion;
@@ -82,11 +83,9 @@ public class VcsLogPathsIndex extends VcsLogFullDetailsIndex<Integer> {
myPathsIndexer.getPathsEnumerator().force();
}
@NotNull
public TIntHashSet getCommitsForPaths(@NotNull Collection<FilePath> paths) throws IOException, StorageException {
Set<Integer> allPathIds = ContainerUtil.newHashSet();
for (FilePath path : paths) {
allPathIds.add(myPathsIndexer.myPathsEnumerator.enumerate(path.getPath()));
}
Set<Integer> allPathIds = getPathIds(paths);
TIntHashSet result = new TIntHashSet();
Set<Integer> renames = allPathIds;
@@ -98,6 +97,15 @@ public class VcsLogPathsIndex extends VcsLogFullDetailsIndex<Integer> {
return result;
}
@NotNull
private Set<Integer> getPathIds(@NotNull Collection<FilePath> paths) throws IOException {
Set<Integer> allPathIds = ContainerUtil.newHashSet();
for (FilePath path : paths) {
allPathIds.add(myPathsIndexer.myPathsEnumerator.enumerate(path.getPath()));
}
return allPathIds;
}
@NotNull
public Set<FilePath> getFileNames(@NotNull FilePath path, int commit) throws IOException, StorageException {
int startId = myPathsIndexer.myPathsEnumerator.enumerate(path.getPath());
@@ -138,6 +146,37 @@ public class VcsLogPathsIndex extends VcsLogFullDetailsIndex<Integer> {
return result;
}
public void iterateCommits(@NotNull Collection<FilePath> paths, @NotNull ObjIntConsumer<Couple<FilePath>> consumer)
throws IOException, StorageException {
Set<Integer> startIds = getPathIds(paths);
Set<Integer> allIds = ContainerUtil.newHashSet(startIds);
Set<Integer> newIds = ContainerUtil.newHashSet();
while (!startIds.isEmpty()) {
for (int currentPathId : startIds) {
FilePath currentPath = VcsUtil.getFilePath(myPathsIndexer.myPathsEnumerator.valueOf(currentPathId));
iterateCommitIdsAndValues(currentPathId, (renamedPathId, commitId) -> {
FilePath renamedPath = null;
if (renamedPathId != null) {
if (!allIds.contains(renamedPathId)) {
newIds.add(renamedPathId);
}
try {
renamedPath = VcsUtil.getFilePath(myPathsIndexer.myPathsEnumerator.valueOf(renamedPathId));
}
catch (IOException e) {
LOG.error(e);
}
}
consumer.accept(Couple.of(currentPath, renamedPath), commitId);
});
}
startIds = ContainerUtil.newHashSet(newIds);
allIds.addAll(startIds);
newIds.clear();
}
}
@NotNull
public Set<Integer> addCommitsAndGetRenames(@NotNull Set<Integer> newPathIds,
@NotNull Set<Integer> allPathIds,
@@ -15,20 +15,228 @@
*/
package com.intellij.vcs.log.ui.history;
import com.intellij.openapi.util.Pair;
import com.intellij.openapi.util.Ref;
import com.intellij.openapi.vcs.FilePath;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.util.ObjectUtils;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.containers.Stack;
import com.intellij.vcs.log.Hash;
import com.intellij.vcs.log.VcsLogFilterCollection;
import com.intellij.vcs.log.VcsRef;
import com.intellij.vcs.log.data.CompressedRefs;
import com.intellij.vcs.log.data.DataPack;
import com.intellij.vcs.log.data.VcsLogData;
import com.intellij.vcs.log.data.index.IndexDataGetter;
import com.intellij.vcs.log.graph.PermanentGraph;
import com.intellij.vcs.log.graph.VisibleGraph;
import com.intellij.vcs.log.graph.api.LinearGraph;
import com.intellij.vcs.log.graph.api.LiteLinearGraph;
import com.intellij.vcs.log.graph.impl.facade.PermanentGraphImpl;
import com.intellij.vcs.log.graph.impl.facade.ReachableNodes;
import com.intellij.vcs.log.graph.impl.facade.VisibleGraphImpl;
import com.intellij.vcs.log.graph.impl.permanent.PermanentCommitsInfoImpl;
import com.intellij.vcs.log.graph.utils.LinearGraphUtils;
import com.intellij.vcs.log.graph.utils.impl.BitSetFlags;
import com.intellij.vcs.log.visible.VcsLogFilterer;
import com.intellij.vcs.log.visible.VisiblePack;
import com.intellij.vcsUtil.VcsUtil;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.util.Collections;
import java.util.Optional;
import java.util.Set;
class FileHistoryFilterer extends VcsLogFilterer {
@NotNull private final FilePath myFilePath;
@NotNull private final IndexDataGetter myIndexDataGetter;
@NotNull private final VirtualFile myRoot;
public FileHistoryFilterer(@NotNull VcsLogData logData, @NotNull FilePath filePath) {
super(logData.getLogProviders(), logData.getStorage(), logData.getTopCommitsCache(), logData.getCommitDetailsGetter(),
logData.getIndex());
myFilePath = filePath;
myIndexDataGetter = ObjectUtils.assertNotNull(myIndex.getDataGetter());
myRoot = ObjectUtils.assertNotNull(VcsUtil.getVcsRootFor(logData.getProject(), myFilePath));
}
// this is empty now
// that's going to change soon
@NotNull
protected VisiblePack createVisiblePack(@NotNull DataPack dataPack,
@NotNull PermanentGraph.SortType sortType,
@NotNull VcsLogFilterCollection filters,
@Nullable Set<Integer> matchingHeads,
@Nullable Set<Integer> matchingCommits,
boolean canRequestMore) {
if (!myIndex.isIndexed(myRoot) || matchingCommits == null) {
return super.createVisiblePack(dataPack, sortType, filters, matchingHeads, matchingCommits, canRequestMore);
}
// I realize, I'm calculating this data for the second time.
// No worries! It's going to be fine later!
IndexDataGetter.FileNamesData namesData = myIndexDataGetter.buildFileNamesData(myFilePath);
if (!namesData.hasRenames()) {
return super.createVisiblePack(dataPack, sortType, filters, matchingHeads, matchingCommits, canRequestMore);
}
VisibleGraph<Integer> visibleGraph = createVisibleGraph(dataPack, sortType, matchingHeads, matchingCommits);
if (visibleGraph instanceof VisibleGraphImpl) {
FileHistoryRefiner refiner = new FileHistoryRefiner(visibleGraph, namesData);
if (refiner.refine(((VisibleGraphImpl)visibleGraph).getLinearGraph(), getCurrentRow(dataPack, visibleGraph, namesData), myFilePath)) {
// creating a vg is the most expensive task, so trying to avoid that when unnecessary
return new VisiblePack(dataPack, createVisibleGraph(dataPack, sortType, matchingHeads, refiner.getMatchingCommits()),
canRequestMore,
filters);
}
}
return new VisiblePack(dataPack, visibleGraph, canRequestMore, filters);
}
private int getCurrentRow(@NotNull DataPack pack,
@NotNull VisibleGraph<Integer> visibleGraph,
@NotNull IndexDataGetter.FileNamesData fileIndexData) {
PermanentGraph<Integer> permanentGraph = pack.getPermanentGraph();
if (permanentGraph instanceof PermanentGraphImpl) {
CompressedRefs refs = pack.getRefsModel().getAllRefsByRoot().get(myRoot);
Optional<VcsRef> headOptional = refs.streamBranches().filter(br -> br.getName().equals("HEAD")).findFirst();
if (headOptional.isPresent()) {
VcsRef head = headOptional.get();
assert head.getRoot().equals(myRoot);
return findAncestorRowAffectingFile((PermanentGraphImpl<Integer>)permanentGraph, head.getCommitHash(), visibleGraph, fileIndexData);
}
}
return 0;
}
private int findAncestorRowAffectingFile(@NotNull PermanentGraphImpl<Integer> permanentGraph,
@NotNull Hash hash,
@NotNull VisibleGraph<Integer> visibleGraph,
@NotNull IndexDataGetter.FileNamesData fileNamesData) {
Ref<Integer> result = new Ref<>();
PermanentCommitsInfoImpl<Integer> commitsInfo = permanentGraph.getPermanentCommitsInfo();
ReachableNodes reachableNodes = new ReachableNodes(LinearGraphUtils.asLiteLinearGraph(permanentGraph.getLinearGraph()));
reachableNodes.walk(Collections.singleton(commitsInfo.getNodeId(myStorage.getCommitIndex(hash, myRoot))), true,
currentNode -> {
int id = commitsInfo.getCommitId(currentNode);
if (fileNamesData.affects(id, myFilePath)) {
result.set(currentNode);
return false; // stop walk, we have found it
}
return true; // continue walk
});
if (!result.isNull()) {
Integer rowIndex = visibleGraph.getVisibleRowIndex(commitsInfo.getCommitId(result.get()));
return ObjectUtils.assertNotNull(rowIndex);
}
return 0;
}
private static void walk(@NotNull LiteLinearGraph graph, int start, @NotNull NodeVisitor visitor) {
BitSetFlags visited = new BitSetFlags(graph.nodesCount(), false);
BitSetFlags visitedInSameDirection = new BitSetFlags(graph.nodesCount(), false);
Stack<Pair<Integer, Boolean>> stack = new Stack<>();
stack.push(new Pair<>(start, true)); // commit + direction of travel
while (!stack.empty()) {
int currentNode = stack.peek().first;
boolean down = stack.peek().second;
if (!visited.get(currentNode)) {
visited.set(currentNode, true);
visitor.enterNode(currentNode);
}
boolean found = false;
for (int nextNode : graph.getNodes(currentNode, down ? LiteLinearGraph.NodeFilter.DOWN : LiteLinearGraph.NodeFilter.UP)) {
if (!visited.get(nextNode)) {
stack.push(new Pair<>(nextNode, down));
found = true;
break;
}
}
if (!found) {
if (!visitedInSameDirection.get(currentNode)) {
visitedInSameDirection.set(currentNode, true);
visitor.exitNode(currentNode);
}
for (int nextNode : graph.getNodes(currentNode, down ? LiteLinearGraph.NodeFilter.UP : LiteLinearGraph.NodeFilter.DOWN)) {
if (!visited.get(nextNode)) {
stack.push(new Pair<>(nextNode, !down));
found = true;
break;
}
}
}
if (!found) {
stack.pop();
}
}
}
public interface NodeVisitor {
void enterNode(int node);
void exitNode(int node);
}
private static class FileHistoryRefiner implements NodeVisitor {
@NotNull private final VisibleGraph<Integer> myVisibleGraph;
@NotNull private final IndexDataGetter.FileNamesData myNamesData;
@NotNull private final Stack<FilePath> myPaths;
@NotNull private final Set<Integer> myMatchingCommits;
private boolean myWasChanged;
public FileHistoryRefiner(@NotNull VisibleGraph<Integer> visibleGraph,
@NotNull IndexDataGetter.FileNamesData namesData) {
myVisibleGraph = visibleGraph;
myNamesData = namesData;
myPaths = new Stack<>();
myMatchingCommits = ContainerUtil.newHashSet();
myWasChanged = false;
}
public boolean refine(@NotNull LinearGraph graph, int row, @NotNull FilePath startPath) {
myPaths.push(startPath);
walk(LinearGraphUtils.asLiteLinearGraph(graph), row, this);
return myWasChanged;
}
@NotNull
public Set<Integer> getMatchingCommits() {
return myMatchingCommits;
}
@Override
public void enterNode(int node) {
FilePath currentPath = myPaths.peek();
Integer commit = myVisibleGraph.getRowInfo(node).getCommit();
FilePath previousPath = myNamesData.getPreviousPath(commit, currentPath);
if (previousPath != null) {
myMatchingCommits.add(commit);
if (!currentPath.equals(previousPath)) myPaths.push(previousPath);
myNamesData.retain(commit, currentPath, previousPath);
}
else {
myNamesData.remove(commit);
myWasChanged = true;
}
}
@Override
public void exitNode(int node) {
Integer commit = myVisibleGraph.getRowInfo(node).getCommit();
if (myMatchingCommits.contains(commit)) {
if (myNamesData.getRenamedPath(commit, myPaths.peek()) != null) myPaths.pop();
}
}
}
}
@@ -42,11 +42,11 @@ public class VcsLogFilterer {
private static final Logger LOG = Logger.getInstance(VcsLogFilterer.class);
@NotNull private final VcsLogStorage myStorage;
@NotNull protected final VcsLogStorage myStorage;
@NotNull private final TopCommitsCache myTopCommitsDetailsCache;
@NotNull private final DataGetter<VcsFullCommitDetails> myCommitDetailsGetter;
@NotNull private final Map<VirtualFile, VcsLogProvider> myLogProviders;
@NotNull private final VcsLogIndex myIndex;
@NotNull protected final VcsLogIndex myIndex;
public VcsLogFilterer(@NotNull Map<VirtualFile, VcsLogProvider> providers,
@NotNull VcsLogStorage storage,
@@ -77,17 +77,37 @@ public class VcsLogFilterer {
Set<Integer> matchingHeads = getMatchingHeads(dataPack.getRefsModel(), visibleRoots, filters);
FilterResult filterResult = filterByDetails(dataPack, filters, commitCount, visibleRoots, matchingHeads);
VisiblePack visiblePack =
createVisiblePack(dataPack, sortType, filters, matchingHeads, filterResult.matchingCommits, filterResult.canRequestMore);
LOG.debug(StopWatch.formatTime(System.currentTimeMillis() - start) + " for filtering by " + filters);
return Pair.create(visiblePack, filterResult.commitCount);
}
@NotNull
protected VisiblePack createVisiblePack(@NotNull DataPack dataPack,
@NotNull PermanentGraph.SortType sortType,
@NotNull VcsLogFilterCollection filters,
@Nullable Set<Integer> matchingHeads,
@Nullable Set<Integer> matchingCommits,
boolean canRequestMore) {
VisibleGraph<Integer> visibleGraph = createVisibleGraph(dataPack, sortType, matchingHeads, matchingCommits);
return new VisiblePack(dataPack, visibleGraph, canRequestMore, filters);
}
@NotNull
protected VisibleGraph<Integer> createVisibleGraph(@NotNull DataPack dataPack,
@NotNull PermanentGraph.SortType sortType,
@Nullable Set<Integer> matchingHeads,
@Nullable Set<Integer> matchingCommits) {
VisibleGraph<Integer> visibleGraph;
if (matchesNothing(matchingHeads) || matchesNothing(filterResult.matchingCommits)) {
if (matchesNothing(matchingHeads) || matchesNothing(matchingCommits)) {
visibleGraph = EmptyVisibleGraph.getInstance();
}
else {
visibleGraph = dataPack.getPermanentGraph().createVisibleGraph(sortType, matchingHeads, filterResult.matchingCommits);
visibleGraph = dataPack.getPermanentGraph().createVisibleGraph(sortType, matchingHeads, matchingCommits);
}
LOG.debug(StopWatch.formatTime(System.currentTimeMillis() - start) + " for filtering by " + filters);
return Pair.create(new VisiblePack(dataPack, visibleGraph, filterResult.canRequestMore, filters), filterResult.commitCount);
return visibleGraph;
}
@NotNull
@@ -38,10 +38,10 @@ public class VisiblePack implements VcsLogDataPack {
private final boolean myCanRequestMore;
@NotNull private final VcsLogFilterCollection myFilters;
VisiblePack(@NotNull DataPackBase dataPack,
@NotNull VisibleGraph<Integer> graph,
boolean canRequestMore,
@NotNull VcsLogFilterCollection filters) {
public VisiblePack(@NotNull DataPackBase dataPack,
@NotNull VisibleGraph<Integer> graph,
boolean canRequestMore,
@NotNull VcsLogFilterCollection filters) {
myDataPack = dataPack;
myVisibleGraph = graph;
myCanRequestMore = canRequestMore;