From 652c69ab1e5cc9ce7b4b6d44d5f5cab840349494 Mon Sep 17 00:00:00 2001 From: Kirill Likhodedov Date: Mon, 4 Aug 2014 17:01:33 +0400 Subject: [PATCH] [log] IDEA-125987 Consume less temporary memory when loading full log Instead of collecting all Hash commits and then compacting them to integers, report them one by one and compact at once. Implement for Git, keep as is for Mercurial by now. --- .../com/intellij/vcs/log/VcsLogProvider.java | 4 +- .../vcs/log/data/VcsLogMultiRepoJoiner.java | 26 +++--- .../vcs/log/data/VcsLogRefresherImpl.java | 82 +++++++++++-------- .../vcs/log/impl/TestVcsLogProvider.java | 8 +- .../src/git4idea/history/GitHistoryUtils.java | 25 ++++-- .../src/git4idea/log/GitBekParentFixer.java | 57 +++++-------- .../src/git4idea/log/GitLogProvider.java | 15 ++-- .../git4idea/log/GitLogProviderTest.java | 8 +- .../org/zmlx/hg4idea/log/HgLogProvider.java | 9 +- 9 files changed, 129 insertions(+), 105 deletions(-) diff --git a/platform/vcs-log/api/src/com/intellij/vcs/log/VcsLogProvider.java b/platform/vcs-log/api/src/com/intellij/vcs/log/VcsLogProvider.java index 5d4f1fb83cad..89805947869c 100644 --- a/platform/vcs-log/api/src/com/intellij/vcs/log/VcsLogProvider.java +++ b/platform/vcs-log/api/src/com/intellij/vcs/log/VcsLogProvider.java @@ -29,8 +29,8 @@ public interface VcsLogProvider { *

Reads the whole history, but only hashes & parents.

*

Also reports authors/committers of this repository to the given user registry.

*/ - @NotNull - List readAllHashes(@NotNull VirtualFile root, @NotNull Consumer userRegistry) throws VcsException; + void readAllHashes(@NotNull VirtualFile root, @NotNull Consumer userRegistry, + @NotNull Consumer commitConsumer) throws VcsException; /** * Reads those details of the given commits, which are necessary to be shown in the log table. diff --git a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/VcsLogMultiRepoJoiner.java b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/VcsLogMultiRepoJoiner.java index 7006408943a8..072e0e26ace1 100644 --- a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/VcsLogMultiRepoJoiner.java +++ b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/VcsLogMultiRepoJoiner.java @@ -1,36 +1,36 @@ package com.intellij.vcs.log.data; import com.intellij.util.containers.ContainerUtil; -import com.intellij.vcs.log.TimedVcsCommit; +import com.intellij.vcs.log.graph.GraphCommit; import org.jetbrains.annotations.NotNull; import java.util.*; -public class VcsLogMultiRepoJoiner { +public class VcsLogMultiRepoJoiner { @NotNull - public List join(@NotNull Collection> logsFromRepos) { + public List> join(@NotNull Collection>> logsFromRepos) { if (logsFromRepos.size() == 1) { return logsFromRepos.iterator().next(); } int size = 0; - for (List repo : logsFromRepos) { + for (List> repo : logsFromRepos) { size += repo.size(); } - List result = new ArrayList(size); + List> result = new ArrayList>(size); - Map> nextCommits = ContainerUtil.newHashMap(); - for (List log : logsFromRepos) { - Iterator iterator = log.iterator(); + Map, Iterator>> nextCommits = ContainerUtil.newHashMap(); + for (List> log : logsFromRepos) { + Iterator> iterator = log.iterator(); if (iterator.hasNext()) { nextCommits.put(iterator.next(), iterator); } } while (!nextCommits.isEmpty()) { - TimedVcsCommit lastCommit = findLatestCommit(nextCommits.keySet()); - Iterator iterator = nextCommits.get(lastCommit); + GraphCommit lastCommit = findLatestCommit(nextCommits.keySet()); + Iterator> iterator = nextCommits.get(lastCommit); result.add(lastCommit); nextCommits.remove(lastCommit); @@ -43,10 +43,10 @@ public class VcsLogMultiRepoJoiner { } @NotNull - private static TimedVcsCommit findLatestCommit(@NotNull Set commits) { + private GraphCommit findLatestCommit(@NotNull Set> commits) { long maxTimeStamp = Long.MIN_VALUE; - TimedVcsCommit lastCommit = null; - for (TimedVcsCommit commit : commits) { + GraphCommit lastCommit = null; + for (GraphCommit commit : commits) { if (commit.getTimestamp() >= maxTimeStamp) { maxTimeStamp = commit.getTimestamp(); lastCommit = commit; diff --git a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/VcsLogRefresherImpl.java b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/VcsLogRefresherImpl.java index 0c25d7f5709d..45ffc5c182b9 100644 --- a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/VcsLogRefresherImpl.java +++ b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/VcsLogRefresherImpl.java @@ -96,11 +96,11 @@ public class VcsLogRefresherImpl implements VcsLogRefresher { Map> refs = loadRefsFromVcs(myProviders); Set roots = myProviders.keySet(); Map requirements = prepareSimpleRequirements(roots, myRecentCommitCount); - Map> commits = loadRecentCommitsFromVcs(myProviders, requirements, - myUserRegistry, myTopCommitsDetailsCache); - List compoundLog = compound(commits.values()); - List> compactedLog = compactCommits(compoundLog, myHashMap); - DataPack dataPack = DataPack.build(compactedLog, new RefsModel(refs, myHashMap.asIndexGetter()), + Map>> commits = loadRecentCommitsFromVcs(myProviders, requirements, + myUserRegistry, myTopCommitsDetailsCache, + myHashMap); + List> compoundLog = compound(commits.values()); + DataPack dataPack = DataPack.build(compoundLog, new RefsModel(refs, myHashMap.asIndexGetter()), myHashMap.asIndexGetter(), myHashMap.asHashGetter(), myProviders, false); mySingleTaskController.request(RefreshRequest.RELOAD_ALL); // build/rebuild the full log in bg return dataPack; @@ -152,19 +152,21 @@ public class VcsLogRefresherImpl implements VcsLogRefresher { } @NotNull - private static Map> loadRecentCommitsFromVcs(@NotNull Map providers, - @NotNull final Map requirements, - @NotNull final VcsUserRegistryImpl userRegistry, - @NotNull final Map topCommitsDetailsCache) - throws VcsException { + private static Map>> loadRecentCommitsFromVcs( + @NotNull Map providers, + @NotNull final Map requirements, + @NotNull final VcsUserRegistryImpl userRegistry, + @NotNull final Map topCommitsDetailsCache, + @NotNull final VcsLogHashMap hashMap) throws VcsException + { final StopWatch sw = StopWatch.start("loading commits"); - final Map> commits = ContainerUtil.newHashMap(); + final Map>> commits = ContainerUtil.newHashMap(); new ProviderIterator() { @Override public void each(@NotNull VirtualFile root, @NotNull VcsLogProvider provider) throws VcsException { List metadatas = provider.readFirstBlock(root, requirements.get(root)); storeUsersAndDetails(metadatas, userRegistry, topCommitsDetailsCache); - commits.put(root, metadatas); + commits.put(root, compactCommits(metadatas, hashMap)); sw.rootCompleted(root); } }.iterate(providers); @@ -177,9 +179,9 @@ public class VcsLogRefresherImpl implements VcsLogRefresher { * Compounds logs from different repositories into a single multi-repository log. */ @NotNull - private static List compound(@NotNull Collection> commits) { + private static List> compound(@NotNull Collection>> commits) { StopWatch sw = StopWatch.start("multi-repo join"); - List joined = new VcsLogMultiRepoJoiner().join(commits); + List> joined = new VcsLogMultiRepoJoiner().join(commits); sw.report(); return joined; } @@ -192,8 +194,7 @@ public class VcsLogRefresherImpl implements VcsLogRefresher { @NotNull @Override public GraphCommit fun(@NotNull TimedVcsCommit commit) { - return new GraphCommitImpl(hashMap.getCommitIndex(commit.getId()), - ContainerUtil.map(commit.getParents(), hashMap.asIndexGetter()), commit.getTimestamp()); + return compactCommit(commit, hashMap); } }); hashMap.flush(); @@ -201,6 +202,12 @@ public class VcsLogRefresherImpl implements VcsLogRefresher { return map; } + @NotNull + private static GraphCommitImpl compactCommit(@NotNull TimedVcsCommit commit, @NotNull VcsLogHashMap hashMap) { + return new GraphCommitImpl(hashMap.getCommitIndex(commit.getId()), + ContainerUtil.map(commit.getParents(), hashMap.asIndexGetter()), commit.getTimestamp()); + } + private static void storeUsersAndDetails(@NotNull List metadatas, @NotNull VcsUserRegistryImpl userRegistry, @NotNull Map topCommitsDetailsCache) { for (VcsCommitMetadata detail : metadatas) { @@ -218,9 +225,9 @@ public class VcsLogRefresherImpl implements VcsLogRefresher { private final Map myLoadedInfos = ContainerUtil.newHashMap(); private class LogAndRefs { - List log; + List> log; Collection refs; - LogAndRefs(Collection refs, List commits) { + LogAndRefs(Collection refs, List> commits) { this.refs = refs; this.log = commits; } @@ -266,16 +273,16 @@ public class VcsLogRefresherImpl implements VcsLogRefresher { try { if (permanentGraph != null) { loadLogAndRefs(roots, currentRefs, myRecentCommitCount); - List compoundLog = compound(ContainerUtil.map(myLoadedInfos.values(), - new Function>() { + List> compoundLog = compound(ContainerUtil.map(myLoadedInfos.values(), + new Function>>() { @Override - public List fun(LogAndRefs refs) { + public List> fun( + LogAndRefs refs) { return refs.log; } })); - List> preparedLog = compactCommits(compoundLog, myHashMap); Map> allNewRefs = getAllNewRefs(myLoadedInfos, currentRefs); - List> joinedFullLog = join(preparedLog, permanentGraph.getAllCommits(), currentRefs, allNewRefs); + List> joinedFullLog = join(compoundLog, permanentGraph.getAllCommits(), currentRefs, allNewRefs); if (joinedFullLog != null) { return DataPack.build(joinedFullLog, new RefsModel(allNewRefs, myHashMap.asIndexGetter()), myHashMap.asIndexGetter(), myHashMap.asHashGetter(), myProviders, true); @@ -310,8 +317,9 @@ public class VcsLogRefresherImpl implements VcsLogRefresher { Map providers = getProviders(roots); Map> refs = loadRefsFromVcs(providers); Map requirements = prepareRequirements(roots, commitCount, prevRefs, refs); - Map> commits = loadRecentCommitsFromVcs(providers, requirements, - myUserRegistry, myTopCommitsDetailsCache); + Map>> commits = loadRecentCommitsFromVcs(providers, requirements, + myUserRegistry, myTopCommitsDetailsCache, + myHashMap); for (VirtualFile root : roots) { myLoadedInfos.put(root, new LogAndRefs(refs.get(root), commits.get(root))); } @@ -345,7 +353,7 @@ public class VcsLogRefresherImpl implements VcsLogRefresher { } @Nullable - private List> join(@NotNull List> recentCommits, @NotNull List> fullLog, + private List> join(@NotNull List> recentCommits, @NotNull List> fullLog, @NotNull Map> previousRefs, @NotNull Map> newRefs) { StopWatch sw = StopWatch.start("joining new commits"); @@ -374,9 +382,8 @@ public class VcsLogRefresherImpl implements VcsLogRefresher { @NotNull private Pair, Map>> loadFullLog() throws VcsException { StopWatch sw = StopWatch.start("full log reload"); - Collection> commits = readFullLogFromVcs(); - List compoundLog = compound(commits); - List> graphCommits = compactCommits(compoundLog, myHashMap); + Collection>> commits = readFullLogFromVcs(); + List> graphCommits = compound(commits); Map> refMap = loadRefsFromVcs(myProviders); PermanentGraph permanentGraph = DataPack.buildPermanentGraph(graphCommits, new RefsModel(refMap, myHashMap.asIndexGetter()), myHashMap.asIndexGetter(), @@ -386,24 +393,31 @@ public class VcsLogRefresherImpl implements VcsLogRefresher { } @NotNull - private Collection> readFullLogFromVcs() throws VcsException { + private Collection>> readFullLogFromVcs() throws VcsException { final StopWatch sw = StopWatch.start("read full log from VCS"); - final Collection> commits = ContainerUtil.newArrayList(); + final Collection>> logs = ContainerUtil.newArrayList(); new ProviderIterator() { @Override void each(@NotNull VirtualFile root, @NotNull VcsLogProvider provider) throws VcsException { - commits.add(provider.readAllHashes(root, new Consumer() { + final List> graphCommits = ContainerUtil.newArrayList(); + provider.readAllHashes(root, new Consumer() { @Override public void consume(@NotNull VcsUser user) { myUserRegistry.addUser(user); } - })); + }, new Consumer() { + @Override + public void consume(TimedVcsCommit commit) { + graphCommits.add(compactCommit(commit, myHashMap)); + } + }); + logs.add(graphCommits); sw.rootCompleted(root); } }.iterate(myProviders); myUserRegistry.flush(); sw.report(); - return commits; + return logs; } } diff --git a/platform/vcs-log/impl/test/com/intellij/vcs/log/impl/TestVcsLogProvider.java b/platform/vcs-log/impl/test/com/intellij/vcs/log/impl/TestVcsLogProvider.java index d3709b2d8f55..850478b048dc 100644 --- a/platform/vcs-log/impl/test/com/intellij/vcs/log/impl/TestVcsLogProvider.java +++ b/platform/vcs-log/impl/test/com/intellij/vcs/log/impl/TestVcsLogProvider.java @@ -93,9 +93,9 @@ public class TestVcsLogProvider implements VcsLogProvider { return ContainerUtil.map(myCommits.subList(0, requirements.getCommitCount()), myCommitToMetadataConvertor); } - @NotNull @Override - public List readAllHashes(@NotNull VirtualFile root, @NotNull Consumer userRegistry) throws VcsException { + public void readAllHashes(@NotNull VirtualFile root, @NotNull Consumer userRegistry, + @NotNull Consumer commitConsumer) throws VcsException { try { myFullLogSemaphore.acquire(); } @@ -103,7 +103,9 @@ public class TestVcsLogProvider implements VcsLogProvider { throw new RuntimeException(e); } assertRoot(root); - return myCommits; + for (TimedVcsCommit commit : myCommits) { + commitConsumer.consume(commit); + } } private void assertRoot(@NotNull VirtualFile root) { diff --git a/plugins/git4idea/src/git4idea/history/GitHistoryUtils.java b/plugins/git4idea/src/git4idea/history/GitHistoryUtils.java index 114b4beb5435..8273c3e1abdc 100644 --- a/plugins/git4idea/src/git4idea/history/GitHistoryUtils.java +++ b/plugins/git4idea/src/git4idea/history/GitHistoryUtils.java @@ -520,9 +520,19 @@ public class GitHistoryUtils { @NotNull VirtualFile root, @NotNull final Consumer userRegistry, @NotNull List parameters) throws VcsException { + List collector = ContainerUtil.newArrayList(); + readCommits(project, root, userRegistry, parameters, new CollectConsumer(collector)); + return collector; + } + + public static void readCommits(@NotNull final Project project, + @NotNull VirtualFile root, + @NotNull final Consumer userRegistry, + @NotNull List parameters, + @NotNull final Consumer commitConsumer) throws VcsException { final VcsLogObjectsFactory factory = getObjectsFactoryWithDisposeCheck(project); if (factory == null) { - return Collections.emptyList(); + return; } final int COMMIT_BUFFER = 1000; @@ -536,8 +546,6 @@ public class GitHistoryUtils { h.addParameters(parameters); h.endOptions(); - final List commits = ContainerUtil.newArrayList(); - final StringBuilder record = new StringBuilder(); final AtomicInteger records = new AtomicInteger(); final Ref ex = new Ref(); @@ -560,7 +568,10 @@ public class GitHistoryUtils { afterParseRemainder = line.substring(recordEnd + 1); } if (afterParseRemainder != null && records.incrementAndGet() > COMMIT_BUFFER) { // null means can't parse now - commits.addAll(parseCommit(parser, record, userRegistry, factory)); + List commits = parseCommit(parser, record, userRegistry, factory); + for (TimedVcsCommit commit : commits) { + commitConsumer.consume(commit); + } record.setLength(0); record.append(afterParseRemainder); } @@ -573,7 +584,10 @@ public class GitHistoryUtils { @Override public void processTerminated(int exitCode) { try { - commits.addAll(parseCommit(parser, record, userRegistry, factory)); + List commits = parseCommit(parser, record, userRegistry, factory); + for (TimedVcsCommit commit : commits) { + commitConsumer.consume(commit); + } } catch (Exception e) { ex.set(new VcsException(e)); @@ -589,7 +603,6 @@ public class GitHistoryUtils { if (!ex.isNull()) { throw ex.get(); } - return commits; } @NotNull diff --git a/plugins/git4idea/src/git4idea/log/GitBekParentFixer.java b/plugins/git4idea/src/git4idea/log/GitBekParentFixer.java index e469e61ca315..054ed9f90b48 100644 --- a/plugins/git4idea/src/git4idea/log/GitBekParentFixer.java +++ b/plugins/git4idea/src/git4idea/log/GitBekParentFixer.java @@ -27,49 +27,34 @@ import org.jetbrains.annotations.Nullable; import java.util.*; class GitBekParentFixer { - @NotNull - private final static String MAGIC_TEXT = "Merge remote"; - @NotNull - private final VcsLogFilterCollection MAGIC_FILTER = createVcsLogFilterCollection(); + @NotNull private static final String MAGIC_TEXT = "Merge remote"; + @NotNull private static final VcsLogFilterCollection MAGIC_FILTER = createVcsLogFilterCollection(); - @NotNull - private final VirtualFile myRoot; - @NotNull - private final GitLogProvider myGitLogProvider; - @NotNull - private final List myAllCommits; + @NotNull private final Set myWrongCommits; - GitBekParentFixer(@NotNull VirtualFile root, @NotNull GitLogProvider gitLogProvider, @NotNull List allCommits) { - myRoot = root; - myGitLogProvider = gitLogProvider; - myAllCommits = allCommits; + private GitBekParentFixer(@NotNull Set wrongCommits) { + myWrongCommits = wrongCommits; } @NotNull - List getCorrectCommits() throws VcsException { - if (!BekSorter.isBekEnabled()) - return myAllCommits; - - final Set wrongCommits = getWrongCommits(); - return new AbstractList() { - @Override - public TimedVcsCommit get(int index) { - TimedVcsCommit commit = myAllCommits.get(index); - if (!wrongCommits.contains(commit.getId())) - return commit; - - return reverseParents(commit); - } - - @Override - public int size() { - return myAllCommits.size(); - } - }; + static GitBekParentFixer prepare(@NotNull VirtualFile root, @NotNull GitLogProvider provider) throws VcsException { + if (!BekSorter.isBekEnabled()) { + return new GitBekParentFixer(Collections.emptySet()); + } + return new GitBekParentFixer(getWrongCommits(provider, root)); } - private Set getWrongCommits() throws VcsException { - List commitsMatchingFilter = myGitLogProvider.getCommitsMatchingFilter(myRoot, MAGIC_FILTER, -1); + @NotNull + TimedVcsCommit fixCommit(@NotNull TimedVcsCommit commit) { + if (!myWrongCommits.contains(commit.getId())) { + return commit; + } + return reverseParents(commit); + } + + @NotNull + private static Set getWrongCommits(@NotNull GitLogProvider provider, @NotNull VirtualFile root) throws VcsException { + List commitsMatchingFilter = provider.getCommitsMatchingFilter(root, MAGIC_FILTER, -1); return ContainerUtil.map2Set(commitsMatchingFilter, new Function() { @Override public Hash fun(TimedVcsCommit timedVcsCommit) { diff --git a/plugins/git4idea/src/git4idea/log/GitLogProvider.java b/plugins/git4idea/src/git4idea/log/GitLogProvider.java index ecc4709be5d3..435313fa5c7d 100644 --- a/plugins/git4idea/src/git4idea/log/GitLogProvider.java +++ b/plugins/git4idea/src/git4idea/log/GitLogProvider.java @@ -154,18 +154,23 @@ public class GitLogProvider implements VcsLogProvider { }); } - @NotNull @Override - public List readAllHashes(@NotNull VirtualFile root, @NotNull Consumer userRegistry) throws VcsException { + public void readAllHashes(@NotNull VirtualFile root, @NotNull Consumer userRegistry, + @NotNull final Consumer commitConsumer) throws VcsException { if (!isRepositoryReady(root)) { - return Collections.emptyList(); + return; } List parameters = new ArrayList(GitHistoryUtils.LOG_ALL); parameters.add("--sparse"); - List timedVcsCommits = GitHistoryUtils.readCommits(myProject, root, userRegistry, parameters); - return new GitBekParentFixer(root, this, timedVcsCommits).getCorrectCommits(); + final GitBekParentFixer parentFixer = GitBekParentFixer.prepare(root, this); + GitHistoryUtils.readCommits(myProject, root, userRegistry, parameters, new Consumer() { + @Override + public void consume(TimedVcsCommit commit) { + commitConsumer.consume(parentFixer.fixCommit(commit)); + } + }); } @NotNull diff --git a/plugins/git4idea/tests/git4idea/log/GitLogProviderTest.java b/plugins/git4idea/tests/git4idea/log/GitLogProviderTest.java index 797e0271b430..0bb67671622b 100644 --- a/plugins/git4idea/tests/git4idea/log/GitLogProviderTest.java +++ b/plugins/git4idea/tests/git4idea/log/GitLogProviderTest.java @@ -20,6 +20,7 @@ import com.intellij.openapi.util.Condition; import com.intellij.openapi.util.text.StringUtil; import com.intellij.openapi.vcs.VcsException; import com.intellij.util.ArrayUtil; +import com.intellij.util.CollectConsumer; import com.intellij.util.Consumer; import com.intellij.util.Function; import com.intellij.util.containers.ContainerUtil; @@ -84,9 +85,10 @@ public class GitLogProviderTest extends GitSingleRepoTest { prepareSomeHistory(); createTaggedBranch(); List expectedLog = log(); - @SuppressWarnings("unchecked") - List actualLog = myLogProvider.readAllHashes(myProjectRoot, Consumer.EMPTY_CONSUMER); - assertOrderedEquals(expectedLog, actualLog); + List collector = ContainerUtil.newArrayList(); + //noinspection unchecked + myLogProvider.readAllHashes(myProjectRoot, Consumer.EMPTY_CONSUMER, new CollectConsumer(collector)); + assertOrderedEquals(expectedLog, collector); } public void test_get_current_user() throws Exception { diff --git a/plugins/hg4idea/src/org/zmlx/hg4idea/log/HgLogProvider.java b/plugins/hg4idea/src/org/zmlx/hg4idea/log/HgLogProvider.java index 03c98621a8c2..4adec9dff515 100644 --- a/plugins/hg4idea/src/org/zmlx/hg4idea/log/HgLogProvider.java +++ b/plugins/hg4idea/src/org/zmlx/hg4idea/log/HgLogProvider.java @@ -64,10 +64,13 @@ public class HgLogProvider implements VcsLogProvider { return HgHistoryUtil.loadMetadata(myProject, root, requirements.getCommitCount(), Collections.emptyList()); } - @NotNull @Override - public List readAllHashes(@NotNull VirtualFile root, @NotNull Consumer userRegistry) throws VcsException { - return HgHistoryUtil.readAllHashes(myProject, root, userRegistry, Collections.emptyList()); + public void readAllHashes(@NotNull VirtualFile root, @NotNull Consumer userRegistry, + @NotNull Consumer commitConsumer) throws VcsException { + List commits = HgHistoryUtil.readAllHashes(myProject, root, userRegistry, Collections.emptyList()); + for (TimedVcsCommit commit : commits) { + commitConsumer.consume(commit); + } } @NotNull