MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

[vcs-log] index commits one by one

Browse Source 
During indexing when only a part of the repository is being processed, commits were loaded in batches by 1000, then each batch was analysed and stored in the index. After that commits themselves were (and still are) thrown away. Since it turned out commits can be quite big, it is unwise to collect them. So now commits are processed immediately as they are loaded.

IDEA-164274
This commit is contained in:
Julia Beliaeva
2016-12-08 20:59:21 +03:00
parent 5a812dda29
commit 25e8ac86d8
11 changed files with 134 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/vcs-impl/src/com/intellij/vcsUtil/VcsFileUtil.java
+23 -4
View File
@@ -26,6 +26,7 @@ import com.intellij.openapi.vcs.VcsException;
import com.intellij.openapi.vcs.changes.VcsDirtyScopeManager;
import com.intellij.openapi.vfs.VfsUtil;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.util.ThrowableConsumer;
import com.intellij.util.containers.ContainerUtil;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -62,7 +63,7 @@ public class VcsFileUtil {
}
/**
* Execute function for each chunk of arguments. Check for being cancelled in process.
* Execute function for each chunk of arguments and collect the result. Check for being cancelled in process.
*
* @param arguments the arguments to chunk
* @param groupSize size of argument groups that should be put in the same chunk (like a name and a value)
@@ -77,16 +78,34 @@ public class VcsFileUtil {
@NotNull ThrowableNotNullFunction<List<String>, List<? extends T>, VcsException> processor)
throws VcsException {
List<T> result = ContainerUtil.newArrayList();
foreachChunk(arguments, groupSize, chunk -> {
result.addAll(processor.fun(chunk));
});
return result;
}
/**
* Execute function for each chunk of arguments. Check for being cancelled in process.
*
* @param arguments the arguments to chunk
* @param groupSize size of argument groups that should be put in the same chunk (like a name and a value)
* @param consumer consumer to feed each chunk
* @throws VcsException
*/
public static void foreachChunk(@NotNull List<String> arguments,
int groupSize,
@NotNull ThrowableConsumer<List<String>, VcsException> consumer)
throws VcsException {
List<List<String>> chunks = chunkArguments(arguments, groupSize);
for (List<String> chunk : chunks) {
ProgressIndicator indicator = ProgressManager.getInstance().getProgressIndicator();
if (indicator != null) indicator.checkCanceled();
result.addAll(processor.fun(chunk));
consumer.consume(chunk);
}
return result;
}
/**
platform/vcs-log/api/src/com/intellij/vcs/log/VcsLogProvider.java
+20 -1
View File
@@ -5,6 +5,7 @@ import com.intellij.openapi.vcs.VcsException;
import com.intellij.openapi.vcs.VcsKey;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.util.Consumer;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.messages.MessageBus;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -47,6 +48,14 @@ public interface VcsLogProvider {
*/
void readAllFullDetails(@NotNull VirtualFile root, @NotNull Consumer<VcsFullCommitDetails> commitConsumer) throws VcsException;
/**
* Reads full details for specified commits in the repository.
* <p/>
* Reports commits to the consumer to avoid creation & even temporary storage of a too large commits collection.
*/
void readFullDetails(@NotNull VirtualFile root, @NotNull List<String> hashes, @NotNull Consumer<VcsFullCommitDetails> commitConsumer)
throws VcsException;
/**
* Reads those details of the given commits, which are necessary to be shown in the log table.
*/
@@ -55,9 +64,19 @@ public interface VcsLogProvider {
/**
* Read full details of the given commits from the VCS.
* <p>
* Replaced with readFullDetails(VirtualFile root, List<String>, Consumer<VcsFullCommitDetails>) method.
* <p>
* To be removed after 2017.1 release.
*/
@NotNull
List<? extends VcsFullCommitDetails> readFullDetails(@NotNull VirtualFile root, @NotNull List<String> hashes) throws VcsException;
@Deprecated
default List<? extends VcsFullCommitDetails> readFullDetails(@NotNull VirtualFile root, @NotNull List<String> hashes)
throws VcsException {
List<VcsFullCommitDetails> result = ContainerUtil.newArrayList();
readFullDetails(root, hashes, result::add);
return result;
}
/**
* <p>Returns the VCS which is supported by this provider.</p>
platform/vcs-log/impl/src/com/intellij/vcs/log/data/CommitDetailsGetter.java
+4 -1
View File
@@ -3,6 +3,7 @@ package com.intellij.vcs.log.data;
import com.intellij.openapi.Disposable;
import com.intellij.openapi.vcs.VcsException;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.vcs.log.VcsFullCommitDetails;
import com.intellij.vcs.log.VcsLogProvider;
import org.jetbrains.annotations.NotNull;
@@ -32,6 +33,8 @@ public class CommitDetailsGetter extends AbstractDataGetter<VcsFullCommitDetails
@Override
protected List<? extends VcsFullCommitDetails> readDetails(@NotNull VcsLogProvider logProvider, @NotNull VirtualFile root,
@NotNull List<String> hashes) throws VcsException {
return logProvider.readFullDetails(root, hashes);
List<VcsFullCommitDetails> result = ContainerUtil.newArrayList();
logProvider.readFullDetails(root, hashes, result::add);
return result;
}
}
platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogPersistentIndex.java
+2 -1
View File
@@ -533,7 +533,8 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
private boolean indexOneByOne(@NotNull VirtualFile root, @NotNull TIntHashSet commits) {
VcsLogProvider provider = myProviders.get(root);
try {
storeDetails(provider.readFullDetails(root, TroveUtil.map(commits, value -> myHashMap.getCommitId(value).getHash().asString())));
List<String> hashes = TroveUtil.map(commits, value -> myHashMap.getCommitId(value).getHash().asString());
provider.readFullDetails(root, hashes, details -> storeDetails(Collections.singletonList(details)));
}
catch (VcsException e) {
LOG.error(e);
platform/vcs-log/impl/test/com/intellij/vcs/log/impl/TestVcsLogProvider.java
+7 -6
View File
@@ -126,7 +126,14 @@ public class TestVcsLogProvider implements VcsLogProvider {
@Override
public void readAllFullDetails(@NotNull VirtualFile root, @NotNull Consumer<VcsFullCommitDetails> commitConsumer) throws VcsException {
throw new UnsupportedOperationException();
}
@Override
public void readFullDetails(@NotNull VirtualFile root,
@Nullable List<String> hashes,
@NotNull Consumer<VcsFullCommitDetails> commitConsumer) throws VcsException {
throw new UnsupportedOperationException();
}
private void assertRoot(@NotNull VirtualFile root) {
@@ -140,12 +147,6 @@ public class TestVcsLogProvider implements VcsLogProvider {
throw new UnsupportedOperationException();
}
@NotNull
@Override
public List<? extends VcsFullCommitDetails> readFullDetails(@NotNull VirtualFile root, @NotNull List<String> hashes) throws VcsException {
throw new UnsupportedOperationException();
}
@NotNull
@Override
public VcsKey getSupportedVcs() {
plugins/git4idea/src/git4idea/history/GitHistoryUtils.java
+35 -32
View File
@@ -635,25 +635,6 @@ public class GitHistoryUtils {
}
}
/*
Unlike loadDetails, which accepts list of hashes in parameters, loads details for all commits in the repository.
To optimize memory consumption, git log command output is parsed on-the-fly and resulting commits are immediately fed to the consumer
and not stored in memory.
*/
public static void loadAllDetails(@NotNull Project project,
@NotNull VirtualFile root,
@NotNull Consumer<VcsFullCommitDetails> commitConsumer) throws VcsException {
final VcsLogObjectsFactory factory = getObjectsFactoryWithDisposeCheck(project);
if (factory == null) {
return;
}
GitLineHandler h = new GitLineHandler(project, root, GitCommand.LOG);
GitLogParser parser = createParserForDetails(h, project, false, true, ArrayUtil.toStringArray(LOG_ALL));
processHandlerOutputByLine(h, parser, record -> commitConsumer.consume(createCommit(project, root, record, factory)));
}
public static void readCommits(@NotNull Project project,
@NotNull VirtualFile root,
@NotNull List<String> parameters,
@@ -877,7 +858,7 @@ public class GitHistoryUtils {
}
final Set<VcsRef> refs = new OpenTHashSet<>(GitLogProvider.DONT_CONSIDER_SHA);
final List<VcsCommitMetadata> commits =
loadDetails(project, root, true, false, record -> {
collectDetails(project, root, true, false, record -> {
GitCommit commit = createCommit(project, root, record, factory);
Collection<VcsRef> refsInRecord = parseRefs(record.getRefs(), commit.getId(), factory, root);
for (VcsRef ref : refsInRecord) {
@@ -904,7 +885,7 @@ public class GitHistoryUtils {
if (factory == null) {
return Collections.emptyList();
}
return loadDetails(project, root, false, true, record -> createCommit(project, root, record, factory), parameters);
return collectDetails(project, root, false, true, record -> createCommit(project, root, record, factory), parameters);
}
@NotNull
@@ -937,26 +918,48 @@ public class GitHistoryUtils {
}
@NotNull
public static <T> List<T> loadDetails(@NotNull final Project project,
@NotNull final VirtualFile root,
boolean withRefs,
boolean withChanges,
@NotNull NullableFunction<GitLogRecord, T> converter,
String... parameters)
public static <T> List<T> collectDetails(@NotNull Project project,
@NotNull VirtualFile root,
boolean withRefs,
boolean withChanges,
@NotNull NullableFunction<GitLogRecord, T> converter,
String... parameters) throws VcsException {
List<T> commits = ContainerUtil.newArrayList();
loadDetails(project, root, withRefs, withChanges, record -> commits.add(converter.fun(record)), parameters);
return commits;
}
public static void loadDetails(@NotNull Project project,
@NotNull VirtualFile root,
@NotNull Consumer<VcsFullCommitDetails> commitConsumer,
@NotNull String... parameters) throws VcsException {
final VcsLogObjectsFactory factory = getObjectsFactoryWithDisposeCheck(project);
if (factory == null) {
return;
}
loadDetails(project, root, false, true, record -> commitConsumer.consume(createCommit(project, root, record, factory)), parameters);
}
public static void loadDetails(@NotNull Project project,
@NotNull VirtualFile root,
boolean withRefs,
boolean withChanges,
@NotNull Consumer<GitLogRecord> converter,
String... parameters)
throws VcsException {
GitLineHandler h = new GitLineHandler(project, root, GitCommand.LOG);
GitLogParser parser = createParserForDetails(h, project, withRefs, withChanges, parameters);
List<T> commits = ContainerUtil.newArrayList();
StopWatch sw = StopWatch.start("loading details");
processHandlerOutputByLine(h, parser, record -> commits.add(converter.fun(record)));
processHandlerOutputByLine(h, parser, record -> converter.consume(record));
sw.report();
return commits;
}
@NotNull
plugins/git4idea/src/git4idea/log/GitLogProvider.java
+21 -22
View File
@@ -319,7 +319,25 @@ public class GitLogProvider implements VcsLogProvider {
return;
}
GitHistoryUtils.loadAllDetails(myProject, root, commitConsumer);
GitHistoryUtils.loadDetails(myProject, root, commitConsumer, ArrayUtil.toStringArray(GitHistoryUtils.LOG_ALL));
}
@Override
public void readFullDetails(@NotNull VirtualFile root,
@NotNull List<String> hashes,
@NotNull Consumer<VcsFullCommitDetails> commitConsumer) throws VcsException {
if (!isRepositoryReady(root)) {
return;
}
VcsFileUtil
.foreachChunk(hashes, 1, hashesChunk -> {
String noWalk = GitVersionSpecialty.NO_WALK_UNSORTED.existsIn(myVcs.getVersion()) ? "--no-walk=unsorted" : "--no-walk";
List<String> parameters = new ArrayList<>();
parameters.add(noWalk);
parameters.addAll(hashesChunk);
GitHistoryUtils.loadDetails(myProject, root, commitConsumer, ArrayUtil.toStringArray(parameters));
});
}
@NotNull
@@ -337,26 +355,6 @@ public class GitLogProvider implements VcsLogProvider {
});
}
@NotNull
@Override
public List<? extends VcsFullCommitDetails> readFullDetails(@NotNull final VirtualFile root, @NotNull List<String> hashes)
throws VcsException {
//noinspection Convert2Lambda
return VcsFileUtil
.foreachChunk(hashes, new ThrowableNotNullFunction<List<String>, List<? extends VcsFullCommitDetails>, VcsException>() {
@NotNull
@Override
public List<? extends VcsFullCommitDetails> fun(@NotNull List<String> hashes) throws VcsException {
String noWalk = GitVersionSpecialty.NO_WALK_UNSORTED.existsIn(myVcs.getVersion()) ? "--no-walk=unsorted" : "--no-walk";
List<String> params = new ArrayList<>();
params.add(noWalk);
params.addAll(hashes);
return GitHistoryUtils.history(myProject, root, ArrayUtil.toStringArray(params));
}
});
}
@NotNull
private Set<VcsRef> readBranches(@NotNull GitRepository repository) {
StopWatch sw = StopWatch.start("readBranches in " + repository.getRoot().getName());
@@ -450,7 +448,8 @@ public class GitLogProvider implements VcsLogProvider {
List<String> authors = ContainerUtil.map(filterCollection.getUserFilter().getUserNames(root), UserNameRegex.BASIC_INSTANCE);
if (GitVersionSpecialty.LOG_AUTHOR_FILTER_SUPPORTS_VERTICAL_BAR.existsIn(myVcs.getVersion())) {
filterParameters.add(prepareParameter("author", StringUtil.join(authors, "\\|")));
} else {
}
else {
filterParameters.addAll(authors.stream().map(a -> prepareParameter("author", a)).collect(Collectors.toList()));
}
}
plugins/git4idea/src/git4idea/reset/GitUncommitAction.java
+3 -1
View File
@@ -218,7 +218,9 @@ public class GitUncommitAction extends DumbAwareAction {
VirtualFile root = commit.getRoot();
VcsFullCommitDetails details = getChangesFromCache(data, hash, root);
if (details == null) {
details = data.getLogProvider(root).readFullDetails(root, singletonList(hash.asString())).get(0);
Ref<VcsFullCommitDetails> ref = new Ref<>();
data.getLogProvider(root).readFullDetails(root, singletonList(hash.asString()), ref::set);
details = ref.get();
}
return details.getChanges();
}
plugins/git4idea/tests/git4idea/history/GitHistoryUtilsTest.java
+2 -2
View File
@@ -399,7 +399,7 @@ public class GitHistoryUtilsTest extends GitSingleRepoTest {
touch("file.txt", "content");
addCommit(message);
GitHistoryUtils.loadAllDetails(myProject, myRepo.getRoot(), details::add);
GitHistoryUtils.loadDetails(myProject, myRepo.getRoot(), details::add);
VcsFullCommitDetails lastCommit = ContainerUtil.getFirstItem(details);
assertNotNull(lastCommit);
@@ -423,7 +423,7 @@ public class GitHistoryUtilsTest extends GitSingleRepoTest {
expected = ContainerUtil.reverse(expected);
List<String> actualMessages =
GitHistoryUtils.loadDetails(myProject, myRepo.getRoot(), true, false, GitLogRecord::getHash, "--max-count=" + commitCount);
GitHistoryUtils.collectDetails(myProject, myRepo.getRoot(), true, false, GitLogRecord::getHash, "--max-count=" + commitCount);
assertEquals(expected, actualMessages);
}
plugins/git4idea/tests/git4idea/log/GitLogProviderTest.java
+3 -5
View File
@@ -16,13 +16,10 @@
package git4idea.log;
import com.intellij.openapi.components.ServiceManager;
import com.intellij.openapi.util.Condition;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.openapi.vcs.VcsException;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.util.ArrayUtil;
import com.intellij.util.CollectConsumer;
import com.intellij.util.Consumer;
import com.intellij.util.Function;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.vcs.log.*;
@@ -179,12 +176,13 @@ public class GitLogProviderTest extends GitSingleRepoTest {
final List<String> hashes = ContainerUtil.newArrayList();
myLogProvider.readAllHashes(myProjectRoot, timedVcsCommit -> hashes.add(timedVcsCommit.getId().asString()));
List<? extends VcsFullCommitDetails> fullDetails = myLogProvider.readFullDetails(myProjectRoot, hashes);
List<VcsFullCommitDetails> result = ContainerUtil.newArrayList();
myLogProvider.readFullDetails(myProjectRoot, hashes, result::add);
// we do not check for changes here
final Function<VcsShortCommitDetails, String> shortDetailsToString = getShortDetailsToString();
Function<VcsCommitMetadata, String> metadataToString = details -> shortDetailsToString.fun(details) + "\n" + details.getFullMessage();
assertOrderedEquals(ContainerUtil.map(fullDetails, metadataToString), ContainerUtil.map(log, metadataToString));
assertOrderedEquals(ContainerUtil.map(result, metadataToString), ContainerUtil.map(log, metadataToString));
}
@NotNull
plugins/hg4idea/src/org/zmlx/hg4idea/log/HgLogProvider.java
+14 -7
View File
@@ -69,9 +69,9 @@ public class HgLogProvider implements VcsLogProvider {
@NotNull
@Override
public DetailedLogData readFirstBlock(@NotNull VirtualFile root,
@NotNull Requirements requirements) throws VcsException {
@NotNull Requirements requirements) throws VcsException {
List<VcsCommitMetadata> commits = HgHistoryUtil.loadMetadata(myProject, root, requirements.getCommitCount(),
Collections.<String>emptyList());
Collections.<String>emptyList());
return new LogDataImpl(readAllRefs(root), commits);
}
@@ -88,7 +88,14 @@ public class HgLogProvider implements VcsLogProvider {
}
@Override
public void readAllFullDetails(@NotNull final VirtualFile root, @NotNull Consumer<VcsFullCommitDetails> commitConsumer)
public void readAllFullDetails(@NotNull VirtualFile root, @NotNull Consumer<VcsFullCommitDetails> commitConsumer) throws VcsException {
readFullDetails(root, ContainerUtil.newArrayList(), commitConsumer);
}
@Override
public void readFullDetails(@NotNull VirtualFile root,
@NotNull List<String> hashes,
@NotNull Consumer<VcsFullCommitDetails> commitConsumer)
throws VcsException {
// this method currently is very slow and time consuming
// so indexing is not to be used for mercurial for now
@@ -97,8 +104,8 @@ public class HgLogProvider implements VcsLogProvider {
final HgVersion version = hgvcs.getVersion();
final String[] templates = HgBaseLogParser.constructFullTemplateArgument(true, version);
HgCommandResult logResult =
HgHistoryUtil.getLogResult(myProject, root, version, -1, ContainerUtil.newArrayList(), HgChangesetUtil.makeTemplate(templates));
HgCommandResult logResult = HgHistoryUtil.getLogResult(myProject, root, version, -1,
HgHistoryUtil.prepareHashes(hashes), HgChangesetUtil.makeTemplate(templates));
if (logResult == null) return;
if (!logResult.getErrorLines().isEmpty()) throw new VcsException(logResult.getRawError());
HgHistoryUtil.createFullCommitsFromResult(myProject, root, logResult, version, false).forEach(commitConsumer::consume);
@@ -148,7 +155,7 @@ public class HgLogProvider implements VcsLogProvider {
for (HgNameWithHashInfo bookmarkInfo : bookmarks) {
refs.add(myVcsObjectsFactory.createRef(bookmarkInfo.getHash(), bookmarkInfo.getName(),
HgRefManager.BOOKMARK, root));
HgRefManager.BOOKMARK, root));
}
String currentRevision = repository.getCurrentRevision();
if (currentRevision != null) { // null => fresh repository
@@ -163,7 +170,7 @@ public class HgLogProvider implements VcsLogProvider {
}
for (HgNameWithHashInfo localTagInfo : localTags) {
refs.add(myVcsObjectsFactory.createRef(localTagInfo.getHash(), localTagInfo.getName(),
HgRefManager.LOCAL_TAG, root));
HgRefManager.LOCAL_TAG, root));
}
for (HgNameWithHashInfo mqPatchRef : mqAppliedPatches) {
refs.add(myVcsObjectsFactory.createRef(mqPatchRef.getHash(), mqPatchRef.getName(),