MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-02-04 23:39:07 +07:00
Code Issues Packages Projects Releases Wiki Activity

[jps caches] WIP add iteration via netty over commits

Browse Source 
GitOrigin-RevId: 0c552cbdc8bf6c6d6fbc2ea0d545aa1c7c197e96
This commit is contained in:
Mikhail Mazurkevich
2021-09-21 23:37:31 +03:00
committed by intellij-monorepo-bot
parent b9b271f510
commit 69ffffa793
13 changed files with 1231 additions and 318 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
java/compiler/impl/src/com/intellij/compiler/cache/JpsCacheStartupActivity.java vendored
View File

@@ -1,18 +1,10 @@
package com.intellij.compiler.cache;
import com.intellij.compiler.CompilerWorkspaceConfiguration;
import com.intellij.compiler.cache.client.JpsServerAuthExtension;
import com.intellij.compiler.server.BuildManager;
import com.intellij.ide.browsers.BrowserLauncher;
import com.intellij.ide.impl.TrustedProjects;
import com.intellij.ide.util.PropertiesComponent;
//import com.intellij.jps.cache.git.GitRepositoryUtil;
import com.intellij.notification.NotificationAction;
import com.intellij.notification.NotificationType;
import com.intellij.compiler.cache.git.GitRepositoryUtil;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.startup.StartupActivity;
import com.intellij.openapi.util.SystemInfo;
//import git4idea.repo.GitRepository;
import com.intellij.openapi.util.registry.Registry;
import org.jetbrains.annotations.NotNull;
//import static com.intellij.jps.cache.JpsCachesPluginUtil.INTELLIJ_REPO_NAME;
@@ -24,11 +16,13 @@ public final class JpsCacheStartupActivity implements StartupActivity.Background
@Override
public void runActivity(@NotNull Project project) {
JpsServerAuthExtension.checkAuthenticatedInBackgroundThread(project, project, () -> {
System.out.println("All Ok");
});
checkWindowsCRLF(project);
checkAutoBuildEnabled(project);
if (Registry.is("compiler.process.use.portable.caches") && GitRepositoryUtil.isIntelliJRepository(project)) {
JpsServerAuthExtension.checkAuthenticatedInBackgroundThread(project, project, () -> {
System.out.println("All Ok");
});
checkWindowsCRLF(project);
checkAutoBuildEnabled(project);
}
}
private static void checkWindowsCRLF(@NotNull Project project) {

84
java/compiler/impl/src/com/intellij/compiler/cache/git/GitCommitsIterator.java vendored
View File

@@ -1,84 +0,0 @@
//package com.intellij.compiler.cache.git;
//
//import com.intellij.openapi.diagnostic.Logger;
//import com.intellij.openapi.project.Project;
//import com.intellij.openapi.vcs.VcsException;
//import com.intellij.util.SmartList;
//import com.intellij.util.containers.ContainerUtil;
//import git4idea.history.GitHistoryUtils;
//import git4idea.repo.GitRepository;
//import org.jetbrains.annotations.NotNull;
//
//import java.util.Iterator;
//import java.util.List;
//import java.util.NoSuchElementException;
//
//public class GitCommitsIterator implements Iterator<String> {
// private static final Logger LOG = Logger.getInstance(GitCommitsIterator.class);
//
// private static final int MAX_FETCH_SIZE = 1000;
// private static final int FETCH_SIZE = 100;
// private final GitRepository myRepository;
// private final Project myProject;
// private int fetchedCount;
// private List<String> commits;
// private String remote;
// private int currentPosition;
//
// public GitCommitsIterator(@NotNull Project project, @NotNull GitRepository repository, @NotNull String remoteUrl) {
// myRepository = repository;
// myProject = project;
// fetchedCount = 0;
// remote = remoteUrl;
// fetchOldCommits();
// }
//
// @Override
// public boolean hasNext() {
// if (commits.size() > 0) {
// if (currentPosition < commits.size()) return true;
// if (fetchedCount >= MAX_FETCH_SIZE) {
// LOG.info("Exceeded fetch limit for git commits");
// return false;
// }
// fetchOldCommits(commits.get(currentPosition - 1));
// if (commits.size() > 0) {
// currentPosition = 0;
// return true;
// }
// }
// return false;
// }
//
// @Override
// public String next() {
// if (commits.size() == 0 || currentPosition >= commits.size()) throw new NoSuchElementException();
// String result = commits.get(currentPosition);
// currentPosition++;
// return result;
// }
//
// @NotNull
// public String getRemote() {
// return remote;
// }
//
// private void fetchOldCommits() {
// fetchOldCommits("");
// }
//
// private void fetchOldCommits(String latestCommit) {
// try {
// commits =
// ContainerUtil.map(latestCommit.isEmpty() ? GitHistoryUtils.collectTimedCommits(myProject, myRepository.getRoot(), "-n " + FETCH_SIZE) :
// GitHistoryUtils.collectTimedCommits(myProject, myRepository.getRoot(), latestCommit, "-n " + FETCH_SIZE),
// it -> it.getId().asString());
// fetchedCount += commits.size();
// return;
// }
// catch (VcsException e) {
// LOG.warn("Can't get Git hashes for commits", e);
// }
// commits = new SmartList<>();
// }
//}

190
java/compiler/impl/src/com/intellij/compiler/cache/git/GitRepositoryUtil.java vendored
View File

@@ -1,70 +1,120 @@
//package com.intellij.jps.cache.git;
//
//import com.intellij.jps.cache.git.GitCommitsIterator;
//import com.intellij.openapi.diagnostic.Logger;
//import com.intellij.openapi.project.Project;
//import com.intellij.util.containers.ContainerUtil;
//import git4idea.GitUtil;
//import git4idea.commands.Git;
//import git4idea.commands.GitCommandResult;
//import git4idea.config.GitConfigUtil;
//import git4idea.repo.GitRemote;
//import git4idea.repo.GitRepository;
//import org.jetbrains.annotations.NotNull;
//import org.jetbrains.annotations.Nullable;
//
//import java.util.*;
//import java.util.stream.Collectors;
//
//public final class GitRepositoryUtil {
// private static final Logger LOG = Logger.getInstance(GitRepositoryUtil.class);
//
// private GitRepositoryUtil() {}
//
// @NotNull
// public static List<GitCommitsIterator> getCommitsIterator(@NotNull Project project, @NotNull Set<String> remoteUrlNames) {
// if (GitUtil.hasGitRepositories(project)) {
// return GitUtil.getRepositories(project).stream()
// .map(repo -> {
// Set<String> remoteUrls = repo.getRemotes().stream()
// .map(remote -> remote.getUrls())
// .flatMap(Collection::stream)
// .collect(Collectors.toSet());
// String matchedRemoteUrl = ContainerUtil.find(remoteUrls, remoteUrl -> remoteUrlNames.contains(getRemoteRepoName(remoteUrl)));
// if (matchedRemoteUrl == null) return null;
// return new GitCommitsIterator(project, repo, getRemoteRepoName(matchedRemoteUrl));
// }).filter(Objects::nonNull)
// .collect(Collectors.toList());
// }
// LOG.info("Project doesn't contain Git repository");
// return Collections.emptyList();
// }
//
// @Nullable
// public static GitRepository getRepositoryByName(@NotNull Project project, @NotNull String repositoryName) {
// if (GitUtil.hasGitRepositories(project)) {
// return ContainerUtil.find(GitUtil.getRepositories(project), repo -> {
// for (GitRemote remote : repo.getRemotes()) {
// for (String remoteUrl : remote.getUrls()) {
// if (getRemoteRepoName(remoteUrl).equals(repositoryName)) return true;
// }
// }
// return false;
// });
// }
// LOG.info("Project doesn't contain Git repository");
// return null;
// }
//
// public static String getRemoteRepoName(@NotNull String remoteUrl) {
// String[] splittedRemoteUrl = remoteUrl.split("/");
// return splittedRemoteUrl[splittedRemoteUrl.length - 1];
// }
//
// public static boolean isAutoCrlfSetRight(@NotNull GitRepository gitRepository) {
// GitCommandResult result = Git.getInstance().config(gitRepository, GitConfigUtil.CORE_AUTOCRLF);
// String value = result.getOutputAsJoinedString();
// LOG.info("CRLF configuration for " + gitRepository + " project: " + value);
// return value.equalsIgnoreCase("input");
// }
//}
package com.intellij.compiler.cache.git;
import com.intellij.execution.ExecutionException;
import com.intellij.execution.configurations.GeneralCommandLine;
import com.intellij.execution.process.CapturingProcessAdapter;
import com.intellij.execution.process.OSProcessHandler;
import com.intellij.execution.process.ProcessEvent;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.project.Project;
import com.intellij.util.containers.ContainerUtil;
import org.jetbrains.annotations.NotNull;
import java.nio.charset.StandardCharsets;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
public final class GitRepositoryUtil {
private static final Logger LOG = Logger.getInstance(GitRepositoryUtil.class);
private static final String INTELLIJ_REPO_NAME = "intellij.git";
private static final int FETCH_SIZE = 100;
private GitRepositoryUtil() {}
public static boolean isIntelliJRepository(@NotNull Project project) {
String projectBasePath = project.getBasePath();
if (projectBasePath == null) return false;
GeneralCommandLine commandLine = new GeneralCommandLine()
.withParentEnvironmentType(GeneralCommandLine.ParentEnvironmentType.CONSOLE)
.withWorkDirectory(projectBasePath)
.withExePath("git")
.withParameters("remote")
.withParameters("-v");
AtomicBoolean result = new AtomicBoolean();
try {
OSProcessHandler handler = new OSProcessHandler(commandLine.withCharset(StandardCharsets.UTF_8));
handler.addProcessListener(new CapturingProcessAdapter() {
@Override
public void processTerminated(@NotNull ProcessEvent event) {
if (event.getExitCode() != 0) {
LOG.warn("Couldn't fetch repository remote URL " + getOutput().getStderr());
} else {
result.set(getOutput().getStdout().contains(INTELLIJ_REPO_NAME));
}
}
});
handler.startNotify();
handler.waitFor();
}
catch (ExecutionException e) {
LOG.warn("Couldn't execute command for fetching remote URL", e);
}
return result.get();
}
public static @NotNull List<String> fetchRepositoryCommits(@NotNull Project project, @NotNull String latestCommit) {
String projectBasePath = project.getBasePath();
if (projectBasePath == null) return Collections.emptyList();
GeneralCommandLine commandLine = new GeneralCommandLine()
.withParentEnvironmentType(GeneralCommandLine.ParentEnvironmentType.CONSOLE)
.withWorkDirectory(projectBasePath)
.withExePath("git")
.withParameters("log")
.withParameters("--format=\"%H\"");
if (!latestCommit.isEmpty()) {
commandLine.addParameter(latestCommit);
}
commandLine.withParameters("-n")
.withParameters(Integer.toString(FETCH_SIZE));
StringBuilder processOutput = new StringBuilder();
try {
OSProcessHandler handler = new OSProcessHandler(commandLine.withCharset(StandardCharsets.UTF_8));
handler.addProcessListener(new CapturingProcessAdapter() {
@Override
public void processTerminated(@NotNull ProcessEvent event) {
if (event.getExitCode() != 0) {
LOG.warn("Couldn't fetch N commits from the current repository " + getOutput().getStderr());
} else {
processOutput.append(getOutput().getStdout());
}
}
});
handler.startNotify();
handler.waitFor();
}
catch (ExecutionException e) {
LOG.warn("Can't execute command for getting commit hashes", e);
}
String result = processOutput.toString();
if (result.isEmpty()) return Collections.emptyList();
return ContainerUtil.map(processOutput.toString().split("\n"), commit -> commit.substring(1, commit.length() - 1));
}
//@Nullable
//public static GitRepository getRepositoryByName(@NotNull Project project, @NotNull String repositoryName) {
// if (GitUtil.hasGitRepositories(project)) {
// return ContainerUtil.find(GitUtil.getRepositories(project), repo -> {
// for (GitRemote remote : repo.getRemotes()) {
// for (String remoteUrl : remote.getUrls()) {
// if (getRemoteRepoName(remoteUrl).equals(repositoryName)) return true;
// }
// }
// return false;
// });
// }
// LOG.info("Project doesn't contain Git repository");
// return null;
//}
//public static boolean isAutoCrlfSetRight(@NotNull GitRepository gitRepository) {
// GitCommandResult result = Git.getInstance().config(gitRepository, GitConfigUtil.CORE_AUTOCRLF);
// String value = result.getOutputAsJoinedString();
// LOG.info("CRLF configuration for " + gitRepository + " project: " + value);
// return value.equalsIgnoreCase("input");
//}
}

3
java/compiler/impl/src/com/intellij/compiler/server/BuildManager.java
View File

@@ -8,6 +8,7 @@ import com.intellij.compiler.CompilerConfiguration;
import com.intellij.compiler.CompilerConfigurationImpl;
import com.intellij.compiler.CompilerWorkspaceConfiguration;
import com.intellij.compiler.YourKitProfilerService;
import com.intellij.compiler.cache.git.GitRepositoryUtil;
import com.intellij.compiler.impl.CompilerUtil;
import com.intellij.compiler.impl.javaCompiler.BackendCompiler;
import com.intellij.compiler.impl.javaCompiler.eclipse.EclipseCompilerConfiguration;
@@ -1405,7 +1406,7 @@ public final class BuildManager implements Disposable {
}
// portable caches
if (Registry.is("compiler.process.use.portable.caches")) {
if (Registry.is("compiler.process.use.portable.caches") && GitRepositoryUtil.isIntelliJRepository(project)) {
//cmdLine.addParameter("-Didea.resizeable.file.truncate.on.close=true");
//cmdLine.addParameter("-Dkotlin.jps.non.caching.storage=true");
cmdLine.addParameter("-D" + ProjectStamps.PORTABLE_CACHES_PROPERTY + "=true");

8
java/compiler/impl/src/com/intellij/compiler/server/DefaultMessageHandler.java
View File

@@ -2,6 +2,7 @@
package com.intellij.compiler.server;
import com.intellij.compiler.cache.client.JpsServerAuthUtil;
import com.intellij.compiler.cache.git.GitRepositoryUtil;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.progress.ProgressIndicator;
import com.intellij.openapi.project.Project;
@@ -10,6 +11,7 @@ import org.jetbrains.annotations.NotNull;
import org.jetbrains.jps.api.CmdlineProtoUtil;
import org.jetbrains.jps.api.CmdlineRemoteProto;
import java.util.List;
import java.util.Map;
import java.util.UUID;
@@ -63,6 +65,12 @@ public abstract class DefaultMessageHandler implements BuilderMessageHandler {
if (cacheDownloadMessage.getDescriptionText() != null) {
progressIndicator.setText(cacheDownloadMessage.getDescriptionText());
}
break;
case REPOSITORY_COMMITS_REQUEST:
CmdlineRemoteProto.Message.BuilderMessage.LatestCommitMessage latestCommitMessage = msg.getLatestCommitMessage();
List<String> repositoryCommits = GitRepositoryUtil.fetchRepositoryCommits(myProject, latestCommitMessage.getLatestCommit());
channel.writeAndFlush(CmdlineProtoUtil.toMessage(sessionId, CmdlineProtoUtil.createRepositoryCommitsMessage(repositoryCommits)));
break;
case CONSTANT_SEARCH_TASK:
// ignored, because the functionality is deprecated

947
jps/jps-builders/gen/org/jetbrains/jps/api/CmdlineRemoteProto.java
View File

File diff suppressed because it is too large Load Diff

12
jps/jps-builders/proto/cmdline_remote_proto.proto
View File

@@ -35,6 +35,7 @@ message Message {
FS_EVENT = 3;
CONSTANT_SEARCH_RESULT = 4;
AUTHENTICATION_TOKEN = 5;
REPOSITORY_COMMITS_RESULT = 6;
}
message FSEvent {
@@ -80,11 +81,16 @@ message Message {
map<string, string> auth_headers = 1;
}
message RepositoryCommitsResult {
repeated string commit = 1;
}
required Type type = 1;
optional ParametersMessage params_message = 2;
optional FSEvent fs_event = 3;
optional ConstantSearchResult constant_search_result = 4;
optional RequestParams request_params = 5;
optional RepositoryCommitsResult repository_commits_result = 6;
}
message BuilderMessage {
@@ -95,6 +101,7 @@ message Message {
CONSTANT_SEARCH_TASK = 4;
AUTH_TOKEN_REQUEST = 5;
CACHE_DOWNLOAD_MESSAGE = 6;
REPOSITORY_COMMITS_REQUEST = 7;
}
message BuildEvent {
@@ -164,11 +171,16 @@ message Message {
optional string description_text = 2;
}
message LatestCommitMessage {
required string latest_commit = 1;
}
required Type type = 1;
optional BuildEvent build_event = 2;
optional CompileMessage compile_message = 3;
optional ConstantSearchTask constant_search_task = 4;
optional CacheDownloadMessage cache_download_message = 5;
optional LatestCommitMessage latest_commit_message = 6;
}
required UUID session_id = 1;

17
jps/jps-builders/src/org/jetbrains/jps/api/CmdlineProtoUtil.java
View File

@@ -130,6 +130,23 @@ public final class CmdlineProtoUtil {
.setRequestParams(requestParam).build();
}
public static BuilderMessage createRequestRepositoryCommits(@NotNull String latestCommit) {
BuilderMessage.LatestCommitMessage.Builder requestCommitsBuilder = BuilderMessage.LatestCommitMessage.newBuilder();
requestCommitsBuilder.setLatestCommit(latestCommit);
BuilderMessage.Builder newBuilder = BuilderMessage.newBuilder();
newBuilder.setType(BuilderMessage.Type.REPOSITORY_COMMITS_REQUEST);
newBuilder.setLatestCommitMessage(requestCommitsBuilder.build());
return newBuilder.build();
}
public static CmdlineRemoteProto.Message.ControllerMessage createRepositoryCommitsMessage(@NotNull List<String> commits) {
CmdlineRemoteProto.Message.ControllerMessage.RepositoryCommitsResult repositoryCommitsResult =
CmdlineRemoteProto.Message.ControllerMessage.RepositoryCommitsResult.newBuilder().addAllCommit(commits).build();
return CmdlineRemoteProto.Message.ControllerMessage.newBuilder()
.setType(CmdlineRemoteProto.Message.ControllerMessage.Type.REPOSITORY_COMMITS_RESULT)
.setRepositoryCommitsResult(repositoryCommitsResult).build();
}
public static BuilderMessage createCacheDownloadMessage(String text) {
BuilderMessage.CacheDownloadMessage.Builder cacheDownloadMessageBuilder = BuilderMessage.CacheDownloadMessage.newBuilder();
cacheDownloadMessageBuilder.setMainText(text);

15
jps/jps-builders/src/org/jetbrains/jps/cache/client/JpsNettyClient.java vendored
View File

@@ -1,14 +1,17 @@
// Copyright 2000-2021 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package org.jetbrains.jps.cache.client;
import com.intellij.openapi.diagnostic.Logger;
import io.netty.channel.Channel;
import io.netty.channel.ChannelFuture;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.jps.api.CmdlineProtoUtil;
import org.jetbrains.jps.cache.loader.JpsOutputLoaderManager;
import java.util.UUID;
public class JpsNettyClient {
private static final Logger LOG = Logger.getInstance(JpsNettyClient.class);
private final UUID sessionId;
private final Channel channel;
@@ -25,13 +28,23 @@ public class JpsNettyClient {
channel.writeAndFlush(CmdlineProtoUtil.toMessage(sessionId, CmdlineProtoUtil.createCacheDownloadMessage2(message)));
}
public void requestRepositoryCommits(@NotNull String latestCommit) {
try {
ChannelFuture channelFuture = channel.writeAndFlush(CmdlineProtoUtil.toMessage(sessionId, CmdlineProtoUtil.createRequestRepositoryCommits(latestCommit)));
channelFuture.await();
}
catch (InterruptedException e) {
LOG.warn("Can't request repository commits", e);
}
}
public void requestAuthToken() {
try {
ChannelFuture channelFuture = channel.writeAndFlush(CmdlineProtoUtil.toMessage(sessionId, CmdlineProtoUtil.createAuthTokenRequest()));
channelFuture.await();
}
catch (InterruptedException e) {
e.printStackTrace();
LOG.warn("Can't request authentication token", e);
}
}
}

157
jps/jps-builders/src/org/jetbrains/jps/cache/git/GitCommitsIterator.java vendored
View File

@@ -1,84 +1,73 @@
//package org.jetbrains.jps.cache.git;
//
//import com.intellij.openapi.diagnostic.Logger;
//import com.intellij.openapi.project.Project;
//import com.intellij.openapi.vcs.VcsException;
//import com.intellij.util.SmartList;
//import com.intellij.util.containers.ContainerUtil;
//import git4idea.history.GitHistoryUtils;
//import git4idea.repo.GitRepository;
//import org.jetbrains.annotations.NotNull;
//
//import java.util.Iterator;
//import java.util.List;
//import java.util.NoSuchElementException;
//
//public class GitCommitsIterator implements Iterator<String> {
// private static final Logger LOG = Logger.getInstance(GitCommitsIterator.class);
//
// private static final int MAX_FETCH_SIZE = 1000;
// private static final int FETCH_SIZE = 100;
// private final GitRepository myRepository;
// private final Project myProject;
// private int fetchedCount;
// private List<String> commits;
// private String remote;
// private int currentPosition;
//
// public GitCommitsIterator(@NotNull Project project, @NotNull GitRepository repository, @NotNull String remoteUrl) {
// myRepository = repository;
// myProject = project;
// fetchedCount = 0;
// remote = remoteUrl;
// fetchOldCommits();
// }
//
// @Override
// public boolean hasNext() {
// if (commits.size() > 0) {
// if (currentPosition < commits.size()) return true;
// if (fetchedCount >= MAX_FETCH_SIZE) {
// LOG.info("Exceeded fetch limit for git commits");
// return false;
// }
// fetchOldCommits(commits.get(currentPosition - 1));
// if (commits.size() > 0) {
// currentPosition = 0;
// return true;
// }
// }
// return false;
// }
//
// @Override
// public String next() {
// if (commits.size() == 0 || currentPosition >= commits.size()) throw new NoSuchElementException();
// String result = commits.get(currentPosition);
// currentPosition++;
// return result;
// }
//
// @NotNull
// public String getRemote() {
// return remote;
// }
//
// private void fetchOldCommits() {
// fetchOldCommits("");
// }
//
// private void fetchOldCommits(String latestCommit) {
// try {
// commits =
// ContainerUtil.map(latestCommit.isEmpty() ? GitHistoryUtils.collectTimedCommits(myProject, myRepository.getRoot(), "-n " + FETCH_SIZE) :
// GitHistoryUtils.collectTimedCommits(myProject, myRepository.getRoot(), latestCommit, "-n " + FETCH_SIZE),
// it -> it.getId().asString());
// fetchedCount += commits.size();
// return;
// }
// catch (VcsException e) {
// LOG.warn("Can't get Git hashes for commits", e);
// }
// commits = new SmartList<>();
// }
//}
package org.jetbrains.jps.cache.git;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.util.SmartList;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.jps.cache.client.JpsNettyClient;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
public class GitCommitsIterator implements Iterator<String> {
private static final Logger LOG = Logger.getInstance(GitCommitsIterator.class);
private static final int MAX_FETCH_SIZE = 1000;
private final JpsNettyClient myNettyClient;
private static List<String> repositoryCommits;
private static int fetchedCount;
private int currentPosition;
private String remote;
public GitCommitsIterator(@NotNull JpsNettyClient nettyClient, @NotNull String remoteUrl) {
myNettyClient = nettyClient;
fetchedCount = 0;
remote = remoteUrl;
fetchOldCommits();
}
@Override
public boolean hasNext() {
if (repositoryCommits.size() > 0) {
if (currentPosition < repositoryCommits.size()) return true;
if (fetchedCount >= MAX_FETCH_SIZE) {
LOG.info("Exceeded fetch limit for git commits");
return false;
}
fetchOldCommits(repositoryCommits.get(currentPosition - 1));
if (repositoryCommits.size() > 0) {
currentPosition = 0;
return true;
}
}
return false;
}
@Override
public String next() {
if (repositoryCommits.size() == 0 || currentPosition >= repositoryCommits.size()) throw new NoSuchElementException();
String result = repositoryCommits.get(currentPosition);
currentPosition++;
return result;
}
@NotNull
public String getRemote() {
return remote;
}
private void fetchOldCommits() {
fetchOldCommits("");
}
private void fetchOldCommits(String latestCommit) {
// TODO:: FIX awaiting
myNettyClient.requestRepositoryCommits(latestCommit);
repositoryCommits = new SmartList<>();
}
public static void setRepositoryCommits(List<String> commits) {
repositoryCommits = commits;
fetchedCount += repositoryCommits.size();
}
}

83
jps/jps-builders/src/org/jetbrains/jps/cache/loader/JpsOutputLoaderManager.java vendored
View File

@@ -10,11 +10,10 @@ import com.intellij.util.containers.ContainerUtil;
import io.netty.channel.Channel;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.jps.api.CmdlineProtoUtil;
import org.jetbrains.jps.api.CmdlineRemoteProto;
import org.jetbrains.jps.builders.JpsBuildBundle;
import org.jetbrains.jps.cache.client.JpsNettyClient;
import org.jetbrains.jps.cache.client.JpsServerClient;
import org.jetbrains.jps.cache.git.GitCommitsIterator;
import org.jetbrains.jps.cache.loader.JpsOutputLoader.LoaderStatus;
import org.jetbrains.jps.cache.model.BuildTargetState;
import org.jetbrains.jps.cache.model.JpsLoaderContext;
@@ -56,7 +55,7 @@ public class JpsOutputLoaderManager implements Disposable {
public JpsOutputLoaderManager(@NotNull JpsProject project,
@NotNull String projectPath,
@NotNull Channel channel,
@NotNull UUID sessionId) throws InterruptedException {
@NotNull UUID sessionId) {
myNettyClient = new JpsNettyClient(channel, sessionId);
myProjectPath = projectPath;
hasRunningTask = new AtomicBoolean();
@@ -121,67 +120,35 @@ public class JpsOutputLoaderManager implements Disposable {
// ProgressManager.getInstance().runProcessWithProgressAsynchronously(task, processIndicator);
//}
//public void notifyAboutNearestCache() {
// Pair<String, Integer> commitInfo = getNearestCommit(false, false);
// if (commitInfo == null) return;
//
// String notificationContent = commitInfo.second == 1
// ? JpsCacheBundle.message("notification.content.caches.are.for.current.commit")
// : JpsCacheBundle
// .message("notification.content.caches.are.for.commit.commits.prior.to.yours", commitInfo.second - 1);
//
// ApplicationManager.getApplication().invokeLater(() -> {
// STANDARD
// .createNotification(JpsCacheBundle.message("notification.title.compiler.caches.available"), notificationContent,
// NotificationType.INFORMATION)
// .addAction(NotificationAction.createSimpleExpiring(
// JpsCacheBundle.message("action.NotificationAction.JpsOutputLoaderManager.text.update.caches"),
// () -> {
// DOWNLOAD_THROUGH_NOTIFICATION_EVENT_ID.log();
// load(false, false);
// }))
// .notify(myProject);
// });
//}
@Nullable
private Pair<String, Integer> getNearestCommit(boolean isForceUpdate, boolean verbose) {
Map<String, Set<String>> availableCommitsPerRemote = myServerClient.getCacheKeysPerRemote();
String previousCommitId = null;
//String previousCommitId = PropertiesComponent.getInstance().getValue(LATEST_COMMIT_ID);
//List<GitCommitsIterator> repositoryList = GitRepositoryUtil.getCommitsIterator(myProject, availableCommitsPerRemote.keySet());
//String commitId = "";
//int commitsBehind = 0;
//Set<String> availableCommitsForRemote = new HashSet<>();
//for (GitCommitsIterator commitsIterator : repositoryList) {
// availableCommitsForRemote = availableCommitsPerRemote.get(commitsIterator.getRemote());
// if (availableCommitsForRemote.contains(commitId)) continue;
// commitsBehind = 0;
// while (commitsIterator.hasNext() && !availableCommitsForRemote.contains(commitId)) {
// commitId = commitsIterator.next();
// commitsBehind++;
// }
//}
//var group = verbose ? STANDARD : EVENT_LOG;
//if (!availableCommitsForRemote.contains(commitId)) {
// String warning = JpsCacheBundle.message("notification.content.not.found.any.caches.for.latest.commits.in.branch");
// LOG.warn(warning);
// ApplicationManager.getApplication().invokeLater(() -> {
// group.createNotification(JpsCacheBundle.message("notification.title.jps.caches.downloader"), warning, NotificationType.WARNING)
// .notify(myProject);
// });
// return null;
//}
//if (previousCommitId != null && commitId.equals(previousCommitId) && !isForceUpdate) {
// String info = JpsCacheBundle.message("notification.content.system.contains.up.to.date.caches");
// LOG.info(info);
// ApplicationManager.getApplication().invokeLater(() -> {
// group.createNotification(JpsCacheBundle.message("notification.title.jps.caches.downloader"), info, NotificationType.INFORMATION)
// .notify(myProject);
// });
// return null;
//}
return Pair.create(availableCommitsPerRemote.get(INTELLIJ_REPO_NAME).stream().findFirst().get(), 500);
String commitId = "";
GitCommitsIterator commitsIterator = new GitCommitsIterator(myNettyClient, INTELLIJ_REPO_NAME);
Set<String> availableCommitsForRemote = availableCommitsPerRemote.get(commitsIterator.getRemote());
int commitsBehind = 0;
while (commitsIterator.hasNext() && !availableCommitsForRemote.contains(commitId)) {
commitId = commitsIterator.next();
commitsBehind++;
}
if (!availableCommitsForRemote.contains(commitId)) {
String message = JpsBuildBundle.message("notification.content.not.found.any.caches.for.latest.commits.in.branch");
LOG.warn(message);
myNettyClient.sendMainStatusMessage(message);
return null;
}
if (previousCommitId != null && commitId.equals(previousCommitId) && !isForceUpdate) {
String message = JpsBuildBundle.message("notification.content.system.contains.up.to.date.caches");
LOG.info(message);
myNettyClient.sendMainStatusMessage(message);
return null;
}
return Pair.create(commitId, commitsBehind);
}
private void startLoadingForCommit(@NotNull String commitId) {

7
jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildMain.java
View File

@@ -20,6 +20,7 @@ import org.jetbrains.jps.builders.BuildTarget;
import org.jetbrains.jps.builders.JpsBuildBundle;
import org.jetbrains.jps.builders.PreloadedDataExtension;
import org.jetbrains.jps.cache.client.JpsServerAuthUtil;
import org.jetbrains.jps.cache.git.GitCommitsIterator;
import org.jetbrains.jps.incremental.BuilderRegistry;
import org.jetbrains.jps.incremental.MessageHandler;
import org.jetbrains.jps.incremental.Utils;
@@ -246,6 +247,12 @@ public final class BuildMain {
JpsServerAuthUtil.setRequestHeaders(requestParams.getAuthHeadersMap());
return;
}
case REPOSITORY_COMMITS_RESULT: {
CmdlineRemoteProto.Message.ControllerMessage.RepositoryCommitsResult repositoryCommitsResult = controllerMessage.getRepositoryCommitsResult();
System.out.println("Got repository commits: " + repositoryCommitsResult.getCommitList());
GitCommitsIterator.setRepositoryCommits(repositoryCommitsResult.getCommitList());
return;
}
case CONSTANT_SEARCH_RESULT: {
// ignored, functionality deprecated
return;

2
platform/util/resources/misc/registry.properties
View File

@@ -670,7 +670,7 @@ compiler.natural.int.multimap.impl=false
compiler.natural.int.multimap.impl.description=Experimental implementation for class-to-class dependency multimap in JPS caches.\nIf turned off after being turned on, project rebuild is required.
compiler.process.use.portable.caches=false
compiler.process.use.portable.caches.description=Turn on download compilation caches for reducing build time.
compiler.process.use.portable.caches.description=Warning: for Intellij project only! \nTurn on to download compilation caches for reducing build time.
vcs.showConsole=true
vcs.showConsole.description=Show 'Console' tab in VCS toolwindow that logs all write-commands performed by IDE.