From 7da2e4d6a983b6b5be492dce01082b7c8a7bf6f1 Mon Sep 17 00:00:00 2001 From: Mikhail Mazurkevich Date: Sat, 18 Sep 2021 00:15:56 +0300 Subject: [PATCH] [jps caches] WIP on move to JPS GitOrigin-RevId: eaa1fdc8c97c09f7b70d043463509d845de1875d --- .../cache/JpsCacheStartupActivity.java | 80 +++ .../cache/client/JpsServerAuthExtension.java | 81 +++ .../cache/client/JpsServerAuthUtil.java | 22 + .../cache/git/GitCommitsIterator.java | 84 +++ .../compiler/cache/git/GitRepositoryUtil.java | 70 +++ .../cache/ui/JpsLoaderNotifications.java | 13 + .../compiler/server/BuildManager.java | 14 +- .../server/DefaultMessageHandler.java | 8 + .../messages/JavaCompilerBundle.properties | 4 + java/java-impl/src/META-INF/JavaPlugin.xml | 2 + .../jetbrains/jps/api/CmdlineRemoteProto.java | 536 +++++++++++++++++- .../proto/cmdline_remote_proto.proto | 7 + .../jetbrains/jps/api/CmdlineProtoUtil.java | 12 + .../jps/cache/JpsCacheStartupActivity.java | 76 +++ .../jps/cache/JpsCachesPluginUtil.java | 23 + .../action/JpsForceUpdateCachesAction.java | 18 + .../cache/action/JpsUpdateCachesAction.java | 18 + .../jps/cache/client/JpsCachesDownloader.java | 253 +++++++++ .../cache/client/JpsServerAuthExtension.java | 79 +++ .../jps/cache/client/JpsServerAuthUtil.java | 25 + .../jps/cache/client/JpsServerClient.java | 26 + .../jps/cache/client/JpsServerClientImpl.java | 191 +++++++ .../cache/client/JpsServerConnectionUtil.java | 128 +++++ .../jps/cache/git/GitCommitsIterator.java | 84 +++ .../jps/cache/git/GitRepositoryUtil.java | 70 +++ .../jps/cache/loader/JpsCacheLoader.java | 137 +++++ .../loader/JpsCompilationOutputLoader.java | 357 ++++++++++++ .../jps/cache/loader/JpsMetadataLoader.java | 80 +++ .../jps/cache/loader/JpsOutputLoader.java | 24 + .../cache/loader/JpsOutputLoaderManager.java | 383 +++++++++++++ .../jps/cache/model/AffectedModule.java | 57 ++ .../jps/cache/model/BuildTargetState.java | 38 ++ .../jps/cache/model/DownloadableFileUrl.java | 60 ++ .../jps/cache/model/JpsLoaderContext.java | 49 ++ .../jps/cache/model/OutputLoadResult.java | 27 + .../statistics/JpsCacheUsagesCollector.java | 21 + .../jps/cache/ui/JpsLoaderNotifications.java | 13 + .../ui/SegmentedProgressIndicatorManager.java | 129 +++++ .../org/jetbrains/jps/cmdline/BuildMain.java | 6 +- .../jetbrains/jps/cmdline/BuildSession.java | 6 + .../util/resources/misc/registry.properties | 3 + .../jps-cache/resources/META-INF/plugin.xml | 6 +- .../cache/loader/JpsOutputLoaderManager.java | 2 +- 43 files changed, 3302 insertions(+), 20 deletions(-) create mode 100644 java/compiler/impl/src/com/intellij/compiler/cache/JpsCacheStartupActivity.java create mode 100644 java/compiler/impl/src/com/intellij/compiler/cache/client/JpsServerAuthExtension.java create mode 100644 java/compiler/impl/src/com/intellij/compiler/cache/client/JpsServerAuthUtil.java create mode 100644 java/compiler/impl/src/com/intellij/compiler/cache/git/GitCommitsIterator.java create mode 100644 java/compiler/impl/src/com/intellij/compiler/cache/git/GitRepositoryUtil.java create mode 100644 java/compiler/impl/src/com/intellij/compiler/cache/ui/JpsLoaderNotifications.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/cache/JpsCacheStartupActivity.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/cache/JpsCachesPluginUtil.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/cache/action/JpsForceUpdateCachesAction.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/cache/action/JpsUpdateCachesAction.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/cache/client/JpsCachesDownloader.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/cache/client/JpsServerAuthExtension.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/cache/client/JpsServerAuthUtil.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/cache/client/JpsServerClient.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/cache/client/JpsServerClientImpl.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/cache/client/JpsServerConnectionUtil.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/cache/git/GitCommitsIterator.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/cache/git/GitRepositoryUtil.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/cache/loader/JpsCacheLoader.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/cache/loader/JpsCompilationOutputLoader.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/cache/loader/JpsMetadataLoader.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/cache/loader/JpsOutputLoader.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/cache/loader/JpsOutputLoaderManager.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/cache/model/AffectedModule.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/cache/model/BuildTargetState.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/cache/model/DownloadableFileUrl.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/cache/model/JpsLoaderContext.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/cache/model/OutputLoadResult.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/cache/statistics/JpsCacheUsagesCollector.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/cache/ui/JpsLoaderNotifications.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/cache/ui/SegmentedProgressIndicatorManager.java diff --git a/java/compiler/impl/src/com/intellij/compiler/cache/JpsCacheStartupActivity.java b/java/compiler/impl/src/com/intellij/compiler/cache/JpsCacheStartupActivity.java new file mode 100644 index 000000000000..2e6fa6472ea3 --- /dev/null +++ b/java/compiler/impl/src/com/intellij/compiler/cache/JpsCacheStartupActivity.java @@ -0,0 +1,80 @@ +package com.intellij.compiler.cache; + +import com.intellij.compiler.CompilerWorkspaceConfiguration; +import com.intellij.compiler.cache.client.JpsServerAuthExtension; +import com.intellij.compiler.server.BuildManager; +import com.intellij.ide.browsers.BrowserLauncher; +import com.intellij.ide.impl.TrustedProjects; +import com.intellij.ide.util.PropertiesComponent; +//import com.intellij.jps.cache.git.GitRepositoryUtil; +import com.intellij.notification.NotificationAction; +import com.intellij.notification.NotificationType; +import com.intellij.openapi.project.Project; +import com.intellij.openapi.startup.StartupActivity; +import com.intellij.openapi.util.SystemInfo; +//import git4idea.repo.GitRepository; +import org.jetbrains.annotations.NotNull; + +//import static com.intellij.jps.cache.JpsCachesPluginUtil.INTELLIJ_REPO_NAME; +//import static com.intellij.jps.cache.ui.JpsLoaderNotifications.ATTENTION; + +public final class JpsCacheStartupActivity implements StartupActivity.Background { + private static final String NOT_ASK_AGAIN = "JpsCaches.NOT_ASK_AGAIN"; + private static boolean lineEndingsConfiguredCorrectly = true; + + @Override + public void runActivity(@NotNull Project project) { + JpsServerAuthExtension.checkAuthenticatedInBackgroundThread(project, project, () -> { + System.out.println("All Ok"); + }); + checkWindowsCRLF(project); + checkAutoBuildEnabled(project); + } + + private static void checkWindowsCRLF(@NotNull Project project) { + //if (!SystemInfo.isWindows) return; + //GitRepository intellijRepository = GitRepositoryUtil.getRepositoryByName(project, INTELLIJ_REPO_NAME); + //if (intellijRepository == null) return; + //if (!GitRepositoryUtil.isAutoCrlfSetRight(intellijRepository)) { + // lineEndingsConfiguredCorrectly = false; + // ATTENTION + // .createNotification(JpsCacheBundle.message("notification.title.git.crlf.config"), + // JpsCacheBundle.message("notification.content.git.crlf.config"), + // NotificationType.WARNING) + // .addAction(NotificationAction.createSimple(JpsCacheBundle.message("notification.action.git.crlf.config"), + // () -> BrowserLauncher.getInstance().open("https://confluence.jetbrains.com/pages/viewpage.action?title=Git+Repository&spaceKey=IDEA"))) + // .notify(project); + //} + } + + private static void checkAutoBuildEnabled(@NotNull Project project) { + //PropertiesComponent propertiesComponent = PropertiesComponent.getInstance(project); + //if (propertiesComponent.getBoolean(NOT_ASK_AGAIN)) { + // return; + //} + // + //CompilerWorkspaceConfiguration workspaceConfiguration = CompilerWorkspaceConfiguration.getInstance(project); + //if (!workspaceConfiguration.MAKE_PROJECT_ON_SAVE || !TrustedProjects.isTrusted(project)) { + // return; + //} + // + //ATTENTION + // .createNotification(JpsCacheBundle.message("notification.title.automatic.project.build.enabled"), + // JpsCacheBundle.message("notification.content.make.project.automatically.enabled.affect.caches"), + // NotificationType.WARNING) + // .addAction(NotificationAction.createSimpleExpiring( + // JpsCacheBundle.message("action.NotificationAction.JpsCachesDummyProjectComponent.text.disable.property"), () -> { + // workspaceConfiguration.MAKE_PROJECT_ON_SAVE = false; + // BuildManager.getInstance().clearState(project); + // })) + // .addAction(NotificationAction.createSimpleExpiring( + // JpsCacheBundle.message("action.NotificationAction.JpsCachesDummyProjectComponent.text.dont.ask"), () -> { + // propertiesComponent.setValue(NOT_ASK_AGAIN, true); + // })) + // .notify(project); + } + + public static boolean isLineEndingsConfiguredCorrectly() { + return lineEndingsConfiguredCorrectly; + } +} diff --git a/java/compiler/impl/src/com/intellij/compiler/cache/client/JpsServerAuthExtension.java b/java/compiler/impl/src/com/intellij/compiler/cache/client/JpsServerAuthExtension.java new file mode 100644 index 000000000000..5bdd0b402f99 --- /dev/null +++ b/java/compiler/impl/src/com/intellij/compiler/cache/client/JpsServerAuthExtension.java @@ -0,0 +1,81 @@ +package com.intellij.compiler.cache.client; + +import com.intellij.notification.NotificationListener; +import com.intellij.notification.NotificationType; +import com.intellij.openapi.Disposable; +import com.intellij.openapi.application.ApplicationManager; +import com.intellij.openapi.compiler.JavaCompilerBundle; +import com.intellij.openapi.diagnostic.Logger; +import com.intellij.openapi.extensions.ExtensionPointName; +import com.intellij.openapi.project.Project; +import com.intellij.openapi.util.Disposer; +import com.intellij.openapi.util.Key; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.Map; + +import static com.intellij.compiler.cache.ui.JpsLoaderNotifications.ATTENTION; +import static com.intellij.execution.process.ProcessIOExecutorService.INSTANCE; + +/** + * Extension point which provides authentication data for requests to the JPS cache server + */ +public interface JpsServerAuthExtension { + Logger LOG = Logger.getInstance(JpsServerAuthExtension.class); + Key NOTIFICATION_SHOWN_KEY = Key.create("AUTH_NOTIFICATION_SHOWN"); + ExtensionPointName EP_NAME = ExtensionPointName.create("com.intellij.jpsServerAuthExtension"); + + /** + * This method should check if the user was authenticated, if not it should do any needed actions to provide + * auth token for further requests. This method will be called outside of the EDT and should be asynchronous. + * If the user was authenticated the callback should be invoked. + * + * @param presentableReason reason for the token request + * @param parentDisposable controls the lifetime of the authentication + * @param onAuthCompleted callback on authentication complete, if token already exists it also should be invoked + */ + void checkAuthenticated(@NotNull String presentableReason, @NotNull Disposable parentDisposable, @NotNull Runnable onAuthCompleted); + + /** + * The method provides HTTP authentication headers for the requests to the server. + * It will be called in the background thread. The assertion that thread isn't EDT can + * be added to the implementation. If it's not possible to get the authentication headers, + * empty map or `null` can be return. + * @return + */ + Map getAuthHeader(); + + @Nullable + static JpsServerAuthExtension getInstance() { + return EP_NAME.extensions().findFirst().orElse(null); + } + + static void checkAuthenticatedInBackgroundThread(@NotNull Disposable parentDisposable, @NotNull Project project, @NotNull Runnable onAuthCompleted) { + Disposable disposable = Disposer.newDisposable(); + Disposer.register(parentDisposable, disposable); + JpsServerAuthExtension authExtension = getInstance(); + if (authExtension == null) { + Boolean userData = project.getUserData(NOTIFICATION_SHOWN_KEY); + if (userData == null) { + project.putUserData(NOTIFICATION_SHOWN_KEY, Boolean.TRUE); + ApplicationManager.getApplication().invokeLater(() -> { + ATTENTION + .createNotification(JavaCompilerBundle.message("notification.title.jps.caches.downloader"), + JavaCompilerBundle.message("notification.content.internal.authentication.plugin.required.for.correct.work.plugin"), + NotificationType.WARNING) + .setListener(NotificationListener.URL_OPENING_LISTENER) + .notify(project); + }); + } + LOG.warn("JetBrains Internal Authentication plugin is required for the correct work. Please enable it."); + return; + } + INSTANCE.execute(() -> { + authExtension.checkAuthenticated("Jps Caches Downloader", disposable, () -> { + Disposer.dispose(disposable); + onAuthCompleted.run(); + }); + }); + } +} diff --git a/java/compiler/impl/src/com/intellij/compiler/cache/client/JpsServerAuthUtil.java b/java/compiler/impl/src/com/intellij/compiler/cache/client/JpsServerAuthUtil.java new file mode 100644 index 000000000000..392345e3abf2 --- /dev/null +++ b/java/compiler/impl/src/com/intellij/compiler/cache/client/JpsServerAuthUtil.java @@ -0,0 +1,22 @@ +package com.intellij.compiler.cache.client; + +import com.intellij.openapi.compiler.JavaCompilerBundle; +import org.jetbrains.annotations.NotNull; + +import java.util.Map; + +public final class JpsServerAuthUtil { + public static @NotNull Map getRequestHeaders() { + JpsServerAuthExtension authExtension = JpsServerAuthExtension.getInstance(); + if (authExtension == null) { + String message = JavaCompilerBundle.message("notification.content.internal.authentication.plugin.required.for.correct.work.plugin"); + throw new RuntimeException(message); + } + Map authHeader = authExtension.getAuthHeader(); + if (authHeader == null) { + String message = JavaCompilerBundle.message("internal.authentication.plugin.missing.token"); + throw new RuntimeException(message); + } + return authHeader; + } +} diff --git a/java/compiler/impl/src/com/intellij/compiler/cache/git/GitCommitsIterator.java b/java/compiler/impl/src/com/intellij/compiler/cache/git/GitCommitsIterator.java new file mode 100644 index 000000000000..812764f5c1c1 --- /dev/null +++ b/java/compiler/impl/src/com/intellij/compiler/cache/git/GitCommitsIterator.java @@ -0,0 +1,84 @@ +//package com.intellij.compiler.cache.git; +// +//import com.intellij.openapi.diagnostic.Logger; +//import com.intellij.openapi.project.Project; +//import com.intellij.openapi.vcs.VcsException; +//import com.intellij.util.SmartList; +//import com.intellij.util.containers.ContainerUtil; +//import git4idea.history.GitHistoryUtils; +//import git4idea.repo.GitRepository; +//import org.jetbrains.annotations.NotNull; +// +//import java.util.Iterator; +//import java.util.List; +//import java.util.NoSuchElementException; +// +//public class GitCommitsIterator implements Iterator { +// private static final Logger LOG = Logger.getInstance(GitCommitsIterator.class); +// +// private static final int MAX_FETCH_SIZE = 1000; +// private static final int FETCH_SIZE = 100; +// private final GitRepository myRepository; +// private final Project myProject; +// private int fetchedCount; +// private List commits; +// private String remote; +// private int currentPosition; +// +// public GitCommitsIterator(@NotNull Project project, @NotNull GitRepository repository, @NotNull String remoteUrl) { +// myRepository = repository; +// myProject = project; +// fetchedCount = 0; +// remote = remoteUrl; +// fetchOldCommits(); +// } +// +// @Override +// public boolean hasNext() { +// if (commits.size() > 0) { +// if (currentPosition < commits.size()) return true; +// if (fetchedCount >= MAX_FETCH_SIZE) { +// LOG.info("Exceeded fetch limit for git commits"); +// return false; +// } +// fetchOldCommits(commits.get(currentPosition - 1)); +// if (commits.size() > 0) { +// currentPosition = 0; +// return true; +// } +// } +// return false; +// } +// +// @Override +// public String next() { +// if (commits.size() == 0 || currentPosition >= commits.size()) throw new NoSuchElementException(); +// String result = commits.get(currentPosition); +// currentPosition++; +// return result; +// } +// +// @NotNull +// public String getRemote() { +// return remote; +// } +// +// private void fetchOldCommits() { +// fetchOldCommits(""); +// } +// +// private void fetchOldCommits(String latestCommit) { +// try { +// commits = +// ContainerUtil.map(latestCommit.isEmpty() ? GitHistoryUtils.collectTimedCommits(myProject, myRepository.getRoot(), "-n " + FETCH_SIZE) : +// GitHistoryUtils.collectTimedCommits(myProject, myRepository.getRoot(), latestCommit, "-n " + FETCH_SIZE), +// it -> it.getId().asString()); +// fetchedCount += commits.size(); +// return; +// } +// catch (VcsException e) { +// LOG.warn("Can't get Git hashes for commits", e); +// } +// commits = new SmartList<>(); +// } +//} \ No newline at end of file diff --git a/java/compiler/impl/src/com/intellij/compiler/cache/git/GitRepositoryUtil.java b/java/compiler/impl/src/com/intellij/compiler/cache/git/GitRepositoryUtil.java new file mode 100644 index 000000000000..099082225068 --- /dev/null +++ b/java/compiler/impl/src/com/intellij/compiler/cache/git/GitRepositoryUtil.java @@ -0,0 +1,70 @@ +//package com.intellij.jps.cache.git; +// +//import com.intellij.jps.cache.git.GitCommitsIterator; +//import com.intellij.openapi.diagnostic.Logger; +//import com.intellij.openapi.project.Project; +//import com.intellij.util.containers.ContainerUtil; +//import git4idea.GitUtil; +//import git4idea.commands.Git; +//import git4idea.commands.GitCommandResult; +//import git4idea.config.GitConfigUtil; +//import git4idea.repo.GitRemote; +//import git4idea.repo.GitRepository; +//import org.jetbrains.annotations.NotNull; +//import org.jetbrains.annotations.Nullable; +// +//import java.util.*; +//import java.util.stream.Collectors; +// +//public final class GitRepositoryUtil { +// private static final Logger LOG = Logger.getInstance(GitRepositoryUtil.class); +// +// private GitRepositoryUtil() {} +// +// @NotNull +// public static List getCommitsIterator(@NotNull Project project, @NotNull Set remoteUrlNames) { +// if (GitUtil.hasGitRepositories(project)) { +// return GitUtil.getRepositories(project).stream() +// .map(repo -> { +// Set remoteUrls = repo.getRemotes().stream() +// .map(remote -> remote.getUrls()) +// .flatMap(Collection::stream) +// .collect(Collectors.toSet()); +// String matchedRemoteUrl = ContainerUtil.find(remoteUrls, remoteUrl -> remoteUrlNames.contains(getRemoteRepoName(remoteUrl))); +// if (matchedRemoteUrl == null) return null; +// return new GitCommitsIterator(project, repo, getRemoteRepoName(matchedRemoteUrl)); +// }).filter(Objects::nonNull) +// .collect(Collectors.toList()); +// } +// LOG.info("Project doesn't contain Git repository"); +// return Collections.emptyList(); +// } +// +// @Nullable +// public static GitRepository getRepositoryByName(@NotNull Project project, @NotNull String repositoryName) { +// if (GitUtil.hasGitRepositories(project)) { +// return ContainerUtil.find(GitUtil.getRepositories(project), repo -> { +// for (GitRemote remote : repo.getRemotes()) { +// for (String remoteUrl : remote.getUrls()) { +// if (getRemoteRepoName(remoteUrl).equals(repositoryName)) return true; +// } +// } +// return false; +// }); +// } +// LOG.info("Project doesn't contain Git repository"); +// return null; +// } +// +// public static String getRemoteRepoName(@NotNull String remoteUrl) { +// String[] splittedRemoteUrl = remoteUrl.split("/"); +// return splittedRemoteUrl[splittedRemoteUrl.length - 1]; +// } +// +// public static boolean isAutoCrlfSetRight(@NotNull GitRepository gitRepository) { +// GitCommandResult result = Git.getInstance().config(gitRepository, GitConfigUtil.CORE_AUTOCRLF); +// String value = result.getOutputAsJoinedString(); +// LOG.info("CRLF configuration for " + gitRepository + " project: " + value); +// return value.equalsIgnoreCase("input"); +// } +//} \ No newline at end of file diff --git a/java/compiler/impl/src/com/intellij/compiler/cache/ui/JpsLoaderNotifications.java b/java/compiler/impl/src/com/intellij/compiler/cache/ui/JpsLoaderNotifications.java new file mode 100644 index 000000000000..b2fb7ab085b8 --- /dev/null +++ b/java/compiler/impl/src/com/intellij/compiler/cache/ui/JpsLoaderNotifications.java @@ -0,0 +1,13 @@ +package com.intellij.compiler.cache.ui; + +import com.intellij.notification.NotificationDisplayType; +import com.intellij.notification.NotificationGroup; + +public final class JpsLoaderNotifications { + public static final NotificationGroup ATTENTION = new NotificationGroup("Compile Output Loader: Attention", + NotificationDisplayType.STICKY_BALLOON, true); + public static final NotificationGroup STANDARD = new NotificationGroup("Compile Output Loader: Standard", + NotificationDisplayType.BALLOON, true); + public static final NotificationGroup EVENT_LOG = new NotificationGroup("Compile Output Loader: Event Log", + NotificationDisplayType.NONE, true); +} diff --git a/java/compiler/impl/src/com/intellij/compiler/server/BuildManager.java b/java/compiler/impl/src/com/intellij/compiler/server/BuildManager.java index 2b3a727a1592..65b9414d6581 100644 --- a/java/compiler/impl/src/com/intellij/compiler/server/BuildManager.java +++ b/java/compiler/impl/src/com/intellij/compiler/server/BuildManager.java @@ -168,7 +168,6 @@ public final class BuildManager implements Disposable { private final Executor myAutomakeTrigger = AppExecutorUtil.createBoundedApplicationPoolExecutor("BuildManager Auto-Make Trigger", 1); private final Map myProjectDataMap = Collections.synchronizedMap(new HashMap<>()); private final AtomicInteger mySuspendBackgroundTasksCounter = new AtomicInteger(0); - private boolean myGeneratePortableCachesEnabled = false; private final BuildManagerPeriodicTask myAutoMakeTask = new BuildManagerPeriodicTask() { @Override @@ -1406,7 +1405,7 @@ public final class BuildManager implements Disposable { } // portable caches - if (isGeneratePortableCachesEnabled()) { + if (Registry.is("compiler.process.use.portable.caches")) { //cmdLine.addParameter("-Didea.resizeable.file.truncate.on.close=true"); //cmdLine.addParameter("-Dkotlin.jps.non.caching.storage=true"); cmdLine.addParameter("-D" + ProjectStamps.PORTABLE_CACHES_PROPERTY + "=true"); @@ -1721,17 +1720,6 @@ public final class BuildManager implements Disposable { } } - public boolean isGeneratePortableCachesEnabled() { - return myGeneratePortableCachesEnabled; - } - - public void setGeneratePortableCachesEnabled(boolean generatePortableCachesEnabled) { - if (myGeneratePortableCachesEnabled != generatePortableCachesEnabled) { - myGeneratePortableCachesEnabled = generatePortableCachesEnabled; - clearState(); - } - } - private abstract class BuildManagerPeriodicTask implements Runnable { private final Alarm myAlarm; private final AtomicBoolean myInProgress = new AtomicBoolean(false); diff --git a/java/compiler/impl/src/com/intellij/compiler/server/DefaultMessageHandler.java b/java/compiler/impl/src/com/intellij/compiler/server/DefaultMessageHandler.java index 8edca6873098..f93d7f0f6097 100644 --- a/java/compiler/impl/src/com/intellij/compiler/server/DefaultMessageHandler.java +++ b/java/compiler/impl/src/com/intellij/compiler/server/DefaultMessageHandler.java @@ -1,13 +1,16 @@ // Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. package com.intellij.compiler.server; +import com.intellij.compiler.cache.client.JpsServerAuthUtil; import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.progress.ProgressIndicator; import com.intellij.openapi.project.Project; import io.netty.channel.Channel; import org.jetbrains.annotations.NotNull; +import org.jetbrains.jps.api.CmdlineProtoUtil; import org.jetbrains.jps.api.CmdlineRemoteProto; +import java.util.Map; import java.util.UUID; public abstract class DefaultMessageHandler implements BuilderMessageHandler { @@ -45,6 +48,11 @@ public abstract class DefaultMessageHandler implements BuilderMessageHandler { LOG.error("Internal build error:\n" + compileMessage.getText()); } break; + case AUTH_TOKEN_REQUEST: + Map headers = JpsServerAuthUtil.getRequestHeaders(); + channel.writeAndFlush(CmdlineProtoUtil.toMessage(sessionId, CmdlineProtoUtil.createRequestParamsCommand(headers))); + System.out.println("Message got"); + break; case CONSTANT_SEARCH_TASK: // ignored, because the functionality is deprecated break; diff --git a/java/compiler/openapi/resources/messages/JavaCompilerBundle.properties b/java/compiler/openapi/resources/messages/JavaCompilerBundle.properties index b193a7e0783d..45b846e7a625 100644 --- a/java/compiler/openapi/resources/messages/JavaCompilerBundle.properties +++ b/java/compiler/openapi/resources/messages/JavaCompilerBundle.properties @@ -329,3 +329,7 @@ dialog.message.failed.to.determine.host.ip.for.wsl.jdk=Failed to determine host progress.preparing.wsl.build.environment=Preparing WSL build environment... plugins.advertiser.feature.artifact=artifact notification.group.compiler=Build finished + +notification.title.jps.caches.downloader=Jps caches downloader +notification.content.internal.authentication.plugin.required.for.correct.work.plugin=JetBrains Internal Authentication is required for the correct work of the plugin +internal.authentication.plugin.missing.token=Unexpected state: jetbrains.team token is missing. Please report this exception and perform actions 'Logout from JetBrains.team' and 'Login to JetBrains.team' as a workaround \ No newline at end of file diff --git a/java/java-impl/src/META-INF/JavaPlugin.xml b/java/java-impl/src/META-INF/JavaPlugin.xml index 647f733e6b2b..f124717815ac 100644 --- a/java/java-impl/src/META-INF/JavaPlugin.xml +++ b/java/java-impl/src/META-INF/JavaPlugin.xml @@ -273,6 +273,7 @@ interface="com.intellij.ide.projectWizard.generators.BuildSystemJavaNewProjectWizard" dynamic="true"/> + @@ -294,6 +295,7 @@ + diff --git a/jps/jps-builders/gen/org/jetbrains/jps/api/CmdlineRemoteProto.java b/jps/jps-builders/gen/org/jetbrains/jps/api/CmdlineRemoteProto.java index 679e54d561a7..1531b9d7169e 100644 --- a/jps/jps-builders/gen/org/jetbrains/jps/api/CmdlineRemoteProto.java +++ b/jps/jps-builders/gen/org/jetbrains/jps/api/CmdlineRemoteProto.java @@ -1544,6 +1544,17 @@ public final class CmdlineRemoteProto { * @return The constantSearchResult. */ org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.ConstantSearchResult getConstantSearchResult(); + + /** + * optional .org.jetbrains.jpsservice.Message.ControllerMessage.RequestParams request_params = 5; + * @return Whether the requestParams field is set. + */ + boolean hasRequestParams(); + /** + * optional .org.jetbrains.jpsservice.Message.ControllerMessage.RequestParams request_params = 5; + * @return The requestParams. + */ + org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.RequestParams getRequestParams(); } /** * Protobuf type {@code org.jetbrains.jpsservice.Message.ControllerMessage} @@ -1577,6 +1588,10 @@ public final class CmdlineRemoteProto { * CONSTANT_SEARCH_RESULT = 4; */ CONSTANT_SEARCH_RESULT(4), + /** + * AUTHENTICATION_TOKEN = 5; + */ + AUTHENTICATION_TOKEN(5), ; /** @@ -1595,6 +1610,10 @@ public final class CmdlineRemoteProto { * CONSTANT_SEARCH_RESULT = 4; */ public static final int CONSTANT_SEARCH_RESULT_VALUE = 4; + /** + * AUTHENTICATION_TOKEN = 5; + */ + public static final int AUTHENTICATION_TOKEN_VALUE = 5; @java.lang.Override @@ -1618,6 +1637,7 @@ public final class CmdlineRemoteProto { case 2: return CANCEL_BUILD_COMMAND; case 3: return FS_EVENT; case 4: return CONSTANT_SEARCH_RESULT; + case 5: return AUTHENTICATION_TOKEN; default: return null; } } @@ -5334,6 +5354,415 @@ public final class CmdlineRemoteProto { } } + public interface RequestParamsOrBuilder extends + // @@protoc_insertion_point(interface_extends:org.jetbrains.jpsservice.Message.ControllerMessage.RequestParams) + com.google.protobuf.MessageLiteOrBuilder { + + /** + * map<string, string> auth_headers = 1; + */ + int getAuthHeadersCount(); + /** + * map<string, string> auth_headers = 1; + */ + boolean containsAuthHeaders( + java.lang.String key); + /** + * Use {@link #getAuthHeadersMap()} instead. + */ + @java.lang.Deprecated + java.util.Map + getAuthHeaders(); + /** + * map<string, string> auth_headers = 1; + */ + java.util.Map + getAuthHeadersMap(); + /** + * map<string, string> auth_headers = 1; + */ + + java.lang.String getAuthHeadersOrDefault( + java.lang.String key, + java.lang.String defaultValue); + /** + * map<string, string> auth_headers = 1; + */ + + java.lang.String getAuthHeadersOrThrow( + java.lang.String key); + } + /** + * Protobuf type {@code org.jetbrains.jpsservice.Message.ControllerMessage.RequestParams} + */ + public static final class RequestParams extends + com.google.protobuf.GeneratedMessageLite< + RequestParams, RequestParams.Builder> implements + // @@protoc_insertion_point(message_implements:org.jetbrains.jpsservice.Message.ControllerMessage.RequestParams) + RequestParamsOrBuilder { + private RequestParams() { + } + public static final int AUTH_HEADERS_FIELD_NUMBER = 1; + private static final class AuthHeadersDefaultEntryHolder { + static final com.google.protobuf.MapEntryLite< + java.lang.String, java.lang.String> defaultEntry = + com.google.protobuf.MapEntryLite + .newDefaultInstance( + com.google.protobuf.WireFormat.FieldType.STRING, + "", + com.google.protobuf.WireFormat.FieldType.STRING, + ""); + } + private com.google.protobuf.MapFieldLite< + java.lang.String, java.lang.String> authHeaders_ = + com.google.protobuf.MapFieldLite.emptyMapField(); + private com.google.protobuf.MapFieldLite + internalGetAuthHeaders() { + return authHeaders_; + } + private com.google.protobuf.MapFieldLite + internalGetMutableAuthHeaders() { + if (!authHeaders_.isMutable()) { + authHeaders_ = authHeaders_.mutableCopy(); + } + return authHeaders_; + } + @java.lang.Override + + public int getAuthHeadersCount() { + return internalGetAuthHeaders().size(); + } + /** + * map<string, string> auth_headers = 1; + */ + @java.lang.Override + + public boolean containsAuthHeaders( + java.lang.String key) { + key.getClass(); + return internalGetAuthHeaders().containsKey(key); + } + /** + * Use {@link #getAuthHeadersMap()} instead. + */ + @java.lang.Override + @java.lang.Deprecated + public java.util.Map getAuthHeaders() { + return getAuthHeadersMap(); + } + /** + * map<string, string> auth_headers = 1; + */ + @java.lang.Override + + public java.util.Map getAuthHeadersMap() { + return java.util.Collections.unmodifiableMap( + internalGetAuthHeaders()); + } + /** + * map<string, string> auth_headers = 1; + */ + @java.lang.Override + + public java.lang.String getAuthHeadersOrDefault( + java.lang.String key, + java.lang.String defaultValue) { + key.getClass(); + java.util.Map map = + internalGetAuthHeaders(); + return map.containsKey(key) ? map.get(key) : defaultValue; + } + /** + * map<string, string> auth_headers = 1; + */ + @java.lang.Override + + public java.lang.String getAuthHeadersOrThrow( + java.lang.String key) { + key.getClass(); + java.util.Map map = + internalGetAuthHeaders(); + if (!map.containsKey(key)) { + throw new java.lang.IllegalArgumentException(); + } + return map.get(key); + } + /** + * map<string, string> auth_headers = 1; + */ + private java.util.Map + getMutableAuthHeadersMap() { + return internalGetMutableAuthHeaders(); + } + + public static org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.RequestParams parseFrom( + java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return com.google.protobuf.GeneratedMessageLite.parseFrom( + DEFAULT_INSTANCE, data); + } + public static org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.RequestParams parseFrom( + java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return com.google.protobuf.GeneratedMessageLite.parseFrom( + DEFAULT_INSTANCE, data, extensionRegistry); + } + public static org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.RequestParams parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return com.google.protobuf.GeneratedMessageLite.parseFrom( + DEFAULT_INSTANCE, data); + } + public static org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.RequestParams parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return com.google.protobuf.GeneratedMessageLite.parseFrom( + DEFAULT_INSTANCE, data, extensionRegistry); + } + public static org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.RequestParams parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return com.google.protobuf.GeneratedMessageLite.parseFrom( + DEFAULT_INSTANCE, data); + } + public static org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.RequestParams parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return com.google.protobuf.GeneratedMessageLite.parseFrom( + DEFAULT_INSTANCE, data, extensionRegistry); + } + public static org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.RequestParams parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageLite.parseFrom( + DEFAULT_INSTANCE, input); + } + public static org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.RequestParams parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageLite.parseFrom( + DEFAULT_INSTANCE, input, extensionRegistry); + } + public static org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.RequestParams parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return parseDelimitedFrom(DEFAULT_INSTANCE, input); + } + public static org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.RequestParams parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return parseDelimitedFrom(DEFAULT_INSTANCE, input, extensionRegistry); + } + public static org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.RequestParams parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageLite.parseFrom( + DEFAULT_INSTANCE, input); + } + public static org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.RequestParams parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageLite.parseFrom( + DEFAULT_INSTANCE, input, extensionRegistry); + } + + public static Builder newBuilder() { + return (Builder) DEFAULT_INSTANCE.createBuilder(); + } + public static Builder newBuilder(org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.RequestParams prototype) { + return (Builder) DEFAULT_INSTANCE.createBuilder(prototype); + } + + /** + * Protobuf type {@code org.jetbrains.jpsservice.Message.ControllerMessage.RequestParams} + */ + public static final class Builder extends + com.google.protobuf.GeneratedMessageLite.Builder< + org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.RequestParams, Builder> implements + // @@protoc_insertion_point(builder_implements:org.jetbrains.jpsservice.Message.ControllerMessage.RequestParams) + org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.RequestParamsOrBuilder { + // Construct using org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.RequestParams.newBuilder() + private Builder() { + super(DEFAULT_INSTANCE); + } + + + @java.lang.Override + + public int getAuthHeadersCount() { + return instance.getAuthHeadersMap().size(); + } + /** + * map<string, string> auth_headers = 1; + */ + @java.lang.Override + + public boolean containsAuthHeaders( + java.lang.String key) { + key.getClass(); + return instance.getAuthHeadersMap().containsKey(key); + } + + public Builder clearAuthHeaders() { + copyOnWrite(); + instance.getMutableAuthHeadersMap().clear(); + return this; + } + /** + * map<string, string> auth_headers = 1; + */ + + public Builder removeAuthHeaders( + java.lang.String key) { + key.getClass(); + copyOnWrite(); + instance.getMutableAuthHeadersMap().remove(key); + return this; + } + /** + * Use {@link #getAuthHeadersMap()} instead. + */ + @java.lang.Override + @java.lang.Deprecated + public java.util.Map getAuthHeaders() { + return getAuthHeadersMap(); + } + /** + * map<string, string> auth_headers = 1; + */ + @java.lang.Override + public java.util.Map getAuthHeadersMap() { + return java.util.Collections.unmodifiableMap( + instance.getAuthHeadersMap()); + } + /** + * map<string, string> auth_headers = 1; + */ + @java.lang.Override + + public java.lang.String getAuthHeadersOrDefault( + java.lang.String key, + java.lang.String defaultValue) { + key.getClass(); + java.util.Map map = + instance.getAuthHeadersMap(); + return map.containsKey(key) ? map.get(key) : defaultValue; + } + /** + * map<string, string> auth_headers = 1; + */ + @java.lang.Override + + public java.lang.String getAuthHeadersOrThrow( + java.lang.String key) { + key.getClass(); + java.util.Map map = + instance.getAuthHeadersMap(); + if (!map.containsKey(key)) { + throw new java.lang.IllegalArgumentException(); + } + return map.get(key); + } + /** + * map<string, string> auth_headers = 1; + */ + public Builder putAuthHeaders( + java.lang.String key, + java.lang.String value) { + key.getClass(); + value.getClass(); + copyOnWrite(); + instance.getMutableAuthHeadersMap().put(key, value); + return this; + } + /** + * map<string, string> auth_headers = 1; + */ + public Builder putAllAuthHeaders( + java.util.Map values) { + copyOnWrite(); + instance.getMutableAuthHeadersMap().putAll(values); + return this; + } + + // @@protoc_insertion_point(builder_scope:org.jetbrains.jpsservice.Message.ControllerMessage.RequestParams) + } + @java.lang.Override + @java.lang.SuppressWarnings({"unchecked", "fallthrough"}) + protected final java.lang.Object dynamicMethod( + com.google.protobuf.GeneratedMessageLite.MethodToInvoke method, + java.lang.Object arg0, java.lang.Object arg1) { + switch (method) { + case NEW_MUTABLE_INSTANCE: { + return new org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.RequestParams(); + } + case NEW_BUILDER: { + return new Builder(); + } + case BUILD_MESSAGE_INFO: { + java.lang.Object[] objects = new java.lang.Object[] { + "authHeaders_", + AuthHeadersDefaultEntryHolder.defaultEntry, + }; + java.lang.String info = + "\u0001\u0001\u0000\u0000\u0001\u0001\u0001\u0001\u0000\u0000\u00012"; + return newMessageInfo(DEFAULT_INSTANCE, info, objects); + } + // fall through + case GET_DEFAULT_INSTANCE: { + return DEFAULT_INSTANCE; + } + case GET_PARSER: { + com.google.protobuf.Parser parser = PARSER; + if (parser == null) { + synchronized (org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.RequestParams.class) { + parser = PARSER; + if (parser == null) { + parser = + new DefaultInstanceBasedParser( + DEFAULT_INSTANCE); + PARSER = parser; + } + } + } + return parser; + } + case GET_MEMOIZED_IS_INITIALIZED: { + return (byte) 1; + } + case SET_MEMOIZED_IS_INITIALIZED: { + return null; + } + } + throw new UnsupportedOperationException(); + } + + + // @@protoc_insertion_point(class_scope:org.jetbrains.jpsservice.Message.ControllerMessage.RequestParams) + private static final org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.RequestParams DEFAULT_INSTANCE; + static { + RequestParams defaultInstance = new RequestParams(); + // New instances are implicitly immutable so no need to make + // immutable. + DEFAULT_INSTANCE = defaultInstance; + com.google.protobuf.GeneratedMessageLite.registerDefaultInstance( + RequestParams.class, defaultInstance); + } + + public static org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.RequestParams getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + private static volatile com.google.protobuf.Parser PARSER; + + public static com.google.protobuf.Parser parser() { + return DEFAULT_INSTANCE.getParserForType(); + } + } + private int bitField0_; public static final int TYPE_FIELD_NUMBER = 1; private int type_; @@ -5508,6 +5937,52 @@ public final class CmdlineRemoteProto { bitField0_ = (bitField0_ & ~0x00000008); } + public static final int REQUEST_PARAMS_FIELD_NUMBER = 5; + private org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.RequestParams requestParams_; + /** + * optional .org.jetbrains.jpsservice.Message.ControllerMessage.RequestParams request_params = 5; + */ + @java.lang.Override + public boolean hasRequestParams() { + return ((bitField0_ & 0x00000010) != 0); + } + /** + * optional .org.jetbrains.jpsservice.Message.ControllerMessage.RequestParams request_params = 5; + */ + @java.lang.Override + public org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.RequestParams getRequestParams() { + return requestParams_ == null ? org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.RequestParams.getDefaultInstance() : requestParams_; + } + /** + * optional .org.jetbrains.jpsservice.Message.ControllerMessage.RequestParams request_params = 5; + */ + private void setRequestParams(org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.RequestParams value) { + value.getClass(); + requestParams_ = value; + bitField0_ |= 0x00000010; + } + /** + * optional .org.jetbrains.jpsservice.Message.ControllerMessage.RequestParams request_params = 5; + */ + @java.lang.SuppressWarnings({"ReferenceEquality"}) + private void mergeRequestParams(org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.RequestParams value) { + value.getClass(); + if (requestParams_ != null && + requestParams_ != org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.RequestParams.getDefaultInstance()) { + requestParams_ = + org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.RequestParams.newBuilder(requestParams_).mergeFrom(value).buildPartial(); + } else { + requestParams_ = value; + } + bitField0_ |= 0x00000010; + } + /** + * optional .org.jetbrains.jpsservice.Message.ControllerMessage.RequestParams request_params = 5; + */ + private void clearRequestParams() { requestParams_ = null; + bitField0_ = (bitField0_ & ~0x00000010); + } + public static org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage parseFrom( java.nio.ByteBuffer data) throws com.google.protobuf.InvalidProtocolBufferException { @@ -5780,6 +6255,53 @@ public final class CmdlineRemoteProto { return this; } + /** + * optional .org.jetbrains.jpsservice.Message.ControllerMessage.RequestParams request_params = 5; + */ + @java.lang.Override + public boolean hasRequestParams() { + return instance.hasRequestParams(); + } + /** + * optional .org.jetbrains.jpsservice.Message.ControllerMessage.RequestParams request_params = 5; + */ + @java.lang.Override + public org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.RequestParams getRequestParams() { + return instance.getRequestParams(); + } + /** + * optional .org.jetbrains.jpsservice.Message.ControllerMessage.RequestParams request_params = 5; + */ + public Builder setRequestParams(org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.RequestParams value) { + copyOnWrite(); + instance.setRequestParams(value); + return this; + } + /** + * optional .org.jetbrains.jpsservice.Message.ControllerMessage.RequestParams request_params = 5; + */ + public Builder setRequestParams( + org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.RequestParams.Builder builderForValue) { + copyOnWrite(); + instance.setRequestParams(builderForValue.build()); + return this; + } + /** + * optional .org.jetbrains.jpsservice.Message.ControllerMessage.RequestParams request_params = 5; + */ + public Builder mergeRequestParams(org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.RequestParams value) { + copyOnWrite(); + instance.mergeRequestParams(value); + return this; + } + /** + * optional .org.jetbrains.jpsservice.Message.ControllerMessage.RequestParams request_params = 5; + */ + public Builder clearRequestParams() { copyOnWrite(); + instance.clearRequestParams(); + return this; + } + // @@protoc_insertion_point(builder_scope:org.jetbrains.jpsservice.Message.ControllerMessage) } private byte memoizedIsInitialized = 2; @@ -5803,10 +6325,11 @@ public final class CmdlineRemoteProto { "paramsMessage_", "fsEvent_", "constantSearchResult_", + "requestParams_", }; java.lang.String info = - "\u0001\u0004\u0000\u0001\u0001\u0004\u0004\u0000\u0000\u0004\u0001\u150c\u0000\u0002" + - "\u1409\u0001\u0003\u1409\u0002\u0004\u1409\u0003"; + "\u0001\u0005\u0000\u0001\u0001\u0005\u0005\u0000\u0000\u0004\u0001\u150c\u0000\u0002" + + "\u1409\u0001\u0003\u1409\u0002\u0004\u1409\u0003\u0005\u1009\u0004"; return newMessageInfo(DEFAULT_INSTANCE, info, objects); } // fall through @@ -5942,6 +6465,10 @@ public final class CmdlineRemoteProto { * CONSTANT_SEARCH_TASK = 4; */ CONSTANT_SEARCH_TASK(4), + /** + * AUTH_TOKEN_REQUEST = 5; + */ + AUTH_TOKEN_REQUEST(5), ; /** @@ -5960,6 +6487,10 @@ public final class CmdlineRemoteProto { * CONSTANT_SEARCH_TASK = 4; */ public static final int CONSTANT_SEARCH_TASK_VALUE = 4; + /** + * AUTH_TOKEN_REQUEST = 5; + */ + public static final int AUTH_TOKEN_REQUEST_VALUE = 5; @java.lang.Override @@ -5983,6 +6514,7 @@ public final class CmdlineRemoteProto { case 2: return BUILD_EVENT; case 3: return COMPILE_MESSAGE; case 4: return CONSTANT_SEARCH_TASK; + case 5: return AUTH_TOKEN_REQUEST; default: return null; } } diff --git a/jps/jps-builders/proto/cmdline_remote_proto.proto b/jps/jps-builders/proto/cmdline_remote_proto.proto index eb3fc96813cf..a001790f7ca0 100644 --- a/jps/jps-builders/proto/cmdline_remote_proto.proto +++ b/jps/jps-builders/proto/cmdline_remote_proto.proto @@ -34,6 +34,7 @@ message Message { CANCEL_BUILD_COMMAND = 2; FS_EVENT = 3; CONSTANT_SEARCH_RESULT = 4; + AUTHENTICATION_TOKEN = 5; } message FSEvent { @@ -75,10 +76,15 @@ message Message { repeated string path = 4; } + message RequestParams { + map auth_headers = 1; + } + required Type type = 1; optional ParametersMessage params_message = 2; optional FSEvent fs_event = 3; optional ConstantSearchResult constant_search_result = 4; + optional RequestParams request_params = 5; } message BuilderMessage { @@ -87,6 +93,7 @@ message Message { BUILD_EVENT = 2; COMPILE_MESSAGE = 3; CONSTANT_SEARCH_TASK = 4; + AUTH_TOKEN_REQUEST = 5; } message BuildEvent { diff --git a/jps/jps-builders/src/org/jetbrains/jps/api/CmdlineProtoUtil.java b/jps/jps-builders/src/org/jetbrains/jps/api/CmdlineProtoUtil.java index e90ed4009832..77b773414682 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/api/CmdlineProtoUtil.java +++ b/jps/jps-builders/src/org/jetbrains/jps/api/CmdlineProtoUtil.java @@ -122,6 +122,14 @@ public final class CmdlineProtoUtil { return builder.build(); } + public static CmdlineRemoteProto.Message.ControllerMessage createRequestParamsCommand(@NotNull Map headers) { + CmdlineRemoteProto.Message.ControllerMessage.RequestParams requestParam = + CmdlineRemoteProto.Message.ControllerMessage.RequestParams.newBuilder().putAllAuthHeaders(headers).build(); + return CmdlineRemoteProto.Message.ControllerMessage.newBuilder() + .setType(CmdlineRemoteProto.Message.ControllerMessage.Type.AUTHENTICATION_TOKEN) + .setRequestParams(requestParam).build(); + } + public static CmdlineRemoteProto.Message.ControllerMessage createCancelCommand() { return CmdlineRemoteProto.Message.ControllerMessage.newBuilder() .setType(CmdlineRemoteProto.Message.ControllerMessage.Type.CANCEL_BUILD_COMMAND).build(); @@ -241,6 +249,10 @@ public final class CmdlineProtoUtil { return BuilderMessage.newBuilder().setType(BuilderMessage.Type.PARAM_REQUEST).build(); } + public static BuilderMessage createAuthTokenRequest() { + return BuilderMessage.newBuilder().setType(BuilderMessage.Type.AUTH_TOKEN_REQUEST).build(); + } + public static CmdlineRemoteProto.Message toMessage(UUID sessionId, BuilderMessage builderMessage) { return CmdlineRemoteProto.Message.newBuilder().setSessionId(toProtoUUID(sessionId)).setType(CmdlineRemoteProto.Message.Type.BUILDER_MESSAGE).setBuilderMessage(builderMessage).build(); } diff --git a/jps/jps-builders/src/org/jetbrains/jps/cache/JpsCacheStartupActivity.java b/jps/jps-builders/src/org/jetbrains/jps/cache/JpsCacheStartupActivity.java new file mode 100644 index 000000000000..02170647cd3d --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/cache/JpsCacheStartupActivity.java @@ -0,0 +1,76 @@ +//package org.jetbrains.jps.cache; +// +//import com.intellij.compiler.CompilerWorkspaceConfiguration; +//import com.intellij.compiler.server.BuildManager; +//import com.intellij.ide.browsers.BrowserLauncher; +//import com.intellij.ide.impl.TrustedProjects; +//import com.intellij.ide.util.PropertiesComponent; +//import com.intellij.jps.cache.git.GitRepositoryUtil; +//import com.intellij.notification.NotificationAction; +//import com.intellij.notification.NotificationType; +//import com.intellij.openapi.project.Project; +//import com.intellij.openapi.startup.StartupActivity; +//import com.intellij.openapi.util.SystemInfo; +//import git4idea.repo.GitRepository; +//import org.jetbrains.annotations.NotNull; +// +//import static com.intellij.jps.cache.JpsCachesPluginUtil.INTELLIJ_REPO_NAME; +//import static com.intellij.jps.cache.ui.JpsLoaderNotifications.ATTENTION; +// +//public final class JpsCacheStartupActivity implements StartupActivity.DumbAware { +// private static final String NOT_ASK_AGAIN = "JpsCaches.NOT_ASK_AGAIN"; +// private static boolean lineEndingsConfiguredCorrectly = true; +// +// @Override +// public void runActivity(@NotNull Project project) { +// checkWindowsCRLF(project); +// checkAutoBuildEnabled(project); +// } +// +// private static void checkWindowsCRLF(@NotNull Project project) { +// if (!SystemInfo.isWindows) return; +// GitRepository intellijRepository = GitRepositoryUtil.getRepositoryByName(project, INTELLIJ_REPO_NAME); +// if (intellijRepository == null) return; +// if (!GitRepositoryUtil.isAutoCrlfSetRight(intellijRepository)) { +// lineEndingsConfiguredCorrectly = false; +// ATTENTION +// .createNotification(JpsCacheBundle.message("notification.title.git.crlf.config"), +// JpsCacheBundle.message("notification.content.git.crlf.config"), +// NotificationType.WARNING) +// .addAction(NotificationAction.createSimple(JpsCacheBundle.message("notification.action.git.crlf.config"), +// () -> BrowserLauncher.getInstance().open("https://confluence.jetbrains.com/pages/viewpage.action?title=Git+Repository&spaceKey=IDEA"))) +// .notify(project); +// } +// } +// +// private static void checkAutoBuildEnabled(@NotNull Project project) { +// PropertiesComponent propertiesComponent = PropertiesComponent.getInstance(project); +// if (propertiesComponent.getBoolean(NOT_ASK_AGAIN)) { +// return; +// } +// +// CompilerWorkspaceConfiguration workspaceConfiguration = CompilerWorkspaceConfiguration.getInstance(project); +// if (!workspaceConfiguration.MAKE_PROJECT_ON_SAVE || !TrustedProjects.isTrusted(project)) { +// return; +// } +// +// ATTENTION +// .createNotification(JpsCacheBundle.message("notification.title.automatic.project.build.enabled"), +// JpsCacheBundle.message("notification.content.make.project.automatically.enabled.affect.caches"), +// NotificationType.WARNING) +// .addAction(NotificationAction.createSimpleExpiring( +// JpsCacheBundle.message("action.NotificationAction.JpsCachesDummyProjectComponent.text.disable.property"), () -> { +// workspaceConfiguration.MAKE_PROJECT_ON_SAVE = false; +// BuildManager.getInstance().clearState(project); +// })) +// .addAction(NotificationAction.createSimpleExpiring( +// JpsCacheBundle.message("action.NotificationAction.JpsCachesDummyProjectComponent.text.dont.ask"), () -> { +// propertiesComponent.setValue(NOT_ASK_AGAIN, true); +// })) +// .notify(project); +// } +// +// public static boolean isLineEndingsConfiguredCorrectly() { +// return lineEndingsConfiguredCorrectly; +// } +//} diff --git a/jps/jps-builders/src/org/jetbrains/jps/cache/JpsCachesPluginUtil.java b/jps/jps-builders/src/org/jetbrains/jps/cache/JpsCachesPluginUtil.java new file mode 100644 index 000000000000..9442e18983aa --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/cache/JpsCachesPluginUtil.java @@ -0,0 +1,23 @@ +package org.jetbrains.jps.cache; + +import com.intellij.openapi.diagnostic.Logger; +import com.intellij.util.concurrency.AppExecutorUtil; + +import java.util.concurrent.ExecutorService; + +import static com.intellij.execution.process.ProcessIOExecutorService.INSTANCE; + +public final class JpsCachesPluginUtil { + private static final Logger LOG = Logger.getInstance(JpsCachesPluginUtil.class); + public static final String PLUGIN_NAME = "jps-cache-loader"; + public static final String INTELLIJ_REPO_NAME = "intellij.git"; + public static final ExecutorService EXECUTOR_SERVICE = AppExecutorUtil.createBoundedApplicationPoolExecutor("JpsCacheLoader Pool", + INSTANCE, getThreadPoolSize()); + private JpsCachesPluginUtil() {} + + private static int getThreadPoolSize() { + int threadsCount = Runtime.getRuntime().availableProcessors() - 1; + LOG.info("Executor service will be configured with " + threadsCount + " threads"); + return threadsCount; + } +} diff --git a/jps/jps-builders/src/org/jetbrains/jps/cache/action/JpsForceUpdateCachesAction.java b/jps/jps-builders/src/org/jetbrains/jps/cache/action/JpsForceUpdateCachesAction.java new file mode 100644 index 000000000000..a45410748ab1 --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/cache/action/JpsForceUpdateCachesAction.java @@ -0,0 +1,18 @@ +//package org.jetbrains.jps.cache.action; +// +//import com.intellij.jps.cache.client.JpsServerAuthExtension; +//import com.intellij.jps.cache.loader.JpsOutputLoaderManager; +//import com.intellij.openapi.actionSystem.AnActionEvent; +//import com.intellij.openapi.project.DumbAwareAction; +//import com.intellij.openapi.project.Project; +//import org.jetbrains.annotations.NotNull; +// +//public class JpsForceUpdateCachesAction extends DumbAwareAction { +// @Override +// public void actionPerformed(@NotNull AnActionEvent actionEvent) { +// Project project = actionEvent.getProject(); +// if (project == null) return; +// JpsOutputLoaderManager outputLoaderManager = JpsOutputLoaderManager.getInstance(project); +// JpsServerAuthExtension.checkAuthenticatedInBackgroundThread(outputLoaderManager, project, () -> outputLoaderManager.load(true, true)); +// } +//} diff --git a/jps/jps-builders/src/org/jetbrains/jps/cache/action/JpsUpdateCachesAction.java b/jps/jps-builders/src/org/jetbrains/jps/cache/action/JpsUpdateCachesAction.java new file mode 100644 index 000000000000..fd6a425d06d6 --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/cache/action/JpsUpdateCachesAction.java @@ -0,0 +1,18 @@ +//package org.jetbrains.jps.cache.action; +// +//import com.intellij.jps.cache.client.JpsServerAuthExtension; +//import com.intellij.jps.cache.loader.JpsOutputLoaderManager; +//import com.intellij.openapi.actionSystem.AnActionEvent; +//import com.intellij.openapi.project.DumbAwareAction; +//import com.intellij.openapi.project.Project; +//import org.jetbrains.annotations.NotNull; +// +//public class JpsUpdateCachesAction extends DumbAwareAction { +// @Override +// public void actionPerformed(@NotNull AnActionEvent actionEvent) { +// Project project = actionEvent.getProject(); +// if (project == null) return; +// JpsOutputLoaderManager outputLoaderManager = JpsOutputLoaderManager.getInstance(project); +// JpsServerAuthExtension.checkAuthenticatedInBackgroundThread(outputLoaderManager, project, () -> outputLoaderManager.load(false, true)); +// } +//} \ No newline at end of file diff --git a/jps/jps-builders/src/org/jetbrains/jps/cache/client/JpsCachesDownloader.java b/jps/jps-builders/src/org/jetbrains/jps/cache/client/JpsCachesDownloader.java new file mode 100644 index 000000000000..57657547dfc5 --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/cache/client/JpsCachesDownloader.java @@ -0,0 +1,253 @@ +//package org.jetbrains.jps.cache.client; +// +//import com.google.gson.Gson; +//import com.google.gson.reflect.TypeToken; +//import com.intellij.ide.IdeCoreBundle; +//import com.intellij.internal.statistic.eventLog.events.EventId1; +//import com.intellij.jps.cache.client.JpsServerConnectionUtil; +//import com.intellij.jps.cache.ui.SegmentedProgressIndicatorManager; +//import com.intellij.openapi.diagnostic.Logger; +//import com.intellij.openapi.progress.ProcessCanceledException; +//import com.intellij.openapi.progress.ProgressIndicator; +//import com.intellij.openapi.util.Pair; +//import com.intellij.openapi.util.io.FileUtil; +//import com.intellij.openapi.util.io.NioFiles; +//import com.intellij.openapi.util.io.StreamUtil; +//import com.intellij.openapi.util.registry.Registry; +//import com.intellij.openapi.util.text.StringUtil; +//import com.intellij.util.download.DownloadableFileDescription; +//import com.intellij.util.io.CountingGZIPInputStream; +//import com.intellij.util.io.HttpRequests; +//import org.apache.http.Header; +//import org.apache.http.HttpEntity; +//import org.apache.http.HttpResponse; +//import org.apache.http.client.methods.HttpGet; +//import org.apache.http.impl.client.CloseableHttpClient; +//import org.apache.http.impl.client.HttpClientBuilder; +//import org.jetbrains.annotations.NotNull; +//import org.jetbrains.annotations.Nullable; +//import org.jetbrains.jps.cache.model.DownloadableFileUrl; +//import org.jetbrains.jps.cache.ui.SegmentedProgressIndicatorManager; +// +//import java.io.*; +//import java.lang.reflect.Type; +//import java.net.HttpURLConnection; +//import java.net.URLConnection; +//import java.nio.charset.StandardCharsets; +//import java.nio.file.Files; +//import java.nio.file.Path; +//import java.util.ArrayList; +//import java.util.List; +//import java.util.Map; +//import java.util.concurrent.CopyOnWriteArrayList; +//import java.util.concurrent.ExecutionException; +//import java.util.concurrent.Future; +//import java.util.concurrent.atomic.AtomicLong; +// +//import static org.jetbrains.jps.cache.JpsCachesPluginUtil.EXECUTOR_SERVICE; +// +//class JpsCachesDownloader { +// private static final Logger LOG = Logger.getInstance(JpsCachesDownloader.class); +// private static final byte MAX_RETRY_COUNT = 3; +// private static final String CDN_CACHE_HEADER = "X-Cache"; +// private int hitsCount = 0; +// private final List myFilesDescriptions; +// private final SegmentedProgressIndicatorManager myProgressIndicatorManager; +// +// JpsCachesDownloader(@NotNull List filesDescriptions, +// @NotNull SegmentedProgressIndicatorManager indicatorManager) { +// myFilesDescriptions = filesDescriptions; +// myProgressIndicatorManager = indicatorManager; +// } +// +// @NotNull +// List> download(@NotNull File targetDir, @Nullable EventId1 eventId) throws IOException { +// List> downloadedFiles = new CopyOnWriteArrayList<>(); +// List> existingFiles = new CopyOnWriteArrayList<>(); +// +// try { +// myProgressIndicatorManager.setText(this, IdeCoreBundle.message("progress.downloading.0.files.text", myFilesDescriptions.size())); +// long start = System.currentTimeMillis(); +// List> results = new ArrayList<>(); +// final AtomicLong totalSize = new AtomicLong(); +// for (final DownloadableFileUrl description : myFilesDescriptions) { +// results.add(EXECUTOR_SERVICE.submit(() -> { +// SegmentedProgressIndicatorManager.SubTaskProgressIndicator indicator = myProgressIndicatorManager.createSubTaskIndicator(); +// indicator.checkCanceled(); +// +// final File existing = new File(targetDir, description.getDefaultFileName()); +// byte attempt = 0; +// File downloaded = null; +// while (downloaded == null && attempt++ < MAX_RETRY_COUNT) { +// try { +// downloaded = downloadFile(description, existing, indicator); +// } catch (IOException e) { +// int httpStatusCode = -1; +// //if (e instanceof HttpRequests.HttpStatusException) { +// // httpStatusCode = ((HttpRequests.HttpStatusException)e).getStatusCode(); +// // if (httpStatusCode == 404) { +// // LOG.info("File not found to download " + description.getDownloadUrl()); +// // indicator.finished(); +// // return null; +// // } +// //} else { +// // if (Registry.is("jps.cache.check.internet.connection")){ +// // JpsServerConnectionUtil.checkDomainIsReachable("google.com"); +// // JpsServerConnectionUtil.checkDomainIsReachable("d1lc5k9lerg6km.cloudfront.net"); +// // JpsServerConnectionUtil.checkDomainRouting("d1lc5k9lerg6km.cloudfront.net"); +// // } +// //} +// +// // If max attempt count exceeded, rethrow exception further +// if (attempt != MAX_RETRY_COUNT) { +// if (httpStatusCode != -1) { +// LOG.info("Failed to download " + description.getDownloadUrl() + " HTTP code: " + httpStatusCode + ". Attempt " + attempt + " to download file again"); +// } else { +// LOG.info("Failed to download " + description.getDownloadUrl() + " Root cause: " + e + ". Attempt " + attempt + " to download file again"); +// } +// Thread.sleep(250); +// } else { +// throw new IOException(IdeCoreBundle.message("error.file.download.failed", description.getDownloadUrl(), e.getMessage()), e); +// } +// } +// } +// +// assert downloaded != null : "Download result shouldn't be NULL"; +// if (FileUtil.filesEqual(downloaded, existing)) { +// existingFiles.add(Pair.create(existing, description)); +// } +// else { +// totalSize.addAndGet(downloaded.length()); +// downloadedFiles.add(Pair.create(downloaded, description)); +// } +// indicator.finished(); +// return null; +// })); +// } +// +// for (Future result : results) { +// try { +// result.get(); +// } +// catch (InterruptedException e) { +// throw new ProcessCanceledException(); +// } +// catch (ExecutionException e) { +// if (e.getCause() instanceof IOException) { +// throw ((IOException)e.getCause()); +// } +// if (e.getCause() instanceof ProcessCanceledException) { +// throw ((ProcessCanceledException)e.getCause()); +// } +// LOG.error(e); +// } +// } +// long duration = System.currentTimeMillis() - start; +// if (eventId != null) eventId.log(totalSize.get()); +// LOG.info("Downloaded " + StringUtil.formatFileSize(totalSize.get()) + " in " + StringUtil.formatDuration(duration) + +// "(" + duration + "ms). Percentage of CDN cache hits: " + (hitsCount * 100/myFilesDescriptions.size()) + "%"); +// +// List> localFiles = new ArrayList<>(); +// localFiles.addAll(moveToDir(downloadedFiles, targetDir)); +// localFiles.addAll(existingFiles); +// myProgressIndicatorManager.finished(this); +// return localFiles; +// } +// catch (ProcessCanceledException | IOException e) { +// for (Pair pair : downloadedFiles) { +// FileUtil.delete(pair.getFirst()); +// } +// throw e; +// } +// } +// +// @NotNull +// private File downloadFile(@NotNull final DownloadableFileUrl description, @NotNull final File existingFile, +// @NotNull final ProgressIndicator indicator) throws IOException { +// final String presentableUrl = description.getPresentableDownloadUrl(); +// Map headers = JpsServerAuthUtil.getRequestHeaders(); +// //indicator.setText2(IdeCoreBundle.message("progress.connecting.to.download.file.text", presentableUrl)); +// //indicator.setIndeterminate(false); +// +// try (CloseableHttpClient client = HttpClientBuilder.create().build()) { +// HttpGet httpRequest = new HttpGet(description.getDownloadUrl()); +// headers.forEach((k, v) -> httpRequest.setHeader(k, v)); +// HttpResponse response = client.execute(httpRequest); +// HttpEntity responseEntity = response.getEntity(); +// if (response.getStatusLine().getStatusCode() == 200) { +// long size = responseEntity.getContentLength(); +// if (existingFile.exists() && size == existingFile.length()) { +// return existingFile; +// } +// +// Header header = response.getFirstHeader(CDN_CACHE_HEADER); +// if (header != null && header.getValue().startsWith("Hit")) hitsCount++; +// +// //indicator.setText2(IdeCoreBundle.message("progress.download.file.text", description.getPresentableFileName(), presentableUrl)); +// return saveToFile(FileUtil.createTempFile("download.", ".tmp").toPath(), responseEntity).toFile(); +// } +// else { +// String errorText = StreamUtil.readText(new InputStreamReader(responseEntity.getContent(), StandardCharsets.UTF_8)); +// throw new IOException("Request: " + description.getDownloadUrl() + " Error: " + response.getStatusLine().getStatusCode() + " body: " + errorText); +// } +// } +// } +// +// private static List> moveToDir(List> downloadedFiles, +// final File targetDir) throws IOException { +// FileUtil.createDirectory(targetDir); +// List> result = new ArrayList<>(); +// for (Pair pair : downloadedFiles) { +// final DownloadableFileUrl description = pair.getSecond(); +// final String fileName = description.generateFileName(s -> !new File(targetDir, s).exists()); +// final File toFile = new File(targetDir, fileName); +// FileUtil.rename(pair.getFirst(), toFile); +// result.add(Pair.create(toFile, description)); +// } +// return result; +// } +// +// public @NotNull Path saveToFile(@NotNull Path file, HttpEntity responseEntity) throws IOException { +// NioFiles.createDirectories(file.getParent()); +// +// boolean deleteFile = true; +// try (OutputStream out = Files.newOutputStream(file)) { +// copyStreamContent(responseEntity.getContent(), out, responseEntity.getContentLength()); +// deleteFile = false; +// } +// finally { +// if (deleteFile) { +// Files.deleteIfExists(file); +// } +// } +// +// return file; +// } +// +// private long copyStreamContent(@NotNull InputStream inputStream, +// @NotNull OutputStream outputStream, +// long expectedContentLength) throws IOException, ProcessCanceledException { +// +// CountingGZIPInputStream gzipStream = inputStream instanceof CountingGZIPInputStream ? (CountingGZIPInputStream)inputStream : null; +// byte[] buffer = new byte[StreamUtil.BUFFER_SIZE]; +// int count; +// long bytesWritten = 0; +// long bytesRead = 0; +// while ((count = inputStream.read(buffer)) > 0) { +// outputStream.write(buffer, 0, count); +// bytesWritten += count; +// bytesRead = gzipStream != null ? gzipStream.getCompressedBytesRead() : bytesWritten; +// +// } +// if (gzipStream != null) { +// // Amount of read bytes may have changed when 'inputStream.read(buffer)' returns -1 +// // E.g. reading GZIP trailer doesn't produce inflated stream data. +// bytesRead = gzipStream.getCompressedBytesRead(); +// } +// +// if (bytesRead < expectedContentLength) { +// throw new IOException("Connection closed at byte " + bytesRead + ". Expected " + expectedContentLength + " bytes."); +// } +// return bytesWritten; +// } +//} \ No newline at end of file diff --git a/jps/jps-builders/src/org/jetbrains/jps/cache/client/JpsServerAuthExtension.java b/jps/jps-builders/src/org/jetbrains/jps/cache/client/JpsServerAuthExtension.java new file mode 100644 index 000000000000..71b310cc77ec --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/cache/client/JpsServerAuthExtension.java @@ -0,0 +1,79 @@ +//package org.jetbrains.jps.cache.client; +// +//import com.intellij.jps.cache.JpsCacheBundle; +//import com.intellij.notification.NotificationListener; +//import com.intellij.notification.NotificationType; +//import com.intellij.openapi.Disposable; +//import com.intellij.openapi.application.ApplicationManager; +//import com.intellij.openapi.diagnostic.Logger; +//import com.intellij.openapi.extensions.ExtensionPointName; +//import com.intellij.openapi.project.Project; +//import com.intellij.openapi.util.Disposer; +//import com.intellij.openapi.util.Key; +//import org.jetbrains.annotations.NotNull; +//import org.jetbrains.annotations.Nullable; +// +//import java.util.Map; +// +//import static com.intellij.execution.process.ProcessIOExecutorService.INSTANCE; +//import static com.intellij.jps.cache.ui.JpsLoaderNotifications.ATTENTION; +// +///** +// * Extension point which provides authentication data for requests to the JPS cache server +// */ +//public interface JpsServerAuthExtension { +// Logger LOG = Logger.getInstance(JpsServerAuthExtension.class); +// Key NOTIFICATION_SHOWN_KEY = Key.create("AUTH_NOTIFICATION_SHOWN"); +// ExtensionPointName EP_NAME = ExtensionPointName.create("com.intellij.jpsServerAuthExtension"); +// +// /** +// * This method should check if the user was authenticated, if not it should do any needed actions to provide +// * auth token for further requests. This method will be called outside of the EDT and should be asynchronous. +// * If the user was authenticated the callback should be invoked. +// * +// * @param presentableReason reason for the token request +// * @param parentDisposable controls the lifetime of the authentication +// * @param onAuthCompleted callback on authentication complete, if token already exists it also should be invoked +// */ +// void checkAuthenticated(@NotNull String presentableReason, @NotNull Disposable parentDisposable, @NotNull Runnable onAuthCompleted); +// +// /** +// * The method provides HTTP authentication headers for the requests to the server. +// * It will be called in the background thread. The assertion that thread isn't EDT can +// * be added to the implementation. If it's not possible to get the authentication headers, +// * empty map or `null` can be return. +// * @return +// */ +// Map getAuthHeader(); +// +// @Nullable +// static JpsServerAuthExtension getInstance() { +// return EP_NAME.extensions().findFirst().orElse(null); +// } +// +// static void checkAuthenticatedInBackgroundThread(@NotNull Disposable parentDisposable, @NotNull Project project, @NotNull Runnable onAuthCompleted) { +// Disposable disposable = Disposer.newDisposable(); +// Disposer.register(parentDisposable, disposable); +// JpsServerAuthExtension authExtension = getInstance(); +// if (authExtension == null) { +// Boolean userData = project.getUserData(NOTIFICATION_SHOWN_KEY); +// if (userData == null) { +// project.putUserData(NOTIFICATION_SHOWN_KEY, Boolean.TRUE); +// ApplicationManager.getApplication().invokeLater(() -> { +// ATTENTION +// .createNotification(JpsCacheBundle.message("notification.title.jps.caches.downloader"), JpsCacheBundle.message("notification.content.internal.authentication.plugin.required.for.correct.work.plugin"), NotificationType.WARNING) +// .setListener(NotificationListener.URL_OPENING_LISTENER) +// .notify(project); +// }); +// } +// LOG.warn("JetBrains Internal Authentication plugin is required for the correct work. Please enable it."); +// return; +// } +// INSTANCE.execute(() -> { +// authExtension.checkAuthenticated("Jps Caches Downloader", disposable, () -> { +// Disposer.dispose(disposable); +// onAuthCompleted.run(); +// }); +// }); +// } +//} diff --git a/jps/jps-builders/src/org/jetbrains/jps/cache/client/JpsServerAuthUtil.java b/jps/jps-builders/src/org/jetbrains/jps/cache/client/JpsServerAuthUtil.java new file mode 100644 index 000000000000..de988452ef93 --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/cache/client/JpsServerAuthUtil.java @@ -0,0 +1,25 @@ +//package org.jetbrains.jps.cache.client; +// +//import com.intellij.jps.cache.JpsCacheBundle; +//import com.intellij.jps.cache.client.JpsServerAuthExtension; +//import org.apache.http.message.BasicHeader; +//import org.jetbrains.annotations.NotNull; +//import org.jetbrains.jps.cache.JpsCacheBundle; +// +//import java.util.Map; +// +//final class JpsServerAuthUtil { +// static @NotNull Map getRequestHeaders() { +// JpsServerAuthExtension authExtension = JpsServerAuthExtension.getInstance(); +// if (authExtension == null) { +// String message = JpsCacheBundle.message("notification.content.internal.authentication.plugin.required.for.correct.work.plugin"); +// throw new RuntimeException(message); +// } +// Map authHeader = authExtension.getAuthHeader(); +// if (authHeader == null) { +// String message = JpsCacheBundle.message("internal.authentication.plugin.missing.token"); +// throw new RuntimeException(message); +// } +// return authHeader; +// } +//} diff --git a/jps/jps-builders/src/org/jetbrains/jps/cache/client/JpsServerClient.java b/jps/jps-builders/src/org/jetbrains/jps/cache/client/JpsServerClient.java new file mode 100644 index 000000000000..47afd37b237f --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/cache/client/JpsServerClient.java @@ -0,0 +1,26 @@ +//package org.jetbrains.jps.cache.client; +// +//import org.jetbrains.annotations.NotNull; +//import org.jetbrains.annotations.Nullable; +//import org.jetbrains.jps.cache.model.AffectedModule; +//import org.jetbrains.jps.cache.model.OutputLoadResult; +//import org.jetbrains.jps.cache.ui.SegmentedProgressIndicatorManager; +// +//import java.io.File; +//import java.util.List; +//import java.util.Map; +//import java.util.Set; +// +//public interface JpsServerClient { +// @NotNull +// Map> getCacheKeysPerRemote(); +// @Nullable +// File downloadMetadataById(@NotNull String metadataId, @NotNull File targetDir); +// File downloadCacheById(@NotNull SegmentedProgressIndicatorManager downloadIndicatorManager, @NotNull String cacheId, +// @NotNull File targetDir); +// List downloadCompiledModules(@NotNull SegmentedProgressIndicatorManager downloadIndicatorManager, +// @NotNull List affectedModules); +// static JpsServerClient getServerClient() { +// return JpsServerClientImpl.INSTANCE; +// } +//} \ No newline at end of file diff --git a/jps/jps-builders/src/org/jetbrains/jps/cache/client/JpsServerClientImpl.java b/jps/jps-builders/src/org/jetbrains/jps/cache/client/JpsServerClientImpl.java new file mode 100644 index 000000000000..2ca4b15b5a64 --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/cache/client/JpsServerClientImpl.java @@ -0,0 +1,191 @@ +//package org.jetbrains.jps.cache.client; +// +//import com.google.gson.Gson; +//import com.google.gson.reflect.TypeToken; +//import com.intellij.jps.cache.git.GitRepositoryUtil; +//import com.intellij.openapi.application.PathManager; +//import com.intellij.openapi.diagnostic.Logger; +//import com.intellij.openapi.progress.ProcessCanceledException; +//import com.intellij.openapi.progress.ProgressIndicator; +//import com.intellij.openapi.progress.ProgressManager; +//import com.intellij.openapi.util.Pair; +//import com.intellij.openapi.util.io.FileUtil; +//import com.intellij.openapi.util.io.StreamUtil; +//import com.intellij.util.containers.ContainerUtil; +//import org.apache.http.HttpEntity; +//import org.apache.http.HttpResponse; +//import org.apache.http.client.methods.HttpGet; +//import org.apache.http.impl.client.CloseableHttpClient; +//import org.apache.http.impl.client.HttpClientBuilder; +//import org.jetbrains.annotations.NotNull; +//import org.jetbrains.annotations.Nullable; +//import org.jetbrains.jps.cache.JpsCachesPluginUtil; +//import org.jetbrains.jps.cache.model.AffectedModule; +//import org.jetbrains.jps.cache.model.DownloadableFileUrl; +//import org.jetbrains.jps.cache.model.OutputLoadResult; +//import org.jetbrains.jps.cache.ui.SegmentedProgressIndicatorManager; +// +//import java.io.File; +//import java.io.IOException; +//import java.io.InputStream; +//import java.io.InputStreamReader; +//import java.lang.reflect.Type; +//import java.nio.charset.StandardCharsets; +//import java.util.*; +//import java.util.stream.Collectors; +// +//import static com.intellij.jps.cache.statistics.JpsCacheUsagesCollector.DOWNLOAD_BINARY_SIZE_EVENT_ID; +//import static com.intellij.jps.cache.statistics.JpsCacheUsagesCollector.DOWNLOAD_CACHE_SIZE_EVENT_ID; +// +//public final class JpsServerClientImpl implements JpsServerClient { +// private static final Logger LOG = Logger.getInstance(JpsServerClientImpl.class); +// private static final Type GSON_MAPPER = new TypeToken>>() {}.getType(); +// static final JpsServerClientImpl INSTANCE = new JpsServerClientImpl(); +// private final String stringThree; +// +// private JpsServerClientImpl() { +// byte[] decodedBytes = Base64.getDecoder().decode("aHR0cHM6Ly9kMWxjNWs5bGVyZzZrbS5jbG91ZGZyb250Lm5ldA=="); +// stringThree = new String(decodedBytes, StandardCharsets.UTF_8); +// } +// +// @NotNull +// @Override +// public Map> getCacheKeysPerRemote() { +// Map> response = doGetRequest(); +// if (response == null) return Collections.emptyMap(); +// Map> result = new HashMap<>(); +// response.forEach((key, value) -> result.put(GitRepositoryUtil.getRemoteRepoName(key), new HashSet<>(value))); +// return result; +// } +// +// @Nullable +// @Override +// public File downloadMetadataById(@NotNull String metadataId, @NotNull File targetDir) { +// String downloadUrl = stringThree + "/metadata/" + metadataId; +// String fileName = "metadata.json"; +// DownloadableFileUrl description = new DownloadableFileUrl(downloadUrl, fileName); +// ProgressIndicator progressIndicator = ProgressManager.getInstance().getProgressIndicator(); +// JpsCachesDownloader downloader = new JpsCachesDownloader(Collections.singletonList(description), +// new SegmentedProgressIndicatorManager(progressIndicator)); +// +// LOG.debug("Downloading JPS metadata from: " + downloadUrl); +// File metadataFile; +// try { +// List> pairs = downloader.download(targetDir, null); +// Pair first = ContainerUtil.getFirstItem(pairs); +// metadataFile = first != null ? first.first : null; +// if (metadataFile == null) { +// LOG.warn("Failed to download JPS metadata"); +// return null; +// } +// return metadataFile; +// } +// catch (ProcessCanceledException | IOException e) { +// //noinspection InstanceofCatchParameter +// if (e instanceof IOException) LOG.warn("Failed to download JPS metadata from URL: " + downloadUrl, e); +// return null; +// } +// } +// +// @Nullable +// @Override +// public File downloadCacheById(@NotNull SegmentedProgressIndicatorManager downloadIndicatorManager, @NotNull String cacheId, +// @NotNull File targetDir) { +// String downloadUrl = stringThree + "/caches/" + cacheId; +// String fileName = "portable-build-cache.zip"; +// DownloadableFileUrl description = new DownloadableFileUrl(downloadUrl, fileName); +// JpsCachesDownloader downloader = new JpsCachesDownloader(Collections.singletonList(description), downloadIndicatorManager); +// +// LOG.debug("Downloading JPS caches from: " + downloadUrl); +// File zipFile; +// try { +// List> pairs = downloader.download(targetDir, DOWNLOAD_CACHE_SIZE_EVENT_ID); +// downloadIndicatorManager.finished(this); +// +// Pair first = ContainerUtil.getFirstItem(pairs); +// zipFile = first != null ? first.first : null; +// if (zipFile != null) return zipFile; +// LOG.warn("Failed to download JPS caches"); +// } +// catch (ProcessCanceledException | IOException e) { +// //noinspection InstanceofCatchParameter +// if (e instanceof IOException) LOG.warn("Failed to download JPS caches from URL: " + downloadUrl, e); +// } +// return null; +// } +// +// @Override +// public List downloadCompiledModules(@NotNull SegmentedProgressIndicatorManager downloadIndicatorManager, +// @NotNull List affectedModules) { +// File targetDir = new File(PathManager.getPluginTempPath(), JpsCachesPluginUtil.PLUGIN_NAME); +// if (targetDir.exists()) FileUtil.delete(targetDir); +// targetDir.mkdirs(); +// +// Map urlToModuleNameMap = affectedModules.stream().collect(Collectors.toMap( +// module -> stringThree + "/" + module.getType() + "/" + module.getName() + "/" + module.getHash(), +// module -> module)); +// +// List descriptions = ContainerUtil.map(urlToModuleNameMap.entrySet(), +// entry -> new DownloadableFileUrl(entry.getKey(), +// entry.getValue().getOutPath().getName() + ".zip")); +// JpsCachesDownloader downloader = new JpsCachesDownloader(descriptions, downloadIndicatorManager); +// +// List downloadedFiles = new ArrayList<>(); +// try { +// // Downloading process +// List> download = downloader.download(targetDir, DOWNLOAD_BINARY_SIZE_EVENT_ID); +// downloadIndicatorManager.finished(this); +// +// downloadedFiles = ContainerUtil.map(download, pair -> pair.first); +// return ContainerUtil.map(download, pair -> { +// String downloadUrl = pair.second.getDownloadUrl(); +// return new OutputLoadResult(pair.first, downloadUrl, urlToModuleNameMap.get(downloadUrl)); +// }); +// } +// catch (ProcessCanceledException | IOException e) { +// //noinspection InstanceofCatchParameter +// if (e instanceof IOException) LOG.warn("Failed to download JPS compilation outputs", e); +// if (targetDir.exists()) FileUtil.delete(targetDir); +// downloadedFiles.forEach(zipFile -> FileUtil.delete(zipFile)); +// return null; +// } +// } +// +// private @Nullable Map> doGetRequest() { +// Map headers = JpsServerAuthUtil.getRequestHeaders(); +// try (CloseableHttpClient client = HttpClientBuilder.create().build()) { +// String url = stringThree + "/commit_history.json"; +// HttpGet httpRequest = new HttpGet(url); +// headers.forEach((k, v) -> httpRequest.setHeader(k, v)); +// HttpResponse response = client.execute(httpRequest); +// HttpEntity responseEntity = response.getEntity(); +// if (response.getStatusLine().getStatusCode() == 200) { +// Gson gson = new Gson(); +// String contentEncoding = responseEntity.getContentEncoding().getName(); +// InputStream inputStream = responseEntity.getContent(); +// return gson.fromJson(new InputStreamReader(inputStream, contentEncoding), GSON_MAPPER); +// } +// else { +// String errorText = StreamUtil.readText(new InputStreamReader(responseEntity.getContent(), StandardCharsets.UTF_8)); +// LOG.info("Request: " + url + " Error: " + response.getStatusLine().getStatusCode() + " body: " + errorText); +// return null; +// } +// } +// catch (IOException e) { +// LOG.warn("Failed request to cache server", e); +// //JpsLoaderNotifications.ATTENTION +// // .createNotification(JpsCacheBundle.message("notification.title.compiler.caches.loader"), JpsCacheBundle.message("notification.content.failed.request.to.cache.server", e.getMessage()), NotificationType.ERROR) +// // .notify(project); +// } +// return null; +// } +// +// //private static InputStream getInputStream(HttpEntity responseEntity) throws IOException { +// // String contentEncoding = responseEntity.getContentEncoding().getName(); +// // InputStream inputStream = responseEntity.getContent(); +// // if (contentEncoding != null && StringUtil.toLowerCase(contentEncoding).contains("gzip")) { +// // return new GZIPInputStream(inputStream); +// // } +// // return new InputStreamReader(inputStream, contentEncoding); +// //} +//} diff --git a/jps/jps-builders/src/org/jetbrains/jps/cache/client/JpsServerConnectionUtil.java b/jps/jps-builders/src/org/jetbrains/jps/cache/client/JpsServerConnectionUtil.java new file mode 100644 index 000000000000..3ca678283b6e --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/cache/client/JpsServerConnectionUtil.java @@ -0,0 +1,128 @@ +//package org.jetbrains.jps.cache.client; +// +//import com.intellij.execution.ExecutionException; +//import com.intellij.execution.configurations.GeneralCommandLine; +//import com.intellij.execution.process.ProcessOutput; +//import com.intellij.execution.util.ExecUtil; +//import com.intellij.openapi.diagnostic.Logger; +//import com.intellij.openapi.progress.ProcessCanceledException; +//import com.intellij.openapi.progress.ProgressIndicator; +//import com.intellij.openapi.progress.util.ProgressWindow; +//import com.intellij.openapi.project.Project; +//import com.intellij.openapi.util.SystemInfo; +//import com.intellij.openapi.util.io.FileUtil; +//import com.intellij.openapi.util.text.StringUtil; +//import com.intellij.util.io.HttpRequests; +//import org.jetbrains.annotations.NotNull; +// +//import java.io.File; +//import java.io.IOException; +//import java.net.URLConnection; +//import java.nio.charset.StandardCharsets; +//import java.text.DecimalFormat; +//import java.util.Base64; +//import java.util.Map; +// +//import static com.intellij.execution.process.ProcessIOExecutorService.INSTANCE; +// +///** +// * This class was introduced to detect internet connection problems from the client side. +// * One of the cases was found, the constant connection refuse at caches download was related to +// * the internet provider issue. +// * +// * traceroute for the d1lc5k9lerg6km.cloudfront.net +// * 1 * * 192.168.0.1 (192.168.0.1) 2.183 ms +// * 2 100.92.128.1 (100.92.128.1) 9.851 ms 8.761 ms 9.509 ms +// * 3 100.127.1.253 (100.127.1.253) 10.748 ms 9.603 ms 13.316 ms +// * 4 212.48.195.38 (212.48.195.38) 30.396 ms 13.616 ms 9.258 ms +// * 5 100.64.97.116 (100.64.97.116) 9.306 ms !X 8.332 ms !X 14.542 ms !X +// */ +//public class JpsServerConnectionUtil { +// private static final Logger LOG = Logger.getInstance(JpsServerConnectionUtil.class); +// private static final String CDN_CACHE_HEADER = "X-Cache"; +// +// public static void measureConnectionSpeed(@NotNull Project project) { +// INSTANCE.execute(() -> { +// ProgressIndicator indicator = new ProgressWindow(true, project); +// indicator.setIndeterminate(false); +// indicator.setFraction(0.01); +// try { +// Map headers = JpsServerAuthUtil.getRequestHeaders(); +// long start = System.currentTimeMillis(); +// HttpRequests.request(calculateAddress()) +// .tuner(tuner -> headers.forEach((k, v) -> tuner.addRequestProperty(k, v))) +// .connect(new HttpRequests.RequestProcessor<>() { +// @Override +// public File process(@NotNull HttpRequests.Request request) throws IOException { +// URLConnection connection = request.getConnection(); +// int fileSize = connection.getContentLength(); +// String header = connection.getHeaderField(CDN_CACHE_HEADER); +// File downloadedFile = request.saveToFile(FileUtil.createTempFile("download.", ".tmp"), indicator); +// long downloadTime = System.currentTimeMillis() - start; +// long bytesPerSecond = fileSize / downloadTime * 1000; +// if (header != null && header.startsWith("Hit")) { +// LOG.info("Speed of connection to CDN: " + StringUtil.formatFileSize(bytesPerSecond) + "/s; " + formatInternetSpeed(bytesPerSecond * 8)); +// } else { +// LOG.info("Speed of connection to S3: " + StringUtil.formatFileSize(bytesPerSecond) + "/s; " + formatInternetSpeed(bytesPerSecond * 8)); +// } +// FileUtil.delete(downloadedFile); +// return downloadedFile; +// } +// }); +// } +// catch (ProcessCanceledException | IOException e) { +// LOG.warn("Failed to download file for measurement connection speed", e); +// } +// }); +// } +// +// public static void checkDomainIsReachable(@NotNull String domain) { +// try { +// GeneralCommandLine pingCommand = new GeneralCommandLine("ping"); +// if (SystemInfo.isWindows) { +// pingCommand.addParameter("-n 2"); +// } else { +// pingCommand.addParameter("-c 2"); +// } +// pingCommand.addParameter(domain); +// int code = ExecUtil.execAndGetOutput(pingCommand).getExitCode(); +// if (code == 0) { +// LOG.info("Domain " + domain + " is reachable"); +// } else { +// LOG.info("Domain " + domain + " isn't reachable"); +// } +// } +// catch (ExecutionException e) { +// LOG.warn("Failed to check if internet connection is available", e); +// } +// } +// +// public static void checkDomainRouting(@NotNull String domain) { +// try { +// GeneralCommandLine pingCommand = new GeneralCommandLine("traceroute"); +// pingCommand.addParameter(domain); +// ProcessOutput processOutput = ExecUtil.execAndGetOutput(pingCommand); +// int code = processOutput.getExitCode(); +// if (code == 0) { +// LOG.info("traceroute for the " + domain + "\n" + processOutput.getStdout()); +// } else { +// LOG.info("traceroute failed with exception " + processOutput.getStderr()); +// } +// } +// catch (ExecutionException e) { +// LOG.warn("Failed to execute traceroute", e); +// } +// } +// +// private static @NotNull String formatInternetSpeed(long fileSize) { +// int rank = (int)((Math.log10(fileSize) + 0.0000021714778384307465) / 3); // (3 - Math.log10(999.995)) +// double value = fileSize / Math.pow(1000, rank); +// String[] units = {"Bit", "Kbit", "Mbit", "Gbit"}; +// return new DecimalFormat("0.##").format(value) + units[rank] + "/s"; +// } +// +// private static @NotNull String calculateAddress() { +// byte[] decodedBytes = Base64.getDecoder().decode("aHR0cHM6Ly9kMWxjNWs5bGVyZzZrbS5jbG91ZGZyb250Lm5ldC9FWEFNUExFLnR4dA=="); +// return new String(decodedBytes, StandardCharsets.UTF_8); +// } +//} \ No newline at end of file diff --git a/jps/jps-builders/src/org/jetbrains/jps/cache/git/GitCommitsIterator.java b/jps/jps-builders/src/org/jetbrains/jps/cache/git/GitCommitsIterator.java new file mode 100644 index 000000000000..40c0aec113c5 --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/cache/git/GitCommitsIterator.java @@ -0,0 +1,84 @@ +//package org.jetbrains.jps.cache.git; +// +//import com.intellij.openapi.diagnostic.Logger; +//import com.intellij.openapi.project.Project; +//import com.intellij.openapi.vcs.VcsException; +//import com.intellij.util.SmartList; +//import com.intellij.util.containers.ContainerUtil; +//import git4idea.history.GitHistoryUtils; +//import git4idea.repo.GitRepository; +//import org.jetbrains.annotations.NotNull; +// +//import java.util.Iterator; +//import java.util.List; +//import java.util.NoSuchElementException; +// +//public class GitCommitsIterator implements Iterator { +// private static final Logger LOG = Logger.getInstance(GitCommitsIterator.class); +// +// private static final int MAX_FETCH_SIZE = 1000; +// private static final int FETCH_SIZE = 100; +// private final GitRepository myRepository; +// private final Project myProject; +// private int fetchedCount; +// private List commits; +// private String remote; +// private int currentPosition; +// +// public GitCommitsIterator(@NotNull Project project, @NotNull GitRepository repository, @NotNull String remoteUrl) { +// myRepository = repository; +// myProject = project; +// fetchedCount = 0; +// remote = remoteUrl; +// fetchOldCommits(); +// } +// +// @Override +// public boolean hasNext() { +// if (commits.size() > 0) { +// if (currentPosition < commits.size()) return true; +// if (fetchedCount >= MAX_FETCH_SIZE) { +// LOG.info("Exceeded fetch limit for git commits"); +// return false; +// } +// fetchOldCommits(commits.get(currentPosition - 1)); +// if (commits.size() > 0) { +// currentPosition = 0; +// return true; +// } +// } +// return false; +// } +// +// @Override +// public String next() { +// if (commits.size() == 0 || currentPosition >= commits.size()) throw new NoSuchElementException(); +// String result = commits.get(currentPosition); +// currentPosition++; +// return result; +// } +// +// @NotNull +// public String getRemote() { +// return remote; +// } +// +// private void fetchOldCommits() { +// fetchOldCommits(""); +// } +// +// private void fetchOldCommits(String latestCommit) { +// try { +// commits = +// ContainerUtil.map(latestCommit.isEmpty() ? GitHistoryUtils.collectTimedCommits(myProject, myRepository.getRoot(), "-n " + FETCH_SIZE) : +// GitHistoryUtils.collectTimedCommits(myProject, myRepository.getRoot(), latestCommit, "-n " + FETCH_SIZE), +// it -> it.getId().asString()); +// fetchedCount += commits.size(); +// return; +// } +// catch (VcsException e) { +// LOG.warn("Can't get Git hashes for commits", e); +// } +// commits = new SmartList<>(); +// } +//} \ No newline at end of file diff --git a/jps/jps-builders/src/org/jetbrains/jps/cache/git/GitRepositoryUtil.java b/jps/jps-builders/src/org/jetbrains/jps/cache/git/GitRepositoryUtil.java new file mode 100644 index 000000000000..54b59538e621 --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/cache/git/GitRepositoryUtil.java @@ -0,0 +1,70 @@ +//package org.jetbrains.jps.cache.git; +// +//import com.intellij.jps.cache.git.GitCommitsIterator; +//import com.intellij.openapi.diagnostic.Logger; +//import com.intellij.openapi.project.Project; +//import com.intellij.util.containers.ContainerUtil; +//import git4idea.GitUtil; +//import git4idea.commands.Git; +//import git4idea.commands.GitCommandResult; +//import git4idea.config.GitConfigUtil; +//import git4idea.repo.GitRemote; +//import git4idea.repo.GitRepository; +//import org.jetbrains.annotations.NotNull; +//import org.jetbrains.annotations.Nullable; +// +//import java.util.*; +//import java.util.stream.Collectors; +// +//public final class GitRepositoryUtil { +// private static final Logger LOG = Logger.getInstance(GitRepositoryUtil.class); +// +// private GitRepositoryUtil() {} +// +// @NotNull +// public static List getCommitsIterator(@NotNull Project project, @NotNull Set remoteUrlNames) { +// if (GitUtil.hasGitRepositories(project)) { +// return GitUtil.getRepositories(project).stream() +// .map(repo -> { +// Set remoteUrls = repo.getRemotes().stream() +// .map(remote -> remote.getUrls()) +// .flatMap(Collection::stream) +// .collect(Collectors.toSet()); +// String matchedRemoteUrl = ContainerUtil.find(remoteUrls, remoteUrl -> remoteUrlNames.contains(getRemoteRepoName(remoteUrl))); +// if (matchedRemoteUrl == null) return null; +// return new GitCommitsIterator(project, repo, getRemoteRepoName(matchedRemoteUrl)); +// }).filter(Objects::nonNull) +// .collect(Collectors.toList()); +// } +// LOG.info("Project doesn't contain Git repository"); +// return Collections.emptyList(); +// } +// +// @Nullable +// public static GitRepository getRepositoryByName(@NotNull Project project, @NotNull String repositoryName) { +// if (GitUtil.hasGitRepositories(project)) { +// return ContainerUtil.find(GitUtil.getRepositories(project), repo -> { +// for (GitRemote remote : repo.getRemotes()) { +// for (String remoteUrl : remote.getUrls()) { +// if (getRemoteRepoName(remoteUrl).equals(repositoryName)) return true; +// } +// } +// return false; +// }); +// } +// LOG.info("Project doesn't contain Git repository"); +// return null; +// } +// +// public static String getRemoteRepoName(@NotNull String remoteUrl) { +// String[] splittedRemoteUrl = remoteUrl.split("/"); +// return splittedRemoteUrl[splittedRemoteUrl.length - 1]; +// } +// +// public static boolean isAutoCrlfSetRight(@NotNull GitRepository gitRepository) { +// GitCommandResult result = Git.getInstance().config(gitRepository, GitConfigUtil.CORE_AUTOCRLF); +// String value = result.getOutputAsJoinedString(); +// LOG.info("CRLF configuration for " + gitRepository + " project: " + value); +// return value.equalsIgnoreCase("input"); +// } +//} \ No newline at end of file diff --git a/jps/jps-builders/src/org/jetbrains/jps/cache/loader/JpsCacheLoader.java b/jps/jps-builders/src/org/jetbrains/jps/cache/loader/JpsCacheLoader.java new file mode 100644 index 000000000000..91e50431da1f --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/cache/loader/JpsCacheLoader.java @@ -0,0 +1,137 @@ +//package org.jetbrains.jps.cache.loader; +// +//import com.intellij.compiler.server.BuildManager; +//import com.intellij.jps.cache.JpsCacheBundle; +//import com.intellij.jps.cache.client.JpsServerClient; +//import com.intellij.jps.cache.loader.JpsOutputLoader; +//import com.intellij.jps.cache.model.JpsLoaderContext; +//import com.intellij.jps.cache.ui.SegmentedProgressIndicatorManager; +//import com.intellij.openapi.diagnostic.Logger; +//import com.intellij.openapi.progress.ProcessCanceledException; +//import com.intellij.openapi.project.Project; +//import com.intellij.openapi.util.io.FileUtil; +//import com.intellij.util.io.ZipUtil; +//import org.jetbrains.annotations.NotNull; +//import org.jetbrains.annotations.Nullable; +//import org.jetbrains.jps.cache.JpsCacheBundle; +//import org.jetbrains.jps.cache.client.JpsServerClient; +//import org.jetbrains.jps.cache.model.JpsLoaderContext; +//import org.jetbrains.jps.cache.ui.SegmentedProgressIndicatorManager; +// +//import java.io.File; +//import java.io.IOException; +// +//class JpsCacheLoader implements JpsOutputLoader { +// private static final Logger LOG = Logger.getInstance(JpsCacheLoader.class); +// private static final String TIMESTAMPS_FOLDER_NAME = "timestamps"; +// private static final String FS_STATE_FILE = "fs_state.dat"; +// private final BuildManager myBuildManager; +// private final JpsServerClient myClient; +// private final Project myProject; +// private File myTmpCacheFolder; +// +// JpsCacheLoader(JpsServerClient client, @NotNull Project project) { +// myBuildManager = BuildManager.getInstance(); +// myClient = client; +// myProject = project; +// } +// +// @Nullable +// @Override +// public File load(@NotNull JpsLoaderContext context) { +// LOG.info("Loading JPS caches for commit: " + context.getCommitId()); +// myTmpCacheFolder = null; +// +// long start = System.currentTimeMillis(); +// File zipFile = myClient.downloadCacheById(context.getDownloadIndicatorManager(), context.getCommitId(), +// myBuildManager.getBuildSystemDirectory(myProject).toFile()); +// LOG.info("Download of jps caches took: " + (System.currentTimeMillis() - start)); +// return zipFile; +// } +// +// @Override +// public LoaderStatus extract(@Nullable Object loadResults, @NotNull SegmentedProgressIndicatorManager extractIndicatorManager) { +// if (!(loadResults instanceof File)) return LoaderStatus.FAILED; +// +// File zipFile = (File)loadResults; +// File targetDir = myBuildManager.getBuildSystemDirectory(myProject).toFile(); +// File tmpFolder = new File(targetDir, "tmp"); +// try { +// // Start extracting after download +// SegmentedProgressIndicatorManager.SubTaskProgressIndicator subTaskIndicator = extractIndicatorManager.createSubTaskIndicator(); +// extractIndicatorManager.getProgressIndicator().checkCanceled(); +// extractIndicatorManager.setText(this, JpsCacheBundle.message("progress.text.extracting.downloaded.results")); +// subTaskIndicator.setText2(JpsCacheBundle.message("progress.details.extracting.project.caches")); +// long start = System.currentTimeMillis(); +// +// ZipUtil.extract(zipFile, tmpFolder, null); +// FileUtil.delete(zipFile); +// LOG.info("Unzip compilation caches took: " + (System.currentTimeMillis() - start)); +// subTaskIndicator.finished(); +// extractIndicatorManager.finished(this); +// +// myTmpCacheFolder = tmpFolder; +// return LoaderStatus.COMPLETE; +// } +// catch (ProcessCanceledException | IOException e) { +// if (e instanceof IOException) LOG.warn("Failed unzip downloaded compilation caches", e); +// FileUtil.delete(zipFile); +// FileUtil.delete(tmpFolder); +// } +// return LoaderStatus.FAILED; +// } +// +// @Override +// public void rollback() { +// if (myTmpCacheFolder != null && myTmpCacheFolder.exists()) { +// FileUtil.delete(myTmpCacheFolder); +// LOG.debug("JPS cache loader rolled back"); +// } +// } +// +// @Override +// public void apply(@NotNull SegmentedProgressIndicatorManager indicatorManager) { +// if (myTmpCacheFolder == null) { +// LOG.warn("Nothing to apply, download results are empty"); +// return; +// } +// +// File newTimestampFolder = new File(myTmpCacheFolder, TIMESTAMPS_FOLDER_NAME); +// if (newTimestampFolder.exists()) FileUtil.delete(newTimestampFolder); +// +// File currentDirForBuildCache = myBuildManager.getProjectSystemDirectory(myProject); +// indicatorManager.setText(this, JpsCacheBundle.message("progress.text.applying.jps.caches")); +// if (currentDirForBuildCache != null) { +// SegmentedProgressIndicatorManager.SubTaskProgressIndicator subTaskIndicator = indicatorManager.createSubTaskIndicator(); +// subTaskIndicator.setText2(JpsCacheBundle.message("progress.details.applying.downloaded.caches")); +// // Copy timestamp old folder to new cache dir +// File timestamps = new File(currentDirForBuildCache, TIMESTAMPS_FOLDER_NAME); +// if (timestamps.exists()) { +// try { +// newTimestampFolder.mkdirs(); +// FileUtil.copyDir(timestamps, newTimestampFolder); +// } +// catch (IOException e) { +// LOG.warn("Couldn't copy timestamps from old JPS cache", e); +// } +// } +// +// // Create new empty fsStateFile +// File fsStateFile = new File(myTmpCacheFolder, FS_STATE_FILE); +// fsStateFile.delete(); +// try { +// fsStateFile.createNewFile(); +// } +// catch (IOException e) { +// LOG.warn("Couldn't create new empty FsState file", e); +// } +// +// // Remove old cache dir +// FileUtil.delete(currentDirForBuildCache); +// myTmpCacheFolder.renameTo(currentDirForBuildCache); +// subTaskIndicator.finished(); +// LOG.debug("JPS cache downloads finished"); +// } +// indicatorManager.finished(this); +// } +//} \ No newline at end of file diff --git a/jps/jps-builders/src/org/jetbrains/jps/cache/loader/JpsCompilationOutputLoader.java b/jps/jps-builders/src/org/jetbrains/jps/cache/loader/JpsCompilationOutputLoader.java new file mode 100644 index 000000000000..e7bca0c5eb6b --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/cache/loader/JpsCompilationOutputLoader.java @@ -0,0 +1,357 @@ +//package org.jetbrains.jps.cache.loader; +// +//import com.google.common.hash.Hasher; +//import com.google.common.hash.Hashing; +//import com.intellij.openapi.diagnostic.Logger; +//import com.intellij.openapi.progress.ProcessCanceledException; +//import com.intellij.openapi.util.io.FileUtil; +//import com.intellij.util.ArrayUtil; +//import com.intellij.util.containers.ContainerUtil; +//import com.intellij.util.io.ZipUtil; +//import org.jetbrains.annotations.NotNull; +//import org.jetbrains.annotations.Nullable; +//import org.jetbrains.annotations.TestOnly; +//import org.jetbrains.jps.cache.JpsCacheBundle; +//import org.jetbrains.jps.cache.client.JpsServerClient; +//import org.jetbrains.jps.cache.model.AffectedModule; +//import org.jetbrains.jps.cache.model.BuildTargetState; +//import org.jetbrains.jps.cache.model.JpsLoaderContext; +//import org.jetbrains.jps.cache.model.OutputLoadResult; +//import org.jetbrains.jps.cache.ui.SegmentedProgressIndicatorManager; +// +//import java.io.File; +//import java.io.IOException; +//import java.nio.charset.StandardCharsets; +//import java.util.*; +//import java.util.concurrent.ConcurrentHashMap; +//import java.util.concurrent.ExecutionException; +//import java.util.concurrent.Future; +//import java.util.stream.Collectors; +// +//import static com.intellij.jps.cache.JpsCachesPluginUtil.EXECUTOR_SERVICE; +// +//class JpsCompilationOutputLoader implements JpsOutputLoader> { +// private static final Logger LOG = Logger.getInstance(JpsCompilationOutputLoader.class); +// private static final String RESOURCES_PRODUCTION = "resources-production"; +// private static final String JAVA_PRODUCTION = "java-production"; +// private static final String RESOURCES_TEST = "resources-test"; +// private static final String JAVA_TEST = "java-test"; +// private static final String PRODUCTION = "production"; +// private static final String TEST = "test"; +// private final JpsServerClient myClient; +// private final String myBuildDirPath; +// private List myOldModulesPaths; +// private Map myTmpFolderToModuleName; +// +// JpsCompilationOutputLoader(@NotNull JpsServerClient client, @NotNull String buildDirPath) { +// myClient = client; +// myBuildDirPath = buildDirPath; +// } +// +// @Override +// public int calculateDownloads(@NotNull Map> commitSourcesState, +// @Nullable Map> currentSourcesState) { +// return calculateAffectedModules(currentSourcesState, commitSourcesState, true).size(); +// } +// +// @Override +// public List load(@NotNull JpsLoaderContext context) { +// myOldModulesPaths = null; +// myTmpFolderToModuleName = null; +// +// SegmentedProgressIndicatorManager downloadProgressManager = context.getDownloadIndicatorManager(); +// downloadProgressManager.setText(this, JpsCacheBundle.message("progress.text.calculating.affected.modules")); +// List affectedModules = calculateAffectedModules(context.getCurrentSourcesState(), +// context.getCommitSourcesState(), true); +// downloadProgressManager.finished(this); +// downloadProgressManager.getProgressIndicator().checkCanceled(); +// +// if (affectedModules.size() > 0) { +// long start = System.currentTimeMillis(); +// List loadResults = myClient.downloadCompiledModules(downloadProgressManager, affectedModules); +// LOG.info("Download of compilation outputs took: " + (System.currentTimeMillis() - start)); +// return loadResults; +// } +// return Collections.emptyList(); +// } +// +// @Override +// public LoaderStatus extract(@Nullable Object loadResults, @NotNull SegmentedProgressIndicatorManager extractIndicatorManager) { +// if (!(loadResults instanceof List)) return LoaderStatus.FAILED; +// +// //noinspection unchecked +// List outputLoadResults = (List)loadResults; +// Map result = new ConcurrentHashMap<>(); +// try { +// // Extracting results +// long start = System.currentTimeMillis(); +// extractIndicatorManager.setText(this, JpsCacheBundle.message("progress.text.extracting.downloaded.results")); +// List> futureList = ContainerUtil.map(outputLoadResults, loadResult -> +// EXECUTOR_SERVICE.submit(new UnzipOutputTask(result, loadResult, extractIndicatorManager))); +// for (Future future : futureList) { +// future.get(); +// } +// extractIndicatorManager.finished(this); +// myTmpFolderToModuleName = result; +// LOG.info("Unzip compilation output took: " + (System.currentTimeMillis() - start)); +// return LoaderStatus.COMPLETE; +// } +// catch (ProcessCanceledException | InterruptedException | ExecutionException e) { +// if (!(e.getCause() instanceof ProcessCanceledException)) LOG.warn("Failed unzip downloaded compilation outputs", e); +// outputLoadResults.forEach(loadResult -> FileUtil.delete(loadResult.getZipFile())); +// result.forEach((key, value) -> FileUtil.delete(key)); +// } +// return LoaderStatus.FAILED; +// } +// +// @Override +// public void rollback() { +// if (myTmpFolderToModuleName == null) return; +// myTmpFolderToModuleName.forEach((tmpFolder, __) -> { +// if (tmpFolder.isDirectory() && tmpFolder.exists()) FileUtil.delete(tmpFolder); +// }); +// LOG.info("JPS cache loader rolled back"); +// } +// +// @Override +// public void apply(@NotNull SegmentedProgressIndicatorManager indicatorManager) { +// long start = System.currentTimeMillis(); +// if (myOldModulesPaths != null) { +// LOG.info("Removing old compilation outputs " + myOldModulesPaths.size() + " counts"); +// myOldModulesPaths.forEach(file -> { +// if (file.exists()) FileUtil.delete(file); +// }); +// } +// if (myTmpFolderToModuleName == null) { +// LOG.debug("Nothing to apply, download results are empty"); +// return; +// } +// +// indicatorManager.setText(this, JpsCacheBundle.message("progress.text.applying.jps.caches")); +// ContainerUtil.map(myTmpFolderToModuleName.entrySet(), +// entry -> EXECUTOR_SERVICE.submit(() -> { +// String moduleName = entry.getValue(); +// File tmpModuleFolder = entry.getKey(); +// SegmentedProgressIndicatorManager.SubTaskProgressIndicator subTaskIndicator = +// indicatorManager.createSubTaskIndicator(); +// subTaskIndicator.setText2(JpsCacheBundle.message("progress.details.applying.changes.for.module", moduleName)); +// File currentModuleBuildDir = new File(tmpModuleFolder.getParentFile(), moduleName); +// FileUtil.delete(currentModuleBuildDir); +// try { +// FileUtil.rename(tmpModuleFolder, currentModuleBuildDir); +// LOG.debug("Module: " + moduleName + " was replaced successfully"); +// } +// catch (IOException e) { +// LOG.warn("Couldn't replace compilation output for module: " + moduleName, e); +// } +// subTaskIndicator.finished(); +// })) +// .forEach(future -> { +// try { +// future.get(); +// } +// catch (InterruptedException | ExecutionException e) { +// LOG.info("Couldn't apply compilation output", e); +// } +// }); +// indicatorManager.finished(this); +// LOG.info("Applying compilation output took: " + (System.currentTimeMillis() - start)); +// } +// +// @NotNull +// private List calculateAffectedModules(@Nullable Map> currentModulesState, +// @NotNull Map> commitModulesState, +// boolean checkExistance) { +// long start = System.currentTimeMillis(); +// +// List affectedModules = new ArrayList<>(); +// Map oldModulesMap = new HashMap<>(); +// myOldModulesPaths = new ArrayList<>(); +// +// if (currentModulesState == null) { +// commitModulesState.forEach((type, map) -> { +// map.forEach((name, state) -> { +// affectedModules.add(new AffectedModule(type, name, state.getHash(), getBuildDirRelativeFile(state.getRelativePath()))); +// }); +// }); +// LOG.warn("Project doesn't contain metadata, force to download " + affectedModules.size() + " modules."); +// List result = mergeAffectedModules(affectedModules, commitModulesState); +// long total = System.currentTimeMillis() - start; +// LOG.info("Compilation output affected for the " + result.size() + " modules. Computation took " + total + "ms"); +// return result; +// } +// +// // Add new build types +// Set newBuildTypes = new HashSet<>(commitModulesState.keySet()); +// newBuildTypes.removeAll(currentModulesState.keySet()); +// newBuildTypes.forEach(type -> { +// commitModulesState.get(type).forEach((name, state) -> { +// affectedModules.add(new AffectedModule(type, name, state.getHash(), getBuildDirRelativeFile(state.getRelativePath()))); +// }); +// }); +// +// // Calculate old paths for remove +// Set oldBuildTypes = new HashSet<>(currentModulesState.keySet()); +// oldBuildTypes.removeAll(commitModulesState.keySet()); +// oldBuildTypes.forEach(type -> { +// currentModulesState.get(type).forEach((name, state) -> { +// oldModulesMap.put(name, state.getRelativePath()); +// }); +// }); +// +// commitModulesState.forEach((type, map) -> { +// Map currentTypeState = currentModulesState.get(type); +// +// // New build type already added above +// if (currentTypeState == null) return; +// +// // Add new build modules +// Set newBuildModules = new HashSet<>(map.keySet()); +// newBuildModules.removeAll(currentTypeState.keySet()); +// newBuildModules.forEach(name -> { +// BuildTargetState state = map.get(name); +// affectedModules.add(new AffectedModule(type, name, state.getHash(), getBuildDirRelativeFile(state.getRelativePath()))); +// }); +// +// // Calculate old modules paths for remove +// Set oldBuildModules = new HashSet<>(currentTypeState.keySet()); +// oldBuildModules.removeAll(map.keySet()); +// oldBuildModules.forEach(name -> { +// BuildTargetState state = currentTypeState.get(name); +// oldModulesMap.put(name, state.getRelativePath()); +// }); +// +// // In another case compare modules inside the same build type +// map.forEach((name, state) -> { +// BuildTargetState currentTargetState = currentTypeState.get(name); +// if (currentTargetState == null || !state.equals(currentTargetState)) { +// affectedModules.add(new AffectedModule(type, name, state.getHash(), getBuildDirRelativeFile(state.getRelativePath()))); +// return; +// } +// +// File outFile = getBuildDirRelativeFile(state.getRelativePath()); +// if (checkExistance && (!outFile.exists() || ArrayUtil.isEmpty(outFile.listFiles()))) { +// affectedModules.add(new AffectedModule(type, name, state.getHash(), outFile)); +// } +// }); +// }); +// +// // Check that old modules not exist in other build types +// myOldModulesPaths = oldModulesMap.entrySet().stream().filter(entry -> { +// for (Map.Entry> commitEntry : commitModulesState.entrySet()) { +// BuildTargetState targetState = commitEntry.getValue().get(entry.getKey()); +// if (targetState != null && targetState.getRelativePath().equals(entry.getValue())) return false; +// } +// return true; +// }).map(entry -> getBuildDirRelativeFile(entry.getValue())) +// .collect(Collectors.toList()); +// List result = mergeAffectedModules(affectedModules, commitModulesState); +// long total = System.currentTimeMillis() - start; +// LOG.info("Compilation output affected for the " + result.size() + " modules. Computation took " + total + "ms"); +// return result; +// } +// +// @NotNull +// private static List mergeAffectedModules(List affectedModules, +// @NotNull Map> commitModulesState) { +// Set result = new HashSet<>(); +// affectedModules.forEach(affectedModule -> { +// if (affectedModule.getType().equals(JAVA_PRODUCTION)) { +// BuildTargetState targetState = commitModulesState.get(RESOURCES_PRODUCTION).get(affectedModule.getName()); +// if (targetState == null) { +// result.add(affectedModule); +// return; +// } +// String hash = calculateStringHash(affectedModule.getHash() + targetState.getHash()); +// result.add(new AffectedModule(PRODUCTION, affectedModule.getName(), hash, affectedModule.getOutPath())); +// } +// else if (affectedModule.getType().equals(RESOURCES_PRODUCTION)) { +// BuildTargetState targetState = commitModulesState.get(JAVA_PRODUCTION).get(affectedModule.getName()); +// if (targetState == null) { +// result.add(affectedModule); +// return; +// } +// String hash = calculateStringHash(targetState.getHash() + affectedModule.getHash()); +// result.add(new AffectedModule(PRODUCTION, affectedModule.getName(), hash, affectedModule.getOutPath())); +// } +// else if (affectedModule.getType().equals(JAVA_TEST)) { +// BuildTargetState targetState = commitModulesState.get(RESOURCES_TEST).get(affectedModule.getName()); +// if (targetState == null) { +// result.add(affectedModule); +// return; +// } +// String hash = calculateStringHash(affectedModule.getHash() + targetState.getHash()); +// result.add(new AffectedModule(TEST, affectedModule.getName(), hash, affectedModule.getOutPath())); +// } +// else if (affectedModule.getType().equals(RESOURCES_TEST)) { +// BuildTargetState targetState = commitModulesState.get(JAVA_TEST).get(affectedModule.getName()); +// if (targetState == null) { +// result.add(affectedModule); +// return; +// } +// String hash = calculateStringHash(targetState.getHash() + affectedModule.getHash()); +// result.add(new AffectedModule(TEST, affectedModule.getName(), hash, affectedModule.getOutPath())); +// } +// else { +// result.add(affectedModule); +// } +// }); +// return new ArrayList<>(result); +// } +// +// private File getBuildDirRelativeFile(String buildDirRelativePath) { +// return new File(buildDirRelativePath.replace("$BUILD_DIR$", myBuildDirPath)); +// } +// +// private static String calculateStringHash(String content) { +// Hasher hasher = Hashing.murmur3_128().newHasher(); +// return hasher.putString(content, StandardCharsets.UTF_8).hash().toString(); +// } +// +// @TestOnly +// List getOldModulesPaths() { +// return myOldModulesPaths; +// } +// +// @TestOnly +// List getAffectedModules(@Nullable Map> currentModulesState, +// @NotNull Map> commitModulesState, boolean checkExistence) { +// return calculateAffectedModules(currentModulesState, commitModulesState, checkExistence); +// } +// +// private static final class UnzipOutputTask implements Runnable { +// private final OutputLoadResult loadResult; +// private final Map result; +// private final SegmentedProgressIndicatorManager extractIndicatorManager; +// +// private UnzipOutputTask(Map result, +// OutputLoadResult loadResult, +// SegmentedProgressIndicatorManager extractIndicatorManager) { +// this.result = result; +// this.loadResult = loadResult; +// this.extractIndicatorManager = extractIndicatorManager; +// } +// +// @Override +// public void run() { +// AffectedModule affectedModule = loadResult.getModule(); +// File outPath = affectedModule.getOutPath(); +// try { +// SegmentedProgressIndicatorManager.SubTaskProgressIndicator subTaskIndicator = extractIndicatorManager.createSubTaskIndicator(); +// extractIndicatorManager.getProgressIndicator().checkCanceled(); +// subTaskIndicator.setText2( +// JpsCacheBundle.message("progress.details.extracting.compilation.outputs.for.module", affectedModule.getName())); +// LOG.debug("Downloaded JPS compiled module from: " + loadResult.getDownloadUrl()); +// File tmpFolder = new File(outPath.getParent(), outPath.getName() + "_tmp"); +// File zipFile = loadResult.getZipFile(); +// ZipUtil.extract(zipFile, tmpFolder, null); +// FileUtil.delete(zipFile); +// result.put(tmpFolder, affectedModule.getName()); +// subTaskIndicator.finished(); +// } +// catch (IOException e) { +// LOG.warn("Couldn't extract download result for module: " + affectedModule.getName(), e); +// } +// } +// } +//} \ No newline at end of file diff --git a/jps/jps-builders/src/org/jetbrains/jps/cache/loader/JpsMetadataLoader.java b/jps/jps-builders/src/org/jetbrains/jps/cache/loader/JpsMetadataLoader.java new file mode 100644 index 000000000000..1e5b222fad13 --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/cache/loader/JpsMetadataLoader.java @@ -0,0 +1,80 @@ +//package org.jetbrains.jps.cache.loader; +// +//import com.google.gson.Gson; +//import com.google.gson.reflect.TypeToken; +//import com.intellij.compiler.server.BuildManager; +//import com.intellij.jps.cache.client.JpsServerClient; +//import com.intellij.jps.cache.model.BuildTargetState; +//import com.intellij.openapi.application.PathManager; +//import com.intellij.openapi.diagnostic.Logger; +//import com.intellij.openapi.project.Project; +//import com.intellij.openapi.util.io.FileUtil; +//import org.jetbrains.annotations.NotNull; +//import org.jetbrains.annotations.Nullable; +//import org.jetbrains.jps.cache.client.JpsServerClient; +//import org.jetbrains.jps.cache.model.BuildTargetState; +// +//import java.io.BufferedReader; +//import java.io.File; +//import java.io.FileReader; +//import java.io.IOException; +//import java.lang.reflect.Type; +//import java.util.Map; +// +//public class JpsMetadataLoader { +// private static final Logger LOG = Logger.getInstance(JpsMetadataLoader.class); +// private static final String SOURCES_STATE_FILE_NAME = "target_sources_state.json"; +// private final BuildManager myBuildManager; +// private final JpsServerClient myClient; +// private final Project myProject; +// private final Type myTokenType; +// private final Gson myGson; +// +// public JpsMetadataLoader(@NotNull Project project, @NotNull JpsServerClient client) { +// myClient = client; +// myGson = new Gson(); +// myProject = project; +// myBuildManager = BuildManager.getInstance(); +// myTokenType = new TypeToken>>() {}.getType(); +// } +// +// @Nullable +// Map> loadMetadataForCommit(@NotNull String metadataId) { +// // On server commitId is the same as metadataId this data simply located in different folders +// File tmpFolder = new File(PathManager.getTempPath()); +// File metadataFile = myClient.downloadMetadataById(metadataId, tmpFolder); +// if (metadataFile == null) return null; +// +// try (BufferedReader bufferedReader = new BufferedReader(new FileReader(metadataFile))) { +// return myGson.fromJson(bufferedReader, myTokenType); +// } +// catch (IOException e) { +// LOG.warn("Couldn't parse content of file " + metadataFile.getName(), e); +// } +// finally { +// if (metadataFile.exists()) FileUtil.delete(metadataFile); +// } +// return null; +// } +// +// @Nullable +// Map> loadCurrentProjectMetadata() { +// File currentBuildCacheFolder = myBuildManager.getProjectSystemDirectory(myProject); +// File currentSourceState = new File(currentBuildCacheFolder, SOURCES_STATE_FILE_NAME); +// if (!currentSourceState.exists()) return null; +// +// try (BufferedReader bufferedReader = new BufferedReader(new FileReader(currentSourceState))) { +// return myGson.fromJson(bufferedReader, myTokenType); +// } +// catch (IOException e) { +// LOG.warn("Couldn't parse current project metadata " + currentSourceState.getName(), e); +// } +// return null; +// } +// +// void dropCurrentProjectMetadata() { +// File currentBuildCacheFolder = myBuildManager.getProjectSystemDirectory(myProject); +// File currentSourceState = new File(currentBuildCacheFolder, SOURCES_STATE_FILE_NAME); +// if (currentSourceState.exists()) FileUtil.delete(currentSourceState); +// } +//} diff --git a/jps/jps-builders/src/org/jetbrains/jps/cache/loader/JpsOutputLoader.java b/jps/jps-builders/src/org/jetbrains/jps/cache/loader/JpsOutputLoader.java new file mode 100644 index 000000000000..772d3397f033 --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/cache/loader/JpsOutputLoader.java @@ -0,0 +1,24 @@ +//package org.jetbrains.jps.cache.loader; +// +//import org.jetbrains.jps.cache.model.BuildTargetState; +//import org.jetbrains.jps.cache.model.JpsLoaderContext; +//import org.jetbrains.annotations.NotNull; +//import org.jetbrains.annotations.Nullable; +//import org.jetbrains.jps.cache.ui.SegmentedProgressIndicatorManager; +// +//import java.util.Map; +// +//interface JpsOutputLoader { +// T load(@NotNull JpsLoaderContext context); +// LoaderStatus extract(@Nullable Object loadResults, @NotNull SegmentedProgressIndicatorManager extractIndicatorManager); +// void rollback(); +// void apply(@NotNull SegmentedProgressIndicatorManager indicatorManager); +// default int calculateDownloads(@NotNull Map> commitSourcesState, +// @Nullable Map> currentSourcesState) { +// return 1; +// } +// +// enum LoaderStatus { +// COMPLETE, FAILED +// } +//} \ No newline at end of file diff --git a/jps/jps-builders/src/org/jetbrains/jps/cache/loader/JpsOutputLoaderManager.java b/jps/jps-builders/src/org/jetbrains/jps/cache/loader/JpsOutputLoaderManager.java new file mode 100644 index 000000000000..7c68166a05c4 --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/cache/loader/JpsOutputLoaderManager.java @@ -0,0 +1,383 @@ +//package org.jetbrains.jps.cache.loader; +// +//import com.intellij.compiler.CompilerWorkspaceConfiguration; +//import com.intellij.compiler.server.BuildManager; +//import com.intellij.compiler.server.PortableCachesLoadListener; +//import com.intellij.ide.util.PropertiesComponent; +//import com.intellij.jps.cache.JpsCacheBundle; +//import com.intellij.jps.cache.JpsCacheStartupActivity; +//import com.intellij.jps.cache.client.JpsServerClient; +//import com.intellij.jps.cache.git.GitCommitsIterator; +//import com.intellij.jps.cache.git.GitRepositoryUtil; +//import com.intellij.jps.cache.loader.JpsCompilationOutputLoader; +//import com.intellij.jps.cache.loader.JpsOutputLoader.LoaderStatus; +//import com.intellij.jps.cache.model.BuildTargetState; +//import com.intellij.jps.cache.model.JpsLoaderContext; +//import com.intellij.jps.cache.ui.SegmentedProgressIndicatorManager; +//import com.intellij.notification.NotificationAction; +//import com.intellij.notification.NotificationType; +//import com.intellij.openapi.Disposable; +//import com.intellij.openapi.application.ApplicationManager; +//import com.intellij.openapi.diagnostic.Logger; +//import com.intellij.openapi.progress.ProcessCanceledException; +//import com.intellij.openapi.progress.ProgressIndicator; +//import com.intellij.openapi.progress.ProgressManager; +//import com.intellij.openapi.progress.Task; +//import com.intellij.openapi.progress.impl.BackgroundableProcessIndicator; +//import com.intellij.openapi.project.Project; +//import com.intellij.openapi.util.Pair; +//import com.intellij.openapi.util.io.FileUtil; +//import com.intellij.openapi.vfs.VirtualFile; +//import com.intellij.util.containers.ContainerUtil; +//import org.jdom.Element; +//import org.jetbrains.annotations.NotNull; +//import org.jetbrains.annotations.Nullable; +//import org.jetbrains.jps.cache.JpsCacheBundle; +//import org.jetbrains.jps.cache.client.JpsServerClient; +//import org.jetbrains.jps.cache.git.GitRepositoryUtil; +//import org.jetbrains.jps.cache.loader.JpsOutputLoader.LoaderStatus; +//import org.jetbrains.jps.cache.model.BuildTargetState; +//import org.jetbrains.jps.cache.model.JpsLoaderContext; +//import org.jetbrains.jps.cache.ui.SegmentedProgressIndicatorManager; +//import org.jetbrains.jps.model.serialization.JDomSerializationUtil; +//import org.jetbrains.jps.model.serialization.JpsLoaderBase; +//import org.jetbrains.jps.model.serialization.PathMacroUtil; +//import org.jetbrains.jps.util.JpsPathUtil; +// +//import java.io.File; +//import java.nio.file.Path; +//import java.nio.file.Paths; +//import java.util.*; +//import java.util.concurrent.CompletableFuture; +//import java.util.concurrent.ExecutionException; +//import java.util.concurrent.atomic.AtomicBoolean; +// +//import static com.intellij.execution.process.ProcessIOExecutorService.INSTANCE; +//import static com.intellij.jps.cache.statistics.JpsCacheUsagesCollector.*; +//import static com.intellij.jps.cache.ui.JpsLoaderNotifications.*; +//import static org.jetbrains.jps.model.serialization.java.JpsJavaModelSerializerExtension.OUTPUT_TAG; +//import static org.jetbrains.jps.model.serialization.java.JpsJavaModelSerializerExtension.URL_ATTRIBUTE; +// +//public class JpsOutputLoaderManager implements Disposable { +// private static final Logger LOG = Logger.getInstance(JpsOutputLoaderManager.class); +// private static final String LATEST_COMMIT_ID = "JpsOutputLoaderManager.latestCommitId"; +// private static final double SEGMENT_SIZE = 0.33; +// private final AtomicBoolean hasRunningTask; +// private final CompilerWorkspaceConfiguration myWorkspaceConfiguration; +// private List> myJpsOutputLoadersLoaders; +// private final JpsMetadataLoader myMetadataLoader; +// private final JpsServerClient myServerClient; +// private final String myBuildOutDir; +// private final Project myProject; +// +// @Override +// public void dispose() { } +// +// @NotNull +// public static JpsOutputLoaderManager getInstance(@NotNull Project project) { +// return project.getService(JpsOutputLoaderManager.class); +// } +// +// public JpsOutputLoaderManager(@NotNull Project project) { +// myProject = project; +// hasRunningTask = new AtomicBoolean(); +// myBuildOutDir = getBuildOutDir(myProject); +// myServerClient = JpsServerClient.getServerClient(); +// myMetadataLoader = new JpsMetadataLoader(project, myServerClient); +// myWorkspaceConfiguration = CompilerWorkspaceConfiguration.getInstance(myProject); +// // Configure build manager +// BuildManager buildManager = BuildManager.getInstance(); +// if (!buildManager.isGeneratePortableCachesEnabled()) buildManager.setGeneratePortableCachesEnabled(true); +// } +// +// public void load(boolean isForceUpdate, boolean verbose) { +// Task.Backgroundable task = new Task.Backgroundable(myProject, JpsCacheBundle.message("progress.title.updating.compiler.caches")) { +// @Override +// public void run(@NotNull ProgressIndicator indicator) { +// Pair commitInfo = getNearestCommit(isForceUpdate, verbose); +// if (commitInfo != null) { +// assert myProject != null; +// myProject.getMessageBus().syncPublisher(PortableCachesLoadListener.TOPIC).loadingStarted(); +// // Drop JPS metadata to force plugin for downloading all compilation outputs +// if (isForceUpdate) { +// myMetadataLoader.dropCurrentProjectMetadata(); +// File outDir = new File(myBuildOutDir); +// if (outDir.exists()) { +// indicator.setText(JpsCacheBundle.message("progress.text.clean.output.directories")); +// FileUtil.delete(outDir); +// } +// LOG.info("Compilation output folder empty"); +// } +// startLoadingForCommit(commitInfo.first); +// } +// hasRunningTask.set(false); +// } +// }; +// +// if (!canRunNewLoading()) return; +// BackgroundableProcessIndicator processIndicator = new BackgroundableProcessIndicator(task); +// processIndicator.setIndeterminate(false); +// ProgressManager.getInstance().runProcessWithProgressAsynchronously(task, processIndicator); +// } +// +// public void notifyAboutNearestCache() { +// Pair commitInfo = getNearestCommit(false, false); +// if (commitInfo == null) return; +// +// String notificationContent = commitInfo.second == 1 +// ? JpsCacheBundle.message("notification.content.caches.are.for.current.commit") +// : JpsCacheBundle +// .message("notification.content.caches.are.for.commit.commits.prior.to.yours", commitInfo.second - 1); +// +// ApplicationManager.getApplication().invokeLater(() -> { +// STANDARD +// .createNotification(JpsCacheBundle.message("notification.title.compiler.caches.available"), notificationContent, +// NotificationType.INFORMATION) +// .addAction(NotificationAction.createSimpleExpiring( +// JpsCacheBundle.message("action.NotificationAction.JpsOutputLoaderManager.text.update.caches"), +// () -> { +// DOWNLOAD_THROUGH_NOTIFICATION_EVENT_ID.log(); +// load(false, false); +// })) +// .notify(myProject); +// }); +// } +// +// @Nullable +// private Pair getNearestCommit(boolean isForceUpdate, boolean verbose) { +// Map> availableCommitsPerRemote = myServerClient.getCacheKeysPerRemote(myProject); +// +// String previousCommitId = PropertiesComponent.getInstance().getValue(LATEST_COMMIT_ID); +// List repositoryList = GitRepositoryUtil.getCommitsIterator(myProject, availableCommitsPerRemote.keySet()); +// String commitId = ""; +// int commitsBehind = 0; +// Set availableCommitsForRemote = new HashSet<>(); +// for (GitCommitsIterator commitsIterator : repositoryList) { +// availableCommitsForRemote = availableCommitsPerRemote.get(commitsIterator.getRemote()); +// if (availableCommitsForRemote.contains(commitId)) continue; +// commitsBehind = 0; +// while (commitsIterator.hasNext() && !availableCommitsForRemote.contains(commitId)) { +// commitId = commitsIterator.next(); +// commitsBehind++; +// } +// } +// var group = verbose ? STANDARD : EVENT_LOG; +// if (!availableCommitsForRemote.contains(commitId)) { +// String warning = JpsCacheBundle.message("notification.content.not.found.any.caches.for.latest.commits.in.branch"); +// LOG.warn(warning); +// ApplicationManager.getApplication().invokeLater(() -> { +// group.createNotification(JpsCacheBundle.message("notification.title.jps.caches.downloader"), warning, NotificationType.WARNING) +// .notify(myProject); +// }); +// return null; +// } +// if (previousCommitId != null && commitId.equals(previousCommitId) && !isForceUpdate) { +// String info = JpsCacheBundle.message("notification.content.system.contains.up.to.date.caches"); +// LOG.info(info); +// ApplicationManager.getApplication().invokeLater(() -> { +// group.createNotification(JpsCacheBundle.message("notification.title.jps.caches.downloader"), info, NotificationType.INFORMATION) +// .notify(myProject); +// }); +// return null; +// } +// return Pair.create(commitId, commitsBehind); +// } +// +// private void startLoadingForCommit(@NotNull String commitId) { +// long startTime = System.nanoTime(); +// ProgressIndicator indicator = ProgressManager.getInstance().getProgressIndicator(); +// indicator.setText(JpsCacheBundle.message("progress.text.fetching.cache.for.commit", commitId)); +// +// // Loading metadata for commit +// Map> commitSourcesState = myMetadataLoader.loadMetadataForCommit(commitId); +// if (commitSourcesState == null) { +// LOG.warn("Couldn't load metadata for commit: " + commitId); +// myProject.getMessageBus().syncPublisher(PortableCachesLoadListener.TOPIC).loadingFinished(false); +// return; +// } +// +// // Calculate downloads +// Map> currentSourcesState = myMetadataLoader.loadCurrentProjectMetadata(); +// int totalDownloads = +// getLoaders(myProject).stream().mapToInt(loader -> loader.calculateDownloads(commitSourcesState, currentSourcesState)).sum(); +// indicator.setFraction(0.01); +// +// try { +// // Computation with loaders results. If at least one of them failed rollback all job +// initLoaders(commitId, indicator, totalDownloads, commitSourcesState, currentSourcesState).thenAccept(loaderStatus -> { +// LOG.info("Loading finished with " + loaderStatus + " status"); +// try { +// SegmentedProgressIndicatorManager indicatorManager = +// new SegmentedProgressIndicatorManager(indicator, totalDownloads, SEGMENT_SIZE); +// CompletableFuture.allOf(getLoaders(myProject).stream() +// .map(loader -> applyChanges(loaderStatus, loader, indicator, indicatorManager)) +// .toArray(CompletableFuture[]::new)) +// .thenRun(() -> saveStateAndNotify(loaderStatus, commitId, startTime)) +// .get(); +// } +// catch (InterruptedException | ExecutionException e) { +// LOG.warn("Unexpected exception rollback all progress", e); +// onFail(); +// getLoaders(myProject).forEach(loader -> loader.rollback()); +// myProject.getMessageBus().syncPublisher(PortableCachesLoadListener.TOPIC).loadingFinished(false); +// indicator.setText(JpsCacheBundle.message("progress.text.rolling.back.downloaded.caches")); +// } +// }).handle((result, ex) -> handleExceptions(result, ex, indicator)).get(); +// } +// catch (InterruptedException | ExecutionException e) { +// LOG.warn("Couldn't fetch jps compilation caches", e); +// onFail(); +// myProject.getMessageBus().syncPublisher(PortableCachesLoadListener.TOPIC).loadingFinished(false); +// } +// } +// +// @Nullable +// private static String getBuildOutDir(@NotNull Project project) { +// VirtualFile projectFile = project.getProjectFile(); +// String projectBasePath = project.getBasePath(); +// if (projectFile == null || projectBasePath == null) { +// LOG.warn("Project files doesn't exist"); +// return null; +// } +// String fileExtension = projectFile.getExtension(); +// if (fileExtension != null && fileExtension.equals("irp")) { +// LOG.warn("File base project not supported"); +// return null; +// } +// +// Path configFile = Paths.get(FileUtil.toCanonicalPath(projectFile.getPath())); +// Element componentTag = JDomSerializationUtil.findComponent(JpsLoaderBase.tryLoadRootElement(configFile), "ProjectRootManager"); +// if (componentTag == null) { +// LOG.warn("Component tag in config file doesn't exist"); +// return null; +// } +// Element output = componentTag.getChild(OUTPUT_TAG); +// if (output == null) { +// LOG.warn("Output tag in config file doesn't exist"); +// return null; +// } +// String url = output.getAttributeValue(URL_ATTRIBUTE); +// if (url == null) { +// LOG.warn("URL attribute in output tag doesn't exist"); +// return null; +// } +// return JpsPathUtil.urlToPath(url).replace("$" + PathMacroUtil.PROJECT_DIR_MACRO_NAME + "$", projectBasePath); +// } +// +// private synchronized boolean canRunNewLoading() { +// if (hasRunningTask.get()) { +// LOG.warn("Jps cache loading already in progress, can't start the new one"); +// return false; +// } +// if (myBuildOutDir == null) { +// LOG.warn("Build output dir is not configured for the project"); +// return false; +// } +// if (myWorkspaceConfiguration.MAKE_PROJECT_ON_SAVE) { +// LOG.warn("Project automatic build should be disabled, it can affect portable caches"); +// return false; +// } +// if (!JpsCacheStartupActivity.isLineEndingsConfiguredCorrectly()) { +// LOG.warn("Git line-endings not configured correctly for the project"); +// return false; +// } +// hasRunningTask.set(true); +// return true; +// } +// +// private CompletableFuture initLoaders(String commitId, ProgressIndicator indicator, int totalDownloads, +// Map> commitSourcesState, +// Map> currentSourcesState) { +// List> loaders = getLoaders(myProject); +// +// // Create indicator with predefined segment size +// SegmentedProgressIndicatorManager downloadIndicatorManager = +// new SegmentedProgressIndicatorManager(indicator, totalDownloads, SEGMENT_SIZE); +// SegmentedProgressIndicatorManager extractIndicatorManager = +// new SegmentedProgressIndicatorManager(indicator, totalDownloads, SEGMENT_SIZE); +// JpsLoaderContext loaderContext = +// JpsLoaderContext.createNewContext(commitId, downloadIndicatorManager, commitSourcesState, currentSourcesState); +// +// // Start loaders with own context +// List> completableFutures = ContainerUtil.map(loaders, loader -> +// CompletableFuture.supplyAsync(() -> loader.extract(loader.load(loaderContext), extractIndicatorManager), INSTANCE)); +// +// // Reduce loaders statuses into the one +// CompletableFuture initialFuture = completableFutures.get(0); +// if (completableFutures.size() > 1) { +// for (int i = 1; i < completableFutures.size(); i++) { +// initialFuture = initialFuture.thenCombine(completableFutures.get(i), JpsOutputLoaderManager::combine); +// } +// } +// return initialFuture; +// } +// +// private static CompletableFuture applyChanges(LoaderStatus loaderStatus, JpsOutputLoader loader, ProgressIndicator indicator, +// SegmentedProgressIndicatorManager indicatorManager) { +// if (loaderStatus == LoaderStatus.FAILED) { +// indicator.setText(JpsCacheBundle.message("progress.text.rolling.back")); +// return CompletableFuture.runAsync(() -> loader.rollback(), INSTANCE); +// } +// return CompletableFuture.runAsync(() -> loader.apply(indicatorManager), INSTANCE); +// } +// +// private void saveStateAndNotify(LoaderStatus loaderStatus, String commitId, long startTime) { +// if (loaderStatus == LoaderStatus.FAILED) { +// onFail(); +// myProject.getMessageBus().syncPublisher(PortableCachesLoadListener.TOPIC).loadingFinished(false); +// return; +// } +// +// PropertiesComponent.getInstance().setValue(LATEST_COMMIT_ID, commitId); +// BuildManager.getInstance().clearState(myProject); +// long endTime = System.nanoTime() - startTime; +// ApplicationManager.getApplication().invokeLater(() -> { +// STANDARD +// .createNotification(JpsCacheBundle.message("notification.title.compiler.caches.loader"), +// JpsCacheBundle.message("notification.content.update.compiler.caches.completed.successfully.in.s", +// endTime / 1_000_000_000), +// NotificationType.INFORMATION) +// .notify(myProject); +// }); +// DOWNLOAD_DURATION_EVENT_ID.log(endTime); +// myProject.getMessageBus().syncPublisher(PortableCachesLoadListener.TOPIC).loadingFinished(true); +// LOG.info("Loading finished"); +// } +// +// private Void handleExceptions(Void result, Throwable ex, ProgressIndicator indicator) { +// if (ex != null) { +// Throwable cause = ex.getCause(); +// if (cause instanceof ProcessCanceledException) { +// LOG.info("Jps caches download canceled"); +// } +// else { +// LOG.warn("Couldn't fetch jps compilation caches", ex); +// onFail(); +// } +// getLoaders(myProject).forEach(loader -> loader.rollback()); +// myProject.getMessageBus().syncPublisher(PortableCachesLoadListener.TOPIC).loadingFinished(false); +// indicator.setText(JpsCacheBundle.message("progress.text.rolling.back.downloaded.caches")); +// } +// return result; +// } +// +// private List> getLoaders(@NotNull Project project) { +// if (myJpsOutputLoadersLoaders != null) return myJpsOutputLoadersLoaders; +// myJpsOutputLoadersLoaders = Arrays.asList(new JpsCacheLoader(myServerClient, project), +// new JpsCompilationOutputLoader(myServerClient, myBuildOutDir)); +// return myJpsOutputLoadersLoaders; +// } +// +// private static LoaderStatus combine(LoaderStatus firstStatus, LoaderStatus secondStatus) { +// if (firstStatus == LoaderStatus.FAILED || secondStatus == LoaderStatus.FAILED) return LoaderStatus.FAILED; +// return LoaderStatus.COMPLETE; +// } +// +// private void onFail() { +// ApplicationManager.getApplication().invokeLater(() -> { +// ATTENTION.createNotification(JpsCacheBundle.message("notification.title.compiler.caches.loader"), +// JpsCacheBundle.message("notification.content.update.compiler.caches.failed"), NotificationType.WARNING) +// .notify(myProject); +// }); +// } +//} diff --git a/jps/jps-builders/src/org/jetbrains/jps/cache/model/AffectedModule.java b/jps/jps-builders/src/org/jetbrains/jps/cache/model/AffectedModule.java new file mode 100644 index 000000000000..0f4bc37f5cb9 --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/cache/model/AffectedModule.java @@ -0,0 +1,57 @@ +package org.jetbrains.jps.cache.model; + +import java.io.File; + +public class AffectedModule { + private final String type; + private final String name; + private final String hash; + private final File outPath; + + public AffectedModule(String type, String name, String hash, File outPath) { + this.type = type; + this.name = name; + this.hash = hash; + this.outPath = outPath; + } + + public String getType() { + return type; + } + + public String getName() { + return name; + } + + public String getHash() { + return hash; + } + + public File getOutPath() { + return outPath; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + AffectedModule module = (AffectedModule)o; + + if (type != null ? !type.equals(module.type) : module.type != null) return false; + if (name != null ? !name.equals(module.name) : module.name != null) return false; + if (hash != null ? !hash.equals(module.hash) : module.hash != null) return false; + if (outPath != null ? !outPath.equals(module.outPath) : module.outPath != null) return false; + + return true; + } + + @Override + public int hashCode() { + int result = type != null ? type.hashCode() : 0; + result = 31 * result + (name != null ? name.hashCode() : 0); + result = 31 * result + (hash != null ? hash.hashCode() : 0); + result = 31 * result + (outPath != null ? outPath.hashCode() : 0); + return result; + } +} diff --git a/jps/jps-builders/src/org/jetbrains/jps/cache/model/BuildTargetState.java b/jps/jps-builders/src/org/jetbrains/jps/cache/model/BuildTargetState.java new file mode 100644 index 000000000000..342c6b7b204c --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/cache/model/BuildTargetState.java @@ -0,0 +1,38 @@ +package org.jetbrains.jps.cache.model; + +import java.util.Objects; + +public final class BuildTargetState { + private final String hash; + private final String relativePath; + + private BuildTargetState(String hash, String relativePath) { + this.hash = hash; + this.relativePath = relativePath; + } + + public String getHash() { + return hash; + } + + public String getRelativePath() { + return relativePath; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + BuildTargetState state = (BuildTargetState)o; + if (!Objects.equals(hash, state.hash)) return false; + if (!Objects.equals(relativePath, state.relativePath)) return false; + return true; + } + + @Override + public int hashCode() { + int result = hash != null ? hash.hashCode() : 0; + result = 31 * result + (relativePath != null ? relativePath.hashCode() : 0); + return result; + } +} \ No newline at end of file diff --git a/jps/jps-builders/src/org/jetbrains/jps/cache/model/DownloadableFileUrl.java b/jps/jps-builders/src/org/jetbrains/jps/cache/model/DownloadableFileUrl.java new file mode 100644 index 000000000000..ac045aee4858 --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/cache/model/DownloadableFileUrl.java @@ -0,0 +1,60 @@ +// Copyright 2000-2021 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package org.jetbrains.jps.cache.model; + +import com.intellij.openapi.util.Condition; +import com.intellij.openapi.util.Conditions; +import com.intellij.openapi.util.io.FileUtilRt; +import com.intellij.util.text.UniqueNameGenerator; +import org.jetbrains.annotations.NotNull; + +public class DownloadableFileUrl { + private final String myFileName; + private final String myFileExtension; + private final String myDownloadUrl; + + public DownloadableFileUrl(@NotNull String downloadUrl, @NotNull String fileName) { + String fileExtension = FileUtilRt.getExtension(fileName); + myFileName = FileUtilRt.getNameWithoutExtension(fileName); + myFileExtension = fileExtension.length() > 0 && !fileExtension.startsWith(".") ? "." + fileExtension : fileExtension; + myDownloadUrl = downloadUrl; + } + + @NotNull + public String getDownloadUrl() { + return myDownloadUrl; + } + + @NotNull + public String getPresentableFileName() { + return myFileName + myFileExtension; + } + + @NotNull + public String getPresentableDownloadUrl() { + return myDownloadUrl; + } + + @NotNull + public String getDefaultFileName() { + return generateFileName(Conditions.alwaysTrue()); + } + + @NotNull + public String generateFileName(@NotNull Condition validator) { + return UniqueNameGenerator.generateUniqueName("", myFileName, myFileExtension, "_", "", validator); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + DownloadableFileUrl that = (DownloadableFileUrl)o; + return myDownloadUrl.equals(that.myDownloadUrl); + } + + @Override + public int hashCode() { + return myDownloadUrl.hashCode(); + } +} diff --git a/jps/jps-builders/src/org/jetbrains/jps/cache/model/JpsLoaderContext.java b/jps/jps-builders/src/org/jetbrains/jps/cache/model/JpsLoaderContext.java new file mode 100644 index 000000000000..c9880524bc38 --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/cache/model/JpsLoaderContext.java @@ -0,0 +1,49 @@ +//package org.jetbrains.jps.cache.model; +// +//import org.jetbrains.annotations.NotNull; +//import org.jetbrains.annotations.Nullable; +//import org.jetbrains.jps.cache.ui.SegmentedProgressIndicatorManager; +// +//import java.util.Map; +// +//public final class JpsLoaderContext { +// private final String commitId; +// private final SegmentedProgressIndicatorManager downloadIndicatorManager; +// private final Map> commitSourcesState; +// private final Map> currentSourcesState; +// +// private JpsLoaderContext(@NotNull String commitId, @NotNull SegmentedProgressIndicatorManager downloadIndicatorManager, +// @NotNull Map> commitSourcesState, +// @Nullable Map> currentSourcesState) { +// this.commitId = commitId; +// this.downloadIndicatorManager = downloadIndicatorManager; +// this.commitSourcesState = commitSourcesState; +// this.currentSourcesState = currentSourcesState; +// } +// +// @NotNull +// public String getCommitId() { +// return commitId; +// } +// +// @NotNull +// public SegmentedProgressIndicatorManager getDownloadIndicatorManager() { +// return downloadIndicatorManager; +// } +// +// @NotNull +// public Map> getCommitSourcesState() { +// return commitSourcesState; +// } +// +// @Nullable +// public Map> getCurrentSourcesState() { +// return currentSourcesState; +// } +// +// public static JpsLoaderContext createNewContext(@NotNull String commitId, @NotNull SegmentedProgressIndicatorManager downloadIndicatorManager, +// @NotNull Map> commitSourcesState, +// @Nullable Map> currentSourcesState) { +// return new JpsLoaderContext(commitId, downloadIndicatorManager, commitSourcesState, currentSourcesState); +// } +//} diff --git a/jps/jps-builders/src/org/jetbrains/jps/cache/model/OutputLoadResult.java b/jps/jps-builders/src/org/jetbrains/jps/cache/model/OutputLoadResult.java new file mode 100644 index 000000000000..12a918014f30 --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/cache/model/OutputLoadResult.java @@ -0,0 +1,27 @@ +package org.jetbrains.jps.cache.model; + +import java.io.File; + +public class OutputLoadResult { + private File zipFile; + private String downloadUrl; + private AffectedModule module; + + public OutputLoadResult(File zipFile, String downloadUrl, AffectedModule module) { + this.zipFile = zipFile; + this.downloadUrl = downloadUrl; + this.module = module; + } + + public File getZipFile() { + return zipFile; + } + + public String getDownloadUrl() { + return downloadUrl; + } + + public AffectedModule getModule() { + return module; + } +} diff --git a/jps/jps-builders/src/org/jetbrains/jps/cache/statistics/JpsCacheUsagesCollector.java b/jps/jps-builders/src/org/jetbrains/jps/cache/statistics/JpsCacheUsagesCollector.java new file mode 100644 index 000000000000..7e71e7809410 --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/cache/statistics/JpsCacheUsagesCollector.java @@ -0,0 +1,21 @@ +//// Copyright 2000-2021 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +//package org.jetbrains.jps.cache.statistics; +// +//import com.intellij.internal.statistic.eventLog.EventLogGroup; +//import com.intellij.internal.statistic.eventLog.events.EventFields; +//import com.intellij.internal.statistic.eventLog.events.EventId; +//import com.intellij.internal.statistic.eventLog.events.EventId1; +//import com.intellij.internal.statistic.service.fus.collectors.CounterUsagesCollector; +// +//public class JpsCacheUsagesCollector extends CounterUsagesCollector { +// private static final EventLogGroup GROUP = new EventLogGroup("jps.cache", 3); +// public static final EventId DOWNLOAD_THROUGH_NOTIFICATION_EVENT_ID = GROUP.registerEvent("download.through.notification"); +// public static final EventId1 DOWNLOAD_CACHE_SIZE_EVENT_ID = GROUP.registerEvent("caches.downloaded", EventFields.Long("download_cache_size")); +// public static final EventId1 DOWNLOAD_BINARY_SIZE_EVENT_ID = GROUP.registerEvent("caches.downloaded", EventFields.Long("download_binary_size")); +// public static final EventId1 DOWNLOAD_DURATION_EVENT_ID = GROUP.registerEvent("caches.downloaded", EventFields.Long("duration")); +// +// @Override +// public EventLogGroup getGroup() { +// return GROUP; +// } +//} diff --git a/jps/jps-builders/src/org/jetbrains/jps/cache/ui/JpsLoaderNotifications.java b/jps/jps-builders/src/org/jetbrains/jps/cache/ui/JpsLoaderNotifications.java new file mode 100644 index 000000000000..d494bacfaf24 --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/cache/ui/JpsLoaderNotifications.java @@ -0,0 +1,13 @@ +//package org.jetbrains.jps.cache.ui; +// +//import com.intellij.notification.NotificationDisplayType; +//import com.intellij.notification.NotificationGroup; +// +//public final class JpsLoaderNotifications { +// public static final NotificationGroup ATTENTION = new NotificationGroup("Compile Output Loader: Attention", +// NotificationDisplayType.STICKY_BALLOON, true); +// public static final NotificationGroup STANDARD = new NotificationGroup("Compile Output Loader: Standard", +// NotificationDisplayType.BALLOON, true); +// public static final NotificationGroup EVENT_LOG = new NotificationGroup("Compile Output Loader: Event Log", +// NotificationDisplayType.NONE, true); +//} diff --git a/jps/jps-builders/src/org/jetbrains/jps/cache/ui/SegmentedProgressIndicatorManager.java b/jps/jps-builders/src/org/jetbrains/jps/cache/ui/SegmentedProgressIndicatorManager.java new file mode 100644 index 000000000000..e3fd4db84b46 --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/cache/ui/SegmentedProgressIndicatorManager.java @@ -0,0 +1,129 @@ +//package org.jetbrains.jps.cache.ui; +// +//import com.intellij.openapi.progress.ProgressIndicator; +//import com.intellij.openapi.progress.util.ProgressWrapper; +//import com.intellij.openapi.util.NlsContexts; +//import com.intellij.util.containers.hash.LinkedHashMap; +//import org.jetbrains.annotations.NotNull; +//import org.jetbrains.annotations.Nullable; +// +//public class SegmentedProgressIndicatorManager { +// private static final LinkedHashMap myText2Stack = new LinkedHashMap<>(); +// private static final LinkedHashMap myTextStack = new LinkedHashMap<>(); +// private static final Object myLock = new Object(); +// private final ProgressIndicator myProgressIndicator; +// private final boolean myProgressWasIndeterminate; +// private final double mySegmentSize; +// private final int myTasksCount; +// +// public SegmentedProgressIndicatorManager(ProgressIndicator progressIndicator) { +// this(progressIndicator, 1, 1); +// } +// +// public SegmentedProgressIndicatorManager(ProgressIndicator progressIndicator, int tasksCount, double segmentSize) { +// myProgressIndicator = progressIndicator; +// myProgressWasIndeterminate = progressIndicator.isIndeterminate(); +// myProgressIndicator.setIndeterminate(false); +// mySegmentSize = segmentSize; +// myTasksCount = tasksCount; +// myText2Stack.clear(); +// myTextStack.clear(); +// } +// +// public SubTaskProgressIndicator createSubTaskIndicator() { +// assert myTasksCount != 0; +// return new SubTaskProgressIndicator(this); +// } +// +// public void updateFraction(double value) { +// double fractionValue = value / myTasksCount * mySegmentSize; +// synchronized (myProgressIndicator) { +// myProgressIndicator.setFraction(myProgressIndicator.getFraction() + fractionValue); +// } +// } +// +// public void setText(@NotNull Object obj, @Nullable @NlsContexts.ProgressText String text) { +// if (text != null) { +// synchronized (myLock) { +// myTextStack.put(obj, text); +// } +// myProgressIndicator.setText(text); +// } +// else { +// String prev; +// synchronized (myLock) { +// myTextStack.remove(obj); +// prev = myTextStack.getLastValue(); +// } +// if (prev != null) { +// myProgressIndicator.setText(prev); +// } +// } +// } +// +// public void setText2(@NotNull SubTaskProgressIndicator subTask, @Nullable @NlsContexts.ProgressDetails String text) { +// if (text != null) { +// synchronized (myLock) { +// myText2Stack.put(subTask, text); +// } +// myProgressIndicator.setText2(text); +// } +// else { +// String prev; +// synchronized (myLock) { +// myText2Stack.remove(subTask); +// prev = myText2Stack.getLastValue(); +// } +// if (prev != null) { +// myProgressIndicator.setText2(prev); +// } +// } +// } +// +// public void finished(Object obj) { +// setText(obj, null); +// myProgressIndicator.setIndeterminate(myProgressWasIndeterminate); +// } +// +// public ProgressIndicator getProgressIndicator() { +// return myProgressIndicator; +// } +// +// public static final class SubTaskProgressIndicator extends ProgressWrapper { +// private final SegmentedProgressIndicatorManager myProgressManager; +// private double myFraction; +// +// private SubTaskProgressIndicator(SegmentedProgressIndicatorManager progressManager) { +// super(progressManager.myProgressIndicator, true); +// myProgressManager = progressManager; +// myFraction = 0; +// } +// +// @Override +// public void setFraction(double newValue) { +// double diffFraction = newValue - myFraction; +// myProgressManager.updateFraction(diffFraction); +// myFraction = newValue; +// } +// +// @Override +// public void setText2(String text) { +// myProgressManager.setText2(this, text); +// } +// +// @Override +// public void setText(String text) { +// myProgressManager.setText(this, text); +// } +// +// @Override +// public double getFraction() { +// return myFraction; +// } +// +// public void finished() { +// setFraction(1); +// myProgressManager.setText2(this, null); +// } +// } +//} diff --git a/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildMain.java b/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildMain.java index 63e19b5fa91e..a8c546c10ae0 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildMain.java +++ b/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildMain.java @@ -239,7 +239,11 @@ public final class BuildMain { } return; } - + case AUTHENTICATION_TOKEN: { + CmdlineRemoteProto.Message.ControllerMessage.RequestParams requestParams = controllerMessage.getRequestParams(); + System.out.println("Got request params: " + requestParams.getAuthHeadersMap()); + return; + } case CONSTANT_SEARCH_RESULT: { // ignored, functionality deprecated return; diff --git a/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildSession.java b/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildSession.java index 78b45910a1d5..67521a9a8077 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildSession.java +++ b/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildSession.java @@ -28,6 +28,7 @@ import org.jetbrains.jps.incremental.TargetTypeRegistry; import org.jetbrains.jps.incremental.Utils; import org.jetbrains.jps.incremental.fs.BuildFSState; import org.jetbrains.jps.incremental.messages.*; +import org.jetbrains.jps.incremental.storage.ProjectStamps; import org.jetbrains.jps.incremental.storage.StampsStorage; import org.jetbrains.jps.model.module.JpsModule; import org.jetbrains.jps.model.serialization.CannotLoadJpsModelException; @@ -159,6 +160,11 @@ final class BuildSession implements Runnable, CanceledStatus { profilingHelper.startProfiling(); } + if (ProjectStamps.PORTABLE_CACHES) { + // Try to download caches + myChannel.writeAndFlush(CmdlineProtoUtil.toMessage(mySessionId, CmdlineProtoUtil.createAuthTokenRequest())); + } + runBuild(new MessageHandler() { @Override public void processMessage(BuildMessage buildMessage) { diff --git a/platform/util/resources/misc/registry.properties b/platform/util/resources/misc/registry.properties index 5b0989564d3b..d0cb19540034 100644 --- a/platform/util/resources/misc/registry.properties +++ b/platform/util/resources/misc/registry.properties @@ -669,6 +669,9 @@ compiler.external.javac.keep.alive.timeout.description=If not used for the speci compiler.natural.int.multimap.impl=false compiler.natural.int.multimap.impl.description=Experimental implementation for class-to-class dependency multimap in JPS caches.\nIf turned off after being turned on, project rebuild is required. +compiler.process.use.portable.caches=false +compiler.process.use.portable.caches.description=Turn on download compilation caches for reducing build time. + vcs.showConsole=true vcs.showConsole.description=Show 'Console' tab in VCS toolwindow that logs all write-commands performed by IDE. vcs.log.bek.sort.disabled=false diff --git a/plugins/jps-cache/resources/META-INF/plugin.xml b/plugins/jps-cache/resources/META-INF/plugin.xml index 880a97dd2251..383b6a542313 100644 --- a/plugins/jps-cache/resources/META-INF/plugin.xml +++ b/plugins/jps-cache/resources/META-INF/plugin.xml @@ -19,9 +19,9 @@ com.intellij.modules.java - + + + diff --git a/plugins/jps-cache/src/com/intellij/jps/cache/loader/JpsOutputLoaderManager.java b/plugins/jps-cache/src/com/intellij/jps/cache/loader/JpsOutputLoaderManager.java index 77a28b275e0c..0810b7e2febd 100644 --- a/plugins/jps-cache/src/com/intellij/jps/cache/loader/JpsOutputLoaderManager.java +++ b/plugins/jps-cache/src/com/intellij/jps/cache/loader/JpsOutputLoaderManager.java @@ -79,7 +79,7 @@ public class JpsOutputLoaderManager implements Disposable { myWorkspaceConfiguration = CompilerWorkspaceConfiguration.getInstance(myProject); // Configure build manager BuildManager buildManager = BuildManager.getInstance(); - if (!buildManager.isGeneratePortableCachesEnabled()) buildManager.setGeneratePortableCachesEnabled(true); + //if (!buildManager.isGeneratePortableCachesEnabled()) buildManager.setGeneratePortableCachesEnabled(true); } public void load(boolean isForceUpdate, boolean verbose) {