MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-01-08 15:09:39 +07:00
Code Issues Packages Projects Releases Wiki Activity

[jps cache] Remove source of JPS Cache plugin. The logic was integrated into the JPS

Browse Source 
GitOrigin-RevId: e023af8d5cfab2030b965d61799dca815b0fa429
This commit is contained in:
Mikhail Mazurkevich
2022-05-11 13:28:39 +03:00
committed by intellij-monorepo-bot
parent 915ea701a7
commit 4ef77f9fa9
45 changed files with 0 additions and 3083 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.idea/modules.xml generated
View File

@@ -658,7 +658,6 @@
<module fileurl="file://$PROJECT_DIR$/platform/script-debugger/protocol/protocol-model-generator/intellij.javascript.protocolModelGenerator.iml" filepath="$PROJECT_DIR$/platform/script-debugger/protocol/protocol-model-generator/intellij.javascript.protocolModelGenerator.iml" />
<module fileurl="file://$PROJECT_DIR$/platform/script-debugger/protocol/protocol-reader/intellij.javascript.protocolReader.iml" filepath="$PROJECT_DIR$/platform/script-debugger/protocol/protocol-reader/intellij.javascript.protocolReader.iml" />
<module fileurl="file://$PROJECT_DIR$/platform/script-debugger/protocol/schema-reader-generator/intellij.javascript.schemaReaderGenerator.iml" filepath="$PROJECT_DIR$/platform/script-debugger/protocol/schema-reader-generator/intellij.javascript.schemaReaderGenerator.iml" />
<module fileurl="file://$PROJECT_DIR$/plugins/jps-cache/intellij.jps.cache.iml" filepath="$PROJECT_DIR$/plugins/jps-cache/intellij.jps.cache.iml" />
<module fileurl="file://$PROJECT_DIR$/json/intellij.json.iml" filepath="$PROJECT_DIR$/json/intellij.json.iml" />
<module fileurl="file://$PROJECT_DIR$/json/tests/intellij.json.tests.iml" filepath="$PROJECT_DIR$/json/tests/intellij.json.tests.iml" />
<module fileurl="file://$PROJECT_DIR$/java/jsp-openapi/intellij.jsp.iml" filepath="$PROJECT_DIR$/java/jsp-openapi/intellij.jsp.iml" />

12
docs/plugin-graph/plugin-graph.json
View File

@@ -4273,18 +4273,6 @@
"pluginId": "XPathView",
"type": 1
}
}, {
"group": "nodes",
"data": {
"id": "QD1",
"name": "intellij.jps.cache",
"n": "i.jps.cache",
"package": null,
"sourceModule": "intellij.jps.cache",
"descriptor": "community/plugins/jps-cache/resources/META-INF/plugin.xml",
"pluginId": "com.jetbrains.jps.cache",
"type": 1
}
}, {
"group": "nodes",
"data": {

2
platform/build-scripts/groovy/org/jetbrains/intellij/build/CommunityLibraryLicenses.kt
View File

@@ -760,8 +760,6 @@ object CommunityLibraryLicenses {
licenseUrl = "https://git.tukaani.org/?p=xz-java.git;a=blob;f=COPYING;h=8dd17645c4610c3d5eed9bcdd2699ecfac00406b;hb=refs/heads/master"),
LibraryLicense(name = "zip-signer", libraryName = "zip-signer",
url = "https://github.com/JetBrains/marketplace-zip-signer").apache(),
LibraryLicense(name = "Zstd-JNI", libraryName = "com.github.luben:zstd",
url = "https://raw.githubusercontent.com/luben/zstd-jni/master/LICENSE").simplifiedBsd(),
jetbrainsLibrary("change-reminder-prediction-model"),
jetbrainsLibrary("cloud-config-client"),

1
platform/build-scripts/groovy/org/jetbrains/intellij/build/CommunityRepositoryModules.groovy
View File

@@ -206,7 +206,6 @@ final class CommunityRepositoryModules {
withModule("intellij.ml.models.local.java")
bundlingRestrictions.includeInEapOnly = true
},
plugin("intellij.jps.cache"),
plugin("intellij.lombok") {
withModule("intellij.lombok.generated")
},

29
plugins/jps-cache/README.md
View File

@@ -1,29 +0,0 @@
## JPS Portable Cache Loader [\[It can be downloaded from the latest installer\]](https://buildserver.labs.intellij.net/buildConfiguration/ijplatform_master_Idea_Installers)
This plugin develops for reducing daily spending time for waiting compilation after pull from repository.
The goal achieves by download already built compilations outputs and caches produced by the JPS build system. This data provides an incremental compilation.
### Important note
**If you think that compilation outputs old enough you can remove them and the plugin will download the fresh one.**
### Enabling
For its work, you should enable plugin. After that, each time when you change the commit
you will get a notification with a proposal to download already existing on the server caches (if they exist).
- Example of notification which you will get after pull: `Compile server contains caches for the 1th commit behind of yours. Do you want to update your data?`
- Example of notification on success loading: `Update compilation caches completed successfully in 142 s` on fail: `Update compilation caches failed`
What was done:
- [x] JPS changes for producing portable and deterministic caches via relativisers (covered only Intellij cases)
- [x] CLI tool for comparing JPS caches
- [x] TC job for providing caches and compilation outputs for as many commits as possible
- [x] Plugin for loading and merging all related for JPS data (it's also possible to upload your own caches to server)
- [x] Create unambiguous matching between module sources and existing compilation outputs (report produced by JPS work)
What should be done(first priority):
- [ ] Reduce download size (for now JPS caches entirely loads for commit around 300Mb)
- [x] Make the same mechanic to provide incremental compilation for any TC agent
#### Additional info
Globally available cache server protected by Space authorization [link](https://cache-redirector.jetbrains.com/www.jetbrains.com/jps-cache/intellij)
List of commits existing on server [link](https://cache-redirector.jetbrains.com/www.jetbrains.com/jps-cache/intellij/commit_history.json)
CI project [link](https://buildserver.labs.intellij.net/project/ijplatform_master_Idea_Experiments_Jps_Caches_Project?mode=builds)

41
plugins/jps-cache/intellij.jps.cache.iml
View File

@@ -1,41 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/resources" type="java-resource" />
<sourceFolder url="file://$MODULE_DIR$/tests" isTestSource="true" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="module" module-name="intellij.platform.lang" />
<orderEntry type="module" module-name="intellij.platform.ide.impl" />
<orderEntry type="module" module-name="intellij.vcs.git" />
<orderEntry type="module" module-name="intellij.platform.vcs.core" />
<orderEntry type="module" module-name="intellij.platform.vcs.log" />
<orderEntry type="module" module-name="intellij.platform.vcs.log.impl" />
<orderEntry type="module" module-name="intellij.java.compiler.impl" />
<orderEntry type="module" module-name="intellij.platform.jps.model.impl" />
<orderEntry type="library" name="jackson" level="project" />
<orderEntry type="library" name="jackson-databind" level="project" />
<orderEntry type="library" name="gson" level="project" />
<orderEntry type="library" name="Guava" level="project" />
<orderEntry type="module" module-name="intellij.platform.testFramework" scope="TEST" />
<orderEntry type="module" module-name="intellij.platform.ide.util.io" />
<orderEntry type="module" module-name="intellij.platform.statistics" />
<orderEntry type="module-library">
<library name="com.github.luben:zstd" type="repository">
<properties maven-id="com.github.luben:zstd-jni:1.4.0-1" />
<CLASSES>
<root url="jar://$MAVEN_REPOSITORY$/com/github/luben/zstd-jni/1.4.0-1/zstd-jni-1.4.0-1.jar!/" />
</CLASSES>
<JAVADOC>
<root url="jar://$MAVEN_REPOSITORY$/com/github/luben/zstd-jni/1.4.0-1/zstd-jni-1.4.0-1-javadoc.jar!/" />
</JAVADOC>
<SOURCES />
</library>
</orderEntry>
<orderEntry type="module" module-name="intellij.platform.util.jdom" />
</component>
</module>

47
plugins/jps-cache/resources/META-INF/plugin.xml
View File

@@ -1,47 +0,0 @@
<idea-plugin>
<id>com.jetbrains.jps.cache</id>
<name>JPS Cache</name>
<vendor>JetBrains</vendor>
<idea-version since-build="201.3803.71"/>
<description><![CDATA[
<p>
This plugin provides an opportunity to download already precompiled data for the Intellij project to reduce time,
consumed by the compilation.<br/>For the correct work, the <b>JetBrains Internal Authentication</b> plugin should be enabled.
It will let you download data faster and without VPN.
</p>
<br/>
<br/>
<b>This plugin is for JetBrains internal use only.</b>
]]></description>
<depends>Git4Idea</depends>
<depends>com.intellij.modules.java</depends>
<extensions defaultExtensionNs="com.intellij">
<projectService serviceImplementation="com.intellij.jps.cache.loader.JpsOutputLoaderManager"/>
<postStartupActivity implementation="com.intellij.jps.cache.JpsCacheStartupActivity"/>
<statistics.counterUsagesCollector implementationClass="com.intellij.jps.cache.statistics.JpsCacheUsagesCollector"/>
<notificationGroup id="Compile Output Loader: Event Log" displayType="NONE"/>
<notificationGroup id="Compile Output Loader: Attention" displayType="STICKY_BALLOON"/>
<notificationGroup id="Compile Output Loader: Standard" displayType="BALLOON"/>
<registryKey key="jps.cache.check.internet.connection" defaultValue="false"
description="Enable checking for internet connection if download retry is executed"/>
</extensions>
<projectListeners>
<listener class="com.intellij.jps.cache.JpsCachesProjectStateListener" topic="git4idea.repo.GitRepositoryChangeListener"/>
</projectListeners>
<actions>
<action id="JpsUpdateCachesAction" class="com.intellij.jps.cache.action.JpsUpdateCachesAction">
<add-to-group group-id="BuildMenu" anchor="last"/>
</action>
<action id="JpsForceUpdateCachesAction" class="com.intellij.jps.cache.action.JpsForceUpdateCachesAction">
<add-to-group group-id="BuildMenu" anchor="last"/>
</action>
</actions>
<resource-bundle>messages.JpsCacheBundle</resource-bundle>
</idea-plugin>

35
plugins/jps-cache/resources/messages/JpsCacheBundle.properties
View File

@@ -1,35 +0,0 @@
action.NotificationAction.JpsCachesDummyProjectComponent.text.disable.property=Disable property
action.NotificationAction.JpsCachesDummyProjectComponent.text.dont.ask=Don't ask again
action.NotificationAction.JpsOutputLoaderManager.text.update.caches=Update caches
notification.title.git.crlf.config=Wrong Git line-endings configuration
notification.content.git.crlf.config=Git line-endings not properly configured for the project. Portable JPS caches can't work correctly with such config.
notification.action.git.crlf.config=Open config description
notification.title.automatic.project.build.enabled=Automatic project build enabled
notification.content.make.project.automatically.enabled.affect.caches="Build project automatically" property is not \
yet supported for JPS portable caches, may lead to full project rebuild
notification.content.failed.request.to.cache.server=Failed request to cache server: {0}
notification.title.compiler.caches.loader=Compiler caches loader
notification.content.update.compiler.caches.failed=Update compiler caches failed
progress.text.rolling.back.downloaded.caches=Rolling back downloaded caches
notification.content.update.compiler.caches.completed.successfully.in.s=Update compiler caches completed successfully in {0} s
progress.text.rolling.back=Rolling back
progress.text.fetching.cache.for.commit=Fetching cache for commit: {0}
notification.title.jps.caches.downloader=Jps caches downloader
notification.content.system.contains.up.to.date.caches=The system contains up-to-date caches
notification.content.not.found.any.caches.for.latest.commits.in.branch=Not found any caches for the latest commits in the branch
notification.title.compiler.caches.available=Compiler caches available
notification.content.caches.are.for.commit.commits.prior.to.yours=Caches are available for a commit, which is {0} {0,choice,1#commit|2#commits} prior to your current revision.
notification.content.caches.are.for.current.commit=Caches are available for the current revision.
progress.text.clean.output.directories=Clean output directories
progress.details.applying.downloaded.caches=Applying downloaded caches
progress.details.extracting.project.caches=Extracting project caches
progress.details.extracting.compilation.outputs.for.module=Extracting compilation outputs for {0} module
progress.details.applying.changes.for.module=Applying changes for {0} module
notification.content.internal.authentication.plugin.required.for.correct.work.plugin=<a href="https://plugins.jetbrains.com/plugin/14567-jetbrains-internal-authentication">JetBrains Internal Authentication</a> is required for the correct work of the plugin
internal.authentication.plugin.missing.token=Unexpected state: jetbrains.team token is missing. Please report this exception and perform actions 'Logout from JetBrains.team' and 'Login to JetBrains.team' as a workaround
progress.title.updating.compiler.caches=Updating compiler caches
progress.text.applying.jps.caches=Applying JPS Caches...
progress.text.extracting.downloaded.results=Extracting downloaded results...
progress.text.calculating.affected.modules=Calculating affected modules
action.JpsUpdateCachesAction.text=Update JPS Caches
action.JpsForceUpdateCachesAction.text=Force Update JPS Caches

28
plugins/jps-cache/src/com/intellij/jps/cache/JpsCacheBundle.java vendored
View File

@@ -1,28 +0,0 @@
package com.intellij.jps.cache;
import com.intellij.DynamicBundle;
import org.jetbrains.annotations.Nls;
import org.jetbrains.annotations.NonNls;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.PropertyKey;
import java.util.function.Supplier;
public final class JpsCacheBundle extends DynamicBundle {
@NonNls private static final String BUNDLE = "messages.JpsCacheBundle";
private static final JpsCacheBundle INSTANCE = new JpsCacheBundle();
private JpsCacheBundle() {
super(BUNDLE);
}
@NotNull
public static @Nls String message(@NotNull @PropertyKey(resourceBundle = BUNDLE) String key, Object @NotNull ... params) {
return INSTANCE.getMessage(key, params);
}
@NotNull
public static Supplier<@Nls String> messagePointer(@NotNull @PropertyKey(resourceBundle = BUNDLE) String key, Object @NotNull ... params) {
return INSTANCE.getLazyMessage(key, params);
}
}

76
plugins/jps-cache/src/com/intellij/jps/cache/JpsCacheStartupActivity.java vendored
View File

@@ -1,76 +0,0 @@
package com.intellij.jps.cache;
import com.intellij.compiler.CompilerWorkspaceConfiguration;
import com.intellij.compiler.server.BuildManager;
import com.intellij.ide.browsers.BrowserLauncher;
import com.intellij.ide.impl.TrustedProjects;
import com.intellij.ide.util.PropertiesComponent;
import com.intellij.jps.cache.git.GitRepositoryUtil;
import com.intellij.notification.NotificationAction;
import com.intellij.notification.NotificationType;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.startup.StartupActivity;
import com.intellij.openapi.util.SystemInfo;
import git4idea.repo.GitRepository;
import org.jetbrains.annotations.NotNull;
import static com.intellij.jps.cache.JpsCachesPluginUtil.INTELLIJ_REPO_NAME;
import static com.intellij.jps.cache.ui.JpsLoaderNotifications.ATTENTION;
public final class JpsCacheStartupActivity implements StartupActivity.DumbAware {
private static final String NOT_ASK_AGAIN = "JpsCaches.NOT_ASK_AGAIN";
private static boolean lineEndingsConfiguredCorrectly = true;
@Override
public void runActivity(@NotNull Project project) {
checkWindowsCRLF(project);
checkAutoBuildEnabled(project);
}
private static void checkWindowsCRLF(@NotNull Project project) {
if (!SystemInfo.isWindows) return;
GitRepository intellijRepository = GitRepositoryUtil.getRepositoryByName(project, INTELLIJ_REPO_NAME);
if (intellijRepository == null) return;
if (!GitRepositoryUtil.isAutoCrlfSetRight(intellijRepository)) {
lineEndingsConfiguredCorrectly = false;
ATTENTION
.createNotification(JpsCacheBundle.message("notification.title.git.crlf.config"),
JpsCacheBundle.message("notification.content.git.crlf.config"),
NotificationType.WARNING)
.addAction(NotificationAction.createSimple(JpsCacheBundle.message("notification.action.git.crlf.config"),
() -> BrowserLauncher.getInstance().open("https://confluence.jetbrains.com/pages/viewpage.action?title=Git+Repository&spaceKey=IDEA")))
.notify(project);
}
}
private static void checkAutoBuildEnabled(@NotNull Project project) {
PropertiesComponent propertiesComponent = PropertiesComponent.getInstance(project);
if (propertiesComponent.getBoolean(NOT_ASK_AGAIN)) {
return;
}
CompilerWorkspaceConfiguration workspaceConfiguration = CompilerWorkspaceConfiguration.getInstance(project);
if (!workspaceConfiguration.MAKE_PROJECT_ON_SAVE || !TrustedProjects.isTrusted(project)) {
return;
}
ATTENTION
.createNotification(JpsCacheBundle.message("notification.title.automatic.project.build.enabled"),
JpsCacheBundle.message("notification.content.make.project.automatically.enabled.affect.caches"),
NotificationType.WARNING)
.addAction(NotificationAction.createSimpleExpiring(
JpsCacheBundle.message("action.NotificationAction.JpsCachesDummyProjectComponent.text.disable.property"), () -> {
workspaceConfiguration.MAKE_PROJECT_ON_SAVE = false;
BuildManager.getInstance().clearState(project);
}))
.addAction(NotificationAction.createSimpleExpiring(
JpsCacheBundle.message("action.NotificationAction.JpsCachesDummyProjectComponent.text.dont.ask"), () -> {
propertiesComponent.setValue(NOT_ASK_AGAIN, true);
}))
.notify(project);
}
public static boolean isLineEndingsConfiguredCorrectly() {
return lineEndingsConfiguredCorrectly;
}
}

23
plugins/jps-cache/src/com/intellij/jps/cache/JpsCachesPluginUtil.java vendored
View File

@@ -1,23 +0,0 @@
package com.intellij.jps.cache;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.util.concurrency.AppExecutorUtil;
import java.util.concurrent.ExecutorService;
import static com.intellij.execution.process.ProcessIOExecutorService.INSTANCE;
public final class JpsCachesPluginUtil {
private static final Logger LOG = Logger.getInstance(JpsCachesPluginUtil.class);
public static final String PLUGIN_NAME = "jps-cache-loader";
public static final String INTELLIJ_REPO_NAME = "intellij.git";
public static final ExecutorService EXECUTOR_SERVICE = AppExecutorUtil.createBoundedApplicationPoolExecutor("JpsCacheLoader Pool",
INSTANCE, getThreadPoolSize());
private JpsCachesPluginUtil() {}
private static int getThreadPoolSize() {
int threadsCount = Runtime.getRuntime().availableProcessors() - 1;
LOG.info("Executor service will be configured with " + threadsCount + " threads");
return threadsCount;
}
}

33
plugins/jps-cache/src/com/intellij/jps/cache/JpsCachesProjectStateListener.java vendored
View File

@@ -1,33 +0,0 @@
package com.intellij.jps.cache;
import com.intellij.jps.cache.client.JpsServerAuthUtil;
import com.intellij.jps.cache.loader.JpsOutputLoaderManager;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.vcs.log.Hash;
import git4idea.GitLocalBranch;
import git4idea.repo.GitRepository;
import git4idea.repo.GitRepositoryChangeListener;
import org.jetbrains.annotations.NotNull;
public class JpsCachesProjectStateListener implements GitRepositoryChangeListener {
private static final Logger LOG = Logger.getInstance(JpsCachesProjectStateListener.class);
private String previousCommitId = "";
@Override
public void repositoryChanged(@NotNull GitRepository repository) {
GitLocalBranch branch = repository.getCurrentBranch();
if (branch == null) {
LOG.warn("Repository is not on a branch");
return;
}
Hash commitHash = repository.getInfo().getRemoteBranchesWithHashes().get(branch.findTrackedBranch(repository));
if (commitHash == null) return;
String currentRevision = commitHash.toString();
if (currentRevision == null || previousCommitId.equals(currentRevision)) return;
previousCommitId = currentRevision;
LOG.info("Remote repository commit changed to " + currentRevision);
JpsOutputLoaderManager outputLoaderManager = JpsOutputLoaderManager.getInstance(repository.getProject());
JpsServerAuthUtil.checkAuthenticatedInBackgroundThread(outputLoaderManager, repository.getProject(),
() -> outputLoaderManager.notifyAboutNearestCache());
}
}

18
plugins/jps-cache/src/com/intellij/jps/cache/action/JpsForceUpdateCachesAction.java vendored
View File

@@ -1,18 +0,0 @@
package com.intellij.jps.cache.action;
import com.intellij.jps.cache.client.JpsServerAuthUtil;
import com.intellij.jps.cache.loader.JpsOutputLoaderManager;
import com.intellij.openapi.actionSystem.AnActionEvent;
import com.intellij.openapi.project.DumbAwareAction;
import com.intellij.openapi.project.Project;
import org.jetbrains.annotations.NotNull;
public class JpsForceUpdateCachesAction extends DumbAwareAction {
@Override
public void actionPerformed(@NotNull AnActionEvent actionEvent) {
Project project = actionEvent.getProject();
if (project == null) return;
JpsOutputLoaderManager outputLoaderManager = JpsOutputLoaderManager.getInstance(project);
JpsServerAuthUtil.checkAuthenticatedInBackgroundThread(outputLoaderManager, project, () -> outputLoaderManager.load(true, true));
}
}

18
plugins/jps-cache/src/com/intellij/jps/cache/action/JpsUpdateCachesAction.java vendored
View File

@@ -1,18 +0,0 @@
package com.intellij.jps.cache.action;
import com.intellij.jps.cache.client.JpsServerAuthUtil;
import com.intellij.jps.cache.loader.JpsOutputLoaderManager;
import com.intellij.openapi.actionSystem.AnActionEvent;
import com.intellij.openapi.project.DumbAwareAction;
import com.intellij.openapi.project.Project;
import org.jetbrains.annotations.NotNull;
public class JpsUpdateCachesAction extends DumbAwareAction {
@Override
public void actionPerformed(@NotNull AnActionEvent actionEvent) {
Project project = actionEvent.getProject();
if (project == null) return;
JpsOutputLoaderManager outputLoaderManager = JpsOutputLoaderManager.getInstance(project);
JpsServerAuthUtil.checkAuthenticatedInBackgroundThread(outputLoaderManager, project, () -> outputLoaderManager.load(false, true));
}
}

186
plugins/jps-cache/src/com/intellij/jps/cache/client/JpsCachesDownloader.java vendored
View File

@@ -1,186 +0,0 @@
package com.intellij.jps.cache.client;
import com.intellij.ide.IdeCoreBundle;
import com.intellij.internal.statistic.eventLog.events.EventId1;
import com.intellij.jps.cache.ui.SegmentedProgressIndicatorManager;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.progress.ProcessCanceledException;
import com.intellij.openapi.progress.ProgressIndicator;
import com.intellij.openapi.util.Pair;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.registry.Registry;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.util.download.DownloadableFileDescription;
import com.intellij.util.io.HttpRequests;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.io.File;
import java.io.IOException;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicLong;
import static com.intellij.jps.cache.JpsCachesPluginUtil.EXECUTOR_SERVICE;
class JpsCachesDownloader {
private static final Logger LOG = Logger.getInstance(JpsCachesDownloader.class);
private static final byte MAX_RETRY_COUNT = 3;
private static final String CDN_CACHE_HEADER = "X-Cache";
private int hitsCount = 0;
private final List<DownloadableFileDescription> myFilesDescriptions;
private final SegmentedProgressIndicatorManager myProgressIndicatorManager;
JpsCachesDownloader(@NotNull List<DownloadableFileDescription> filesDescriptions,
@NotNull SegmentedProgressIndicatorManager indicatorManager) {
myFilesDescriptions = filesDescriptions;
myProgressIndicatorManager = indicatorManager;
}
@NotNull
List<Pair<File, DownloadableFileDescription>> download(@NotNull File targetDir, @Nullable EventId1<Long> eventId) throws IOException {
List<Pair<File, DownloadableFileDescription>> downloadedFiles = new CopyOnWriteArrayList<>();
List<Pair<File, DownloadableFileDescription>> existingFiles = new CopyOnWriteArrayList<>();
try {
myProgressIndicatorManager.setText(this, IdeCoreBundle.message("progress.downloading.0.files.text", myFilesDescriptions.size()));
long start = System.currentTimeMillis();
List<Future<Void>> results = new ArrayList<>();
final AtomicLong totalSize = new AtomicLong();
for (final DownloadableFileDescription description : myFilesDescriptions) {
results.add(EXECUTOR_SERVICE.submit(() -> {
SegmentedProgressIndicatorManager.SubTaskProgressIndicator indicator = myProgressIndicatorManager.createSubTaskIndicator();
indicator.checkCanceled();
final File existing = new File(targetDir, description.getDefaultFileName());
byte attempt = 0;
File downloaded = null;
while (downloaded == null && attempt++ < MAX_RETRY_COUNT) {
try {
downloaded = downloadFile(description, existing, indicator);
} catch (IOException e) {
int httpStatusCode = -1;
if (e instanceof HttpRequests.HttpStatusException) {
httpStatusCode = ((HttpRequests.HttpStatusException)e).getStatusCode();
if (httpStatusCode == 404) {
LOG.info("File not found to download " + description.getDownloadUrl());
indicator.finished();
return null;
}
} else {
if (Registry.is("jps.cache.check.internet.connection")){
JpsServerConnectionUtil.checkDomainIsReachable("google.com");
JpsServerConnectionUtil.checkDomainIsReachable("d1lc5k9lerg6km.cloudfront.net");
JpsServerConnectionUtil.checkDomainRouting("d1lc5k9lerg6km.cloudfront.net");
}
}
// If max attempt count exceeded, rethrow exception further
if (attempt != MAX_RETRY_COUNT) {
if (httpStatusCode != -1) {
LOG.info("Failed to download " + description.getDownloadUrl() + " HTTP code: " + httpStatusCode + ". Attempt " + attempt + " to download file again");
} else {
LOG.info("Failed to download " + description.getDownloadUrl() + " Root cause: " + e + ". Attempt " + attempt + " to download file again");
}
Thread.sleep(250);
} else {
throw new IOException(IdeCoreBundle.message("error.file.download.failed", description.getDownloadUrl(), e.getMessage()), e);
}
}
}
assert downloaded != null : "Download result shouldn't be NULL";
if (FileUtil.filesEqual(downloaded, existing)) {
existingFiles.add(Pair.create(existing, description));
}
else {
totalSize.addAndGet(downloaded.length());
downloadedFiles.add(Pair.create(downloaded, description));
}
indicator.finished();
return null;
}));
}
for (Future<Void> result : results) {
try {
result.get();
}
catch (InterruptedException e) {
throw new ProcessCanceledException();
}
catch (ExecutionException e) {
if (e.getCause() instanceof IOException) {
throw ((IOException)e.getCause());
}
if (e.getCause() instanceof ProcessCanceledException) {
throw ((ProcessCanceledException)e.getCause());
}
LOG.error(e);
}
}
long duration = System.currentTimeMillis() - start;
if (eventId != null) eventId.log(totalSize.get());
LOG.info("Downloaded " + StringUtil.formatFileSize(totalSize.get()) + " in " + StringUtil.formatDuration(duration) +
"(" + duration + "ms). Percentage of CDN cache hits: " + (hitsCount * 100/myFilesDescriptions.size()) + "%");
List<Pair<File, DownloadableFileDescription>> localFiles = new ArrayList<>();
localFiles.addAll(moveToDir(downloadedFiles, targetDir));
localFiles.addAll(existingFiles);
myProgressIndicatorManager.finished(this);
return localFiles;
}
catch (ProcessCanceledException | IOException e) {
for (Pair<File, DownloadableFileDescription> pair : downloadedFiles) {
FileUtil.delete(pair.getFirst());
}
throw e;
}
}
@NotNull
private File downloadFile(@NotNull final DownloadableFileDescription description, @NotNull final File existingFile,
@NotNull final ProgressIndicator indicator) throws IOException {
final String presentableUrl = description.getPresentableDownloadUrl();
Map<String, String> headers = JpsServerAuthUtil.getRequestHeaders();
indicator.setText2(IdeCoreBundle.message("progress.connecting.to.download.file.text", presentableUrl));
indicator.setIndeterminate(false);
return HttpRequests.request(description.getDownloadUrl())
.tuner(tuner -> headers.forEach((k, v) -> tuner.addRequestProperty(k, v)))
.connect(new HttpRequests.RequestProcessor<>() {
@Override
public File process(@NotNull HttpRequests.Request request) throws IOException {
URLConnection connection = request.getConnection();
int size = connection.getContentLength();
if (existingFile.exists() && size == existingFile.length()) {
return existingFile;
}
String header = connection.getHeaderField(CDN_CACHE_HEADER);
if (header != null && header.startsWith("Hit")) hitsCount++;
indicator.setText2(IdeCoreBundle.message("progress.download.file.text", description.getPresentableFileName(), presentableUrl));
return request.saveToFile(FileUtil.createTempFile("download.", ".tmp"), indicator);
}
});
}
private static List<Pair<File, DownloadableFileDescription>> moveToDir(List<Pair<File, DownloadableFileDescription>> downloadedFiles,
final File targetDir) throws IOException {
FileUtil.createDirectory(targetDir);
List<Pair<File, DownloadableFileDescription>> result = new ArrayList<>();
for (Pair<File, DownloadableFileDescription> pair : downloadedFiles) {
final DownloadableFileDescription description = pair.getSecond();
final String fileName = description.generateFileName(s -> !new File(targetDir, s).exists());
final File toFile = new File(targetDir, fileName);
FileUtil.rename(pair.getFirst(), toFile);
result.add(Pair.create(toFile, description));
}
return result;
}
}

65
plugins/jps-cache/src/com/intellij/jps/cache/client/JpsServerAuthUtil.java vendored
View File

@@ -1,65 +0,0 @@
package com.intellij.jps.cache.client;
import com.intellij.compiler.cache.client.JpsServerAuthExtension;
import com.intellij.jps.cache.JpsCacheBundle;
import com.intellij.notification.NotificationListener;
import com.intellij.notification.NotificationType;
import com.intellij.openapi.Disposable;
import com.intellij.openapi.application.ApplicationManager;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.util.Disposer;
import com.intellij.openapi.util.Key;
import org.jetbrains.annotations.NotNull;
import java.util.Map;
import static com.intellij.execution.process.ProcessIOExecutorService.INSTANCE;
import static com.intellij.jps.cache.ui.JpsLoaderNotifications.ATTENTION;
public final class JpsServerAuthUtil {
private static final Logger LOG = Logger.getInstance(JpsServerAuthUtil.class);
private static final Key<Boolean> NOTIFICATION_SHOWN_KEY = Key.create("AUTH_NOTIFICATION_SHOWN");
public static void checkAuthenticatedInBackgroundThread(@NotNull Disposable parentDisposable,
@NotNull Project project,
@NotNull Runnable onAuthCompleted) {
Disposable disposable = Disposer.newDisposable();
Disposer.register(parentDisposable, disposable);
JpsServerAuthExtension authExtension = JpsServerAuthExtension.getInstance();
if (authExtension == null) {
Boolean userData = project.getUserData(NOTIFICATION_SHOWN_KEY);
if (userData == null) {
project.putUserData(NOTIFICATION_SHOWN_KEY, Boolean.TRUE);
ApplicationManager.getApplication().invokeLater(() -> {
ATTENTION
.createNotification(JpsCacheBundle.message("notification.title.jps.caches.downloader"), JpsCacheBundle.message("notification.content.internal.authentication.plugin.required.for.correct.work.plugin"), NotificationType.WARNING)
.setListener(NotificationListener.URL_OPENING_LISTENER)
.notify(project);
});
}
LOG.warn("JetBrains Internal Authentication plugin is required for the correct work. Please enable it.");
return;
}
INSTANCE.execute(() -> {
authExtension.checkAuthenticated("Jps Caches Downloader", disposable, () -> {
Disposer.dispose(disposable);
onAuthCompleted.run();
});
});
}
static @NotNull Map<String, String> getRequestHeaders() {
JpsServerAuthExtension authExtension = JpsServerAuthExtension.getInstance();
if (authExtension == null) {
String message = JpsCacheBundle.message("notification.content.internal.authentication.plugin.required.for.correct.work.plugin");
throw new RuntimeException(message);
}
Map<String, String> authHeader = authExtension.getAuthHeader(false);
if (authHeader == null) {
String message = JpsCacheBundle.message("internal.authentication.plugin.missing.token");
throw new RuntimeException(message);
}
return authHeader;
}
}

27
plugins/jps-cache/src/com/intellij/jps/cache/client/JpsServerClient.java vendored
View File

@@ -1,27 +0,0 @@
package com.intellij.jps.cache.client;
import com.intellij.jps.cache.model.AffectedModule;
import com.intellij.jps.cache.model.OutputLoadResult;
import com.intellij.jps.cache.ui.SegmentedProgressIndicatorManager;
import com.intellij.openapi.project.Project;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.io.File;
import java.util.List;
import java.util.Map;
import java.util.Set;
public interface JpsServerClient {
@NotNull
Map<String, Set<String>> getCacheKeysPerRemote(@NotNull Project project);
@Nullable
File downloadMetadataById(@NotNull String metadataId, @NotNull File targetDir);
File downloadCacheById(@NotNull SegmentedProgressIndicatorManager downloadIndicatorManager, @NotNull String cacheId,
@NotNull File targetDir);
List<OutputLoadResult> downloadCompiledModules(@NotNull SegmentedProgressIndicatorManager downloadIndicatorManager,
@NotNull List<AffectedModule> affectedModules);
static JpsServerClient getServerClient() {
return JpsServerClientImpl.INSTANCE;
}
}

201
plugins/jps-cache/src/com/intellij/jps/cache/client/JpsServerClientImpl.java vendored
View File

@@ -1,201 +0,0 @@
package com.intellij.jps.cache.client;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.intellij.jps.cache.JpsCacheBundle;
import com.intellij.jps.cache.JpsCachesPluginUtil;
import com.intellij.jps.cache.git.GitRepositoryUtil;
import com.intellij.jps.cache.model.AffectedModule;
import com.intellij.jps.cache.model.OutputLoadResult;
import com.intellij.jps.cache.ui.JpsLoaderNotifications;
import com.intellij.jps.cache.ui.SegmentedProgressIndicatorManager;
import com.intellij.notification.NotificationType;
import com.intellij.openapi.application.PathManager;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.progress.ProcessCanceledException;
import com.intellij.openapi.progress.ProgressIndicator;
import com.intellij.openapi.progress.ProgressManager;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.util.Pair;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.io.StreamUtil;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.download.DownloadableFileDescription;
import com.intellij.util.download.DownloadableFileService;
import com.intellij.util.io.HttpRequests;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URLConnection;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.stream.Collectors;
import java.util.zip.GZIPInputStream;
import static com.intellij.jps.cache.statistics.JpsCacheUsagesCollector.DOWNLOAD_BINARY_SIZE_EVENT_ID;
import static com.intellij.jps.cache.statistics.JpsCacheUsagesCollector.DOWNLOAD_CACHE_SIZE_EVENT_ID;
public final class JpsServerClientImpl implements JpsServerClient {
private static final Logger LOG = Logger.getInstance(JpsServerClientImpl.class);
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
static final JpsServerClientImpl INSTANCE = new JpsServerClientImpl();
private final String stringThree;
private JpsServerClientImpl() {
byte[] decodedBytes = Base64.getDecoder().decode("aHR0cHM6Ly9kMWxjNWs5bGVyZzZrbS5jbG91ZGZyb250Lm5ldA==");
stringThree = new String(decodedBytes, StandardCharsets.UTF_8);
}
@NotNull
@Override
public Map<String, Set<String>> getCacheKeysPerRemote(@NotNull Project project) {
Map<String, List<String>> response = doGetRequest(project);
if (response == null) return Collections.emptyMap();
Map<String, Set<String>> result = new HashMap<>();
response.forEach((key, value) -> result.put(GitRepositoryUtil.getRemoteRepoName(key), new HashSet<>(value)));
return result;
}
@Nullable
@Override
public File downloadMetadataById(@NotNull String metadataId, @NotNull File targetDir) {
String downloadUrl = stringThree + "/metadata/" + metadataId;
DownloadableFileService service = DownloadableFileService.getInstance();
String fileName = "metadata.json";
DownloadableFileDescription description = service.createFileDescription(downloadUrl, fileName);
ProgressIndicator progressIndicator = ProgressManager.getInstance().getProgressIndicator();
JpsCachesDownloader downloader = new JpsCachesDownloader(Collections.singletonList(description),
new SegmentedProgressIndicatorManager(progressIndicator));
LOG.debug("Downloading JPS metadata from: " + downloadUrl);
File metadataFile;
try {
List<Pair<File, DownloadableFileDescription>> pairs = downloader.download(targetDir, null);
Pair<File, DownloadableFileDescription> first = ContainerUtil.getFirstItem(pairs);
metadataFile = first != null ? first.first : null;
if (metadataFile == null) {
LOG.warn("Failed to download JPS metadata");
return null;
}
return metadataFile;
}
catch (ProcessCanceledException | IOException e) {
//noinspection InstanceofCatchParameter
if (e instanceof IOException) LOG.warn("Failed to download JPS metadata from URL: " + downloadUrl, e);
return null;
}
}
@Nullable
@Override
public File downloadCacheById(@NotNull SegmentedProgressIndicatorManager downloadIndicatorManager, @NotNull String cacheId,
@NotNull File targetDir) {
String downloadUrl = stringThree + "/caches/" + cacheId;
String fileName = "portable-build-cache.zip";
DownloadableFileService service = DownloadableFileService.getInstance();
DownloadableFileDescription description = service.createFileDescription(downloadUrl, fileName);
JpsCachesDownloader downloader = new JpsCachesDownloader(Collections.singletonList(description), downloadIndicatorManager);
LOG.debug("Downloading JPS caches from: " + downloadUrl);
File zipFile;
try {
List<Pair<File, DownloadableFileDescription>> pairs = downloader.download(targetDir, DOWNLOAD_CACHE_SIZE_EVENT_ID);
downloadIndicatorManager.finished(this);
Pair<File, DownloadableFileDescription> first = ContainerUtil.getFirstItem(pairs);
zipFile = first != null ? first.first : null;
if (zipFile != null) return zipFile;
LOG.warn("Failed to download JPS caches");
}
catch (ProcessCanceledException | IOException e) {
//noinspection InstanceofCatchParameter
if (e instanceof IOException) LOG.warn("Failed to download JPS caches from URL: " + downloadUrl, e);
}
return null;
}
@Override
public List<OutputLoadResult> downloadCompiledModules(@NotNull SegmentedProgressIndicatorManager downloadIndicatorManager,
@NotNull List<AffectedModule> affectedModules) {
File targetDir = new File(PathManager.getPluginTempPath(), JpsCachesPluginUtil.PLUGIN_NAME);
if (targetDir.exists()) FileUtil.delete(targetDir);
targetDir.mkdirs();
Map<String, AffectedModule> urlToModuleNameMap = affectedModules.stream().collect(Collectors.toMap(
module -> stringThree + "/" + module.getType() + "/" + module.getName() + "/" + module.getHash(),
module -> module));
DownloadableFileService service = DownloadableFileService.getInstance();
List<DownloadableFileDescription> descriptions = ContainerUtil.map(urlToModuleNameMap.entrySet(),
entry -> service.createFileDescription(entry.getKey(),
entry.getValue().getOutPath().getName() + ".zip"));
JpsCachesDownloader downloader = new JpsCachesDownloader(descriptions, downloadIndicatorManager);
List<File> downloadedFiles = new ArrayList<>();
try {
// Downloading process
List<Pair<File, DownloadableFileDescription>> download = downloader.download(targetDir, DOWNLOAD_BINARY_SIZE_EVENT_ID);
downloadIndicatorManager.finished(this);
downloadedFiles = ContainerUtil.map(download, pair -> pair.first);
return ContainerUtil.map(download, pair -> {
String downloadUrl = pair.second.getDownloadUrl();
return new OutputLoadResult(pair.first, downloadUrl, urlToModuleNameMap.get(downloadUrl));
});
}
catch (ProcessCanceledException | IOException e) {
//noinspection InstanceofCatchParameter
if (e instanceof IOException) LOG.warn("Failed to download JPS compilation outputs", e);
if (targetDir.exists()) FileUtil.delete(targetDir);
downloadedFiles.forEach(zipFile -> FileUtil.delete(zipFile));
return null;
}
}
private @Nullable Map<String, List<String>> doGetRequest(@NotNull Project project) {
Map<String, String> headers = JpsServerAuthUtil.getRequestHeaders();
try {
return HttpRequests.request(stringThree + "/commit_history.json")
.tuner(tuner -> headers.forEach((k, v) -> tuner.addRequestProperty(k, v)))
.connect(it -> {
URLConnection connection = it.getConnection();
if (connection instanceof HttpURLConnection) {
HttpURLConnection httpConnection = (HttpURLConnection)connection;
if (httpConnection.getResponseCode() == 200) {
return OBJECT_MAPPER.readValue(getInputStream(httpConnection), new TypeReference<>() {
});
}
else {
String statusLine = httpConnection.getResponseCode() + ' ' + httpConnection.getRequestMethod();
String errorText = StreamUtil.readText(new InputStreamReader(httpConnection.getErrorStream(), StandardCharsets.UTF_8));
LOG.info("Request: " + httpConnection.getRequestMethod() + httpConnection.getURL() + " : Error " + statusLine + " body: " + errorText);
}
}
return null;
});
}
catch (IOException e) {
LOG.warn("Failed request to cache server", e);
JpsLoaderNotifications.ATTENTION
.createNotification(JpsCacheBundle.message("notification.title.compiler.caches.loader"), JpsCacheBundle.message("notification.content.failed.request.to.cache.server", e.getMessage()), NotificationType.ERROR)
.notify(project);
}
return null;
}
private static InputStream getInputStream(HttpURLConnection httpConnection) throws IOException {
String contentEncoding = httpConnection.getContentEncoding();
InputStream inputStream = httpConnection.getInputStream();
if (contentEncoding != null && StringUtil.toLowerCase(contentEncoding).contains("gzip")) {
return new GZIPInputStream(inputStream);
}
return inputStream;
}
}

128
plugins/jps-cache/src/com/intellij/jps/cache/client/JpsServerConnectionUtil.java vendored
View File

@@ -1,128 +0,0 @@
package com.intellij.jps.cache.client;
import com.intellij.execution.ExecutionException;
import com.intellij.execution.configurations.GeneralCommandLine;
import com.intellij.execution.process.ProcessOutput;
import com.intellij.execution.util.ExecUtil;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.progress.ProcessCanceledException;
import com.intellij.openapi.progress.ProgressIndicator;
import com.intellij.openapi.progress.util.ProgressWindow;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.util.SystemInfo;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.util.io.HttpRequests;
import org.jetbrains.annotations.NotNull;
import java.io.File;
import java.io.IOException;
import java.net.URLConnection;
import java.nio.charset.StandardCharsets;
import java.text.DecimalFormat;
import java.util.Base64;
import java.util.Map;
import static com.intellij.execution.process.ProcessIOExecutorService.INSTANCE;
/**
* This class was introduced to detect internet connection problems from the client side.
* One of the cases was found, the constant connection refuse at caches download was related to
* the internet provider issue.
*
* traceroute for the d1lc5k9lerg6km.cloudfront.net
* 1 * * 192.168.0.1 (192.168.0.1) 2.183 ms
* 2 100.92.128.1 (100.92.128.1) 9.851 ms 8.761 ms 9.509 ms
* 3 100.127.1.253 (100.127.1.253) 10.748 ms 9.603 ms 13.316 ms
* 4 212.48.195.38 (212.48.195.38) 30.396 ms 13.616 ms 9.258 ms
* 5 100.64.97.116 (100.64.97.116) 9.306 ms !X 8.332 ms !X 14.542 ms !X
*/
public class JpsServerConnectionUtil {
private static final Logger LOG = Logger.getInstance(JpsServerConnectionUtil.class);
private static final String CDN_CACHE_HEADER = "X-Cache";
public static void measureConnectionSpeed(@NotNull Project project) {
INSTANCE.execute(() -> {
ProgressIndicator indicator = new ProgressWindow(true, project);
indicator.setIndeterminate(false);
indicator.setFraction(0.01);
try {
Map<String, String> headers = JpsServerAuthUtil.getRequestHeaders();
long start = System.currentTimeMillis();
HttpRequests.request(calculateAddress())
.tuner(tuner -> headers.forEach((k, v) -> tuner.addRequestProperty(k, v)))
.connect(new HttpRequests.RequestProcessor<>() {
@Override
public File process(@NotNull HttpRequests.Request request) throws IOException {
URLConnection connection = request.getConnection();
int fileSize = connection.getContentLength();
String header = connection.getHeaderField(CDN_CACHE_HEADER);
File downloadedFile = request.saveToFile(FileUtil.createTempFile("download.", ".tmp"), indicator);
long downloadTime = System.currentTimeMillis() - start;
long bytesPerSecond = fileSize / downloadTime * 1000;
if (header != null && header.startsWith("Hit")) {
LOG.info("Speed of connection to CDN: " + StringUtil.formatFileSize(bytesPerSecond) + "/s; " + formatInternetSpeed(bytesPerSecond * 8));
} else {
LOG.info("Speed of connection to S3: " + StringUtil.formatFileSize(bytesPerSecond) + "/s; " + formatInternetSpeed(bytesPerSecond * 8));
}
FileUtil.delete(downloadedFile);
return downloadedFile;
}
});
}
catch (ProcessCanceledException | IOException e) {
LOG.warn("Failed to download file for measurement connection speed", e);
}
});
}
public static void checkDomainIsReachable(@NotNull String domain) {
try {
GeneralCommandLine pingCommand = new GeneralCommandLine("ping");
if (SystemInfo.isWindows) {
pingCommand.addParameter("-n 2");
} else {
pingCommand.addParameter("-c 2");
}
pingCommand.addParameter(domain);
int code = ExecUtil.execAndGetOutput(pingCommand).getExitCode();
if (code == 0) {
LOG.info("Domain " + domain + " is reachable");
} else {
LOG.info("Domain " + domain + " isn't reachable");
}
}
catch (ExecutionException e) {
LOG.warn("Failed to check if internet connection is available", e);
}
}
public static void checkDomainRouting(@NotNull String domain) {
try {
GeneralCommandLine pingCommand = new GeneralCommandLine("traceroute");
pingCommand.addParameter(domain);
ProcessOutput processOutput = ExecUtil.execAndGetOutput(pingCommand);
int code = processOutput.getExitCode();
if (code == 0) {
LOG.info("traceroute for the " + domain + "\n" + processOutput.getStdout());
} else {
LOG.info("traceroute failed with exception " + processOutput.getStderr());
}
}
catch (ExecutionException e) {
LOG.warn("Failed to execute traceroute", e);
}
}
private static @NotNull String formatInternetSpeed(long fileSize) {
int rank = (int)((Math.log10(fileSize) + 0.0000021714778384307465) / 3); // (3 - Math.log10(999.995))
double value = fileSize / Math.pow(1000, rank);
String[] units = {"Bit", "Kbit", "Mbit", "Gbit"};
return new DecimalFormat("0.##").format(value) + units[rank] + "/s";
}
private static @NotNull String calculateAddress() {
byte[] decodedBytes = Base64.getDecoder().decode("aHR0cHM6Ly9kMWxjNWs5bGVyZzZrbS5jbG91ZGZyb250Lm5ldC9FWEFNUExFLnR4dA==");
return new String(decodedBytes, StandardCharsets.UTF_8);
}
}

105
plugins/jps-cache/src/com/intellij/jps/cache/diffExperiment/JpsCacheDiffUploader.kt vendored
View File

@@ -1,105 +0,0 @@
package com.intellij.jps.cache.diffExperiment
import com.intellij.util.io.*
import java.io.File
import java.math.BigDecimal
import java.math.RoundingMode
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicInteger
import kotlin.system.exitProcess
fun main(argv: Array<String>) {
val commitHash = argv[0]
val baseUrl = argv[1]
val zip = File(argv[2]).apply { assert(exists()) { "$this doesn't exit" } }
val currentHashesFile = Paths.get(argv[3])
println("commit: $commitHash, zip: $zip(${Measures.toHumanSize(zip.length())}) url $baseUrl")
val cachesFolder = createTempDir()
// Unzip
measure("Extract to $cachesFolder") {
Decompressor.Zip(zip).extract(cachesFolder)
}
val tools = JpsCacheTools(60)
//Get current state
val currentState = measure("Get state for folder $cachesFolder") {
tools.getStateByFolder(cachesFolder.toPath())
}
val totalSize = currentState.map { it.size }.sum()
println("${currentState.size} files of size ${Measures.toHumanSize(totalSize)}")
// Save current hashes and upload it
currentHashesFile.write(tools.getJsonByState(currentState))
measure("Upload $currentHashesFile (size ${Measures.toHumanSize(Files.size(currentHashesFile))} to $baseUrl") {
uploadJson(tools, currentHashesFile, baseUrl, commitHash)
}
//Upload caches
val success = measure("Upload all caches") {
uploadAllFiles(currentState, tools, baseUrl, cachesFolder)
}
assert(success) { "Failed to finish uploading within ${tools.secondsToWaitForFuture} seconds. Please, increase it." }
exitProcess(0)
}
private fun uploadAllFiles(currentState: List<JpsCacheFileInfo>,
tools: JpsCacheTools,
baseUrl: String,
cachesFolder: File): Boolean {
val count = AtomicInteger(currentState.size)
val statusReporter: (Int) -> Unit = { completed ->
println("$completed%")
}
println("Uploading files, completed: 0%")
val finished = tools.withExecutor(statusReporter) { executor ->
currentState.forEach { fileInfo ->
executor.execute {
val file = File(cachesFolder, fileInfo.path)
assert(file.exists()) { "can't find $file" }
HttpRequests.put(fileInfo.getUrl(baseUrl), null).write(fileInfo.forUploading(cachesFolder))
count.decrementAndGet()
}
}
}
println("${count.get()} left")
return finished
}
private fun <T> measure(title: String, block: () -> T): T {
println("START: $title")
val before = System.currentTimeMillis()
val result: T = block()
val seconds = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - before)
println("END: $title ($seconds seconds)")
return result
}
private fun uploadJson(tools: JpsCacheTools,
currentState: Path,
baseUrl: String,
commitHash: String) {
val manifestFileZipped = Files.createTempFile("manifest", "json.zip")
ZipUtil.compressFile(currentState, manifestFileZipped)
HttpRequests.put(tools.getZippedManifestUrl(baseUrl, commitHash), null).write(manifestFileZipped.readBytes())
manifestFileZipped.delete()
}
enum class Measures {
B, KB, MB, GB;
private val size = BigDecimal.valueOf(1024L).pow(ordinal)
companion object {
fun toHumanSize(value: Long): String {
val decValue = value.toBigDecimal()
val measure = values().reversed().find { it.size < decValue } ?: B
return "${decValue.divide(measure.size, measure.ordinal, RoundingMode.UP)} $measure"
}
}
}

163
plugins/jps-cache/src/com/intellij/jps/cache/diffExperiment/JpsCacheTools.kt vendored
View File

@@ -1,163 +0,0 @@
package com.intellij.jps.cache.diffExperiment
import com.github.luben.zstd.Zstd
import com.github.luben.zstd.ZstdInputStream
import com.google.common.collect.Maps
import com.google.common.hash.Hashing
import com.google.common.io.CountingInputStream
import com.google.common.io.Files
import com.google.gson.Gson
import com.intellij.openapi.util.io.FileUtil
import com.intellij.util.concurrency.AppExecutorUtil
import java.io.File
import java.io.InputStream
import java.io.Reader
import java.nio.charset.StandardCharsets
import java.nio.file.FileVisitResult
import java.nio.file.Path
import java.nio.file.SimpleFileVisitor
import java.nio.file.attribute.BasicFileAttributes
import java.util.concurrent.ExecutorService
import java.util.concurrent.Future
import java.util.concurrent.TimeUnit
import java.util.concurrent.TimeoutException
import kotlin.math.ceil
import java.nio.file.Files as NioFiles
data class JpsCacheFileInfo(val size: Long, val path: String, @Volatile var hash: Long = 0) {
companion object {
private const val minSizeToZip = 512
private fun archive(uncompressed: ByteArray): ByteArray {
return Zstd.compress(uncompressed)
}
}
private val shouldArchive = size > minSizeToZip
fun forUploading(srcFolder: File): ByteArray =
File(srcFolder, path).readBytes().let { bytes -> if (shouldArchive) archive(bytes) else bytes }
fun afterDownloading(destFolder: File, dataStream: InputStream): Long {
val countingDataStream = CountingInputStream(dataStream)
val dest = File(destFolder, path).apply {
parentFile.mkdirs()
}
(if (shouldArchive) ZstdInputStream(countingDataStream) else countingDataStream).use { stream ->
dest.writeBytes(stream.readBytes())
}
return countingDataStream.count
}
fun getUrl(baseUrl: String) = "$baseUrl/$path/$hash"
}
class JpsCacheTools(val secondsToWaitForFuture: Long) {
companion object {
private const val EXPECTED_NUMBER_OF_FILES = 60_000
private val HASH_FUNCTION = Hashing.murmur3_128()
private const val MANIFESTS_PATH = "manifests"
}
inner class ExecutionApi {
private val futures = mutableListOf<Future<*>>()
fun execute(code: () -> Unit) {
futures.add(executor.submit(code))
}
fun waitForFutures(statusListener: ((completed: Int) -> Unit)? = null): Boolean {
val step: Int = ceil(futures.size.toDouble() / 100).toInt()
var completed = 0
var percent = 0
try {
futures.forEach {
it.get(secondsToWaitForFuture, TimeUnit.SECONDS)
statusListener?.let {
completed++
val newPercent = completed / step
if (newPercent != percent) {
percent = newPercent
it(newPercent)
}
}
}
return true
}
catch (_: TimeoutException) {
return false
}
}
}
private val executor: ExecutorService
init {
val numOfThreads = Runtime.getRuntime().availableProcessors() - 1
executor = AppExecutorUtil.createBoundedApplicationPoolExecutor("JPS_Cache", numOfThreads)
}
fun getJsonByState(files: Collection<JpsCacheFileInfo>): String = Gson().toJson(ArrayList(files))
fun getStateByJsonFile(jsonFilePath: Path): List<JpsCacheFileInfo> = getStateByReader(
Files.newReader(jsonFilePath.toFile(), StandardCharsets.UTF_8))
fun getZippedManifestUrl(baseUrl: String, commitHash: String) = "$baseUrl/$MANIFESTS_PATH/$commitHash"
@JvmOverloads
fun getStateByFolder(rootPath: Path,
existing: Map<String, JpsCacheFileInfo>? = null): List<JpsCacheFileInfo> {
val rootFile = rootPath.toFile()
val result: MutableList<JpsCacheFileInfo> = ArrayList(EXPECTED_NUMBER_OF_FILES)
withExecutor { executor ->
NioFiles.walkFileTree(rootPath, object : SimpleFileVisitor<Path>() {
override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult {
val relativePath = FileUtil.getRelativePath(rootFile, file.toFile())!!.replace(File.separatorChar, '/')
val fileSize = attrs.size()
val fileInfo = JpsCacheFileInfo(fileSize, relativePath)
result.add(fileInfo)
existing?.get(relativePath)?.let { existingInfo ->
if (existingInfo.size != fileSize) {
return FileVisitResult.CONTINUE
}
}
executor.execute { fileInfo.hash = getHash(file) }
return FileVisitResult.CONTINUE
}
})
}
return result
}
fun withExecutor(statusListener: ((completed: Int) -> Unit)? = null, code: (executeApi: ExecutionApi) -> Unit): Boolean {
val api = ExecutionApi()
code(api)
return api.waitForFutures(statusListener)
}
fun getFilesToUpdate(folder: Path,
toState: List<JpsCacheFileInfo>): List<JpsCacheFileInfo> {
val toStateMap = mapFromList(toState)
val currentState = getStateByFolder(folder, toStateMap)
return getFilesToUpdate(currentState, toStateMap)
}
fun getFilesToUpdate(fromState: List<JpsCacheFileInfo>,
toState: List<JpsCacheFileInfo>): List<JpsCacheFileInfo> {
return getFilesToUpdate(fromState, mapFromList(toState))
}
private fun getFilesToUpdate(fromState: List<JpsCacheFileInfo>,
toState: Map<String, JpsCacheFileInfo>): List<JpsCacheFileInfo> =
Maps.difference(mapFromList(fromState), toState).let { difference ->
difference.entriesOnlyOnRight().map { it.value } + difference.entriesDiffering().values.map { it.rightValue() }
}
private fun getHash(file: Path): Long = Files.asByteSource(file.toFile()).hash(HASH_FUNCTION).asLong()
private fun mapFromList(list: List<JpsCacheFileInfo>): Map<String, JpsCacheFileInfo> = list.associateBy { it.path }
private fun getStateByReader(readerWithJson: Reader): List<JpsCacheFileInfo> =
Gson().fromJson(readerWithJson, Array<JpsCacheFileInfo>::class.java).toList()
}

84
plugins/jps-cache/src/com/intellij/jps/cache/git/GitCommitsIterator.java vendored
View File

@@ -1,84 +0,0 @@
package com.intellij.jps.cache.git;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.vcs.VcsException;
import com.intellij.util.SmartList;
import com.intellij.util.containers.ContainerUtil;
import git4idea.history.GitHistoryUtils;
import git4idea.repo.GitRepository;
import org.jetbrains.annotations.NotNull;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
public class GitCommitsIterator implements Iterator<String> {
private static final Logger LOG = Logger.getInstance(GitCommitsIterator.class);
private static final int MAX_FETCH_SIZE = 1000;
private static final int FETCH_SIZE = 100;
private final GitRepository myRepository;
private final Project myProject;
private int fetchedCount;
private List<String> commits;
private String remote;
private int currentPosition;
public GitCommitsIterator(@NotNull Project project, @NotNull GitRepository repository, @NotNull String remoteUrl) {
myRepository = repository;
myProject = project;
fetchedCount = 0;
remote = remoteUrl;
fetchOldCommits();
}
@Override
public boolean hasNext() {
if (commits.size() > 0) {
if (currentPosition < commits.size()) return true;
if (fetchedCount >= MAX_FETCH_SIZE) {
LOG.info("Exceeded fetch limit for git commits");
return false;
}
fetchOldCommits(commits.get(currentPosition - 1));
if (commits.size() > 0) {
currentPosition = 0;
return true;
}
}
return false;
}
@Override
public String next() {
if (commits.size() == 0 || currentPosition >= commits.size()) throw new NoSuchElementException();
String result = commits.get(currentPosition);
currentPosition++;
return result;
}
@NotNull
public String getRemote() {
return remote;
}
private void fetchOldCommits() {
fetchOldCommits("");
}
private void fetchOldCommits(String latestCommit) {
try {
commits =
ContainerUtil.map(latestCommit.isEmpty() ? GitHistoryUtils.collectTimedCommits(myProject, myRepository.getRoot(), "-n " + FETCH_SIZE) :
GitHistoryUtils.collectTimedCommits(myProject, myRepository.getRoot(), latestCommit, "-n " + FETCH_SIZE),
it -> it.getId().asString());
fetchedCount += commits.size();
return;
}
catch (VcsException e) {
LOG.warn("Can't get Git hashes for commits", e);
}
commits = new SmartList<>();
}
}

69
plugins/jps-cache/src/com/intellij/jps/cache/git/GitRepositoryUtil.java vendored
View File

@@ -1,69 +0,0 @@
package com.intellij.jps.cache.git;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.project.Project;
import com.intellij.util.containers.ContainerUtil;
import git4idea.GitUtil;
import git4idea.commands.Git;
import git4idea.commands.GitCommandResult;
import git4idea.config.GitConfigUtil;
import git4idea.repo.GitRemote;
import git4idea.repo.GitRepository;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.util.*;
import java.util.stream.Collectors;
public final class GitRepositoryUtil {
private static final Logger LOG = Logger.getInstance(GitRepositoryUtil.class);
private GitRepositoryUtil() {}
@NotNull
public static List<GitCommitsIterator> getCommitsIterator(@NotNull Project project, @NotNull Set<String> remoteUrlNames) {
if (GitUtil.hasGitRepositories(project)) {
return GitUtil.getRepositories(project).stream()
.map(repo -> {
Set<String> remoteUrls = repo.getRemotes().stream()
.map(remote -> remote.getUrls())
.flatMap(Collection::stream)
.collect(Collectors.toSet());
String matchedRemoteUrl = ContainerUtil.find(remoteUrls, remoteUrl -> remoteUrlNames.contains(getRemoteRepoName(remoteUrl)));
if (matchedRemoteUrl == null) return null;
return new GitCommitsIterator(project, repo, getRemoteRepoName(matchedRemoteUrl));
}).filter(Objects::nonNull)
.collect(Collectors.toList());
}
LOG.info("Project doesn't contain Git repository");
return Collections.emptyList();
}
@Nullable
public static GitRepository getRepositoryByName(@NotNull Project project, @NotNull String repositoryName) {
if (GitUtil.hasGitRepositories(project)) {
return ContainerUtil.find(GitUtil.getRepositories(project), repo -> {
for (GitRemote remote : repo.getRemotes()) {
for (String remoteUrl : remote.getUrls()) {
if (getRemoteRepoName(remoteUrl).equals(repositoryName)) return true;
}
}
return false;
});
}
LOG.info("Project doesn't contain Git repository");
return null;
}
public static String getRemoteRepoName(@NotNull String remoteUrl) {
String[] splittedRemoteUrl = remoteUrl.split("/");
return splittedRemoteUrl[splittedRemoteUrl.length - 1];
}
public static boolean isAutoCrlfSetRight(@NotNull GitRepository gitRepository) {
GitCommandResult result = Git.getInstance().config(gitRepository, GitConfigUtil.CORE_AUTOCRLF);
String value = result.getOutputAsJoinedString();
LOG.info("CRLF configuration for " + gitRepository + " project: " + value);
return value.equalsIgnoreCase("input");
}
}

132
plugins/jps-cache/src/com/intellij/jps/cache/loader/JpsCacheLoader.java vendored
View File

@@ -1,132 +0,0 @@
package com.intellij.jps.cache.loader;
import com.intellij.compiler.server.BuildManager;
import com.intellij.jps.cache.JpsCacheBundle;
import com.intellij.jps.cache.client.JpsServerClient;
import com.intellij.jps.cache.model.JpsLoaderContext;
import com.intellij.jps.cache.ui.SegmentedProgressIndicatorManager;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.progress.ProcessCanceledException;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.util.io.ZipUtil;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.io.File;
import java.io.IOException;
class JpsCacheLoader implements JpsOutputLoader<File> {
private static final Logger LOG = Logger.getInstance(JpsCacheLoader.class);
private static final String TIMESTAMPS_FOLDER_NAME = "timestamps";
private static final String FS_STATE_FILE = "fs_state.dat";
private final BuildManager myBuildManager;
private final JpsServerClient myClient;
private final Project myProject;
private File myTmpCacheFolder;
JpsCacheLoader(JpsServerClient client, @NotNull Project project) {
myBuildManager = BuildManager.getInstance();
myClient = client;
myProject = project;
}
@Nullable
@Override
public File load(@NotNull JpsLoaderContext context) {
LOG.info("Loading JPS caches for commit: " + context.getCommitId());
myTmpCacheFolder = null;
long start = System.currentTimeMillis();
File zipFile = myClient.downloadCacheById(context.getDownloadIndicatorManager(), context.getCommitId(),
myBuildManager.getBuildSystemDirectory(myProject).toFile());
LOG.info("Download of jps caches took: " + (System.currentTimeMillis() - start));
return zipFile;
}
@Override
public LoaderStatus extract(@Nullable Object loadResults, @NotNull SegmentedProgressIndicatorManager extractIndicatorManager) {
if (!(loadResults instanceof File)) return LoaderStatus.FAILED;
File zipFile = (File)loadResults;
File targetDir = myBuildManager.getBuildSystemDirectory(myProject).toFile();
File tmpFolder = new File(targetDir, "tmp");
try {
// Start extracting after download
SegmentedProgressIndicatorManager.SubTaskProgressIndicator subTaskIndicator = extractIndicatorManager.createSubTaskIndicator();
extractIndicatorManager.getProgressIndicator().checkCanceled();
extractIndicatorManager.setText(this, JpsCacheBundle.message("progress.text.extracting.downloaded.results"));
subTaskIndicator.setText2(JpsCacheBundle.message("progress.details.extracting.project.caches"));
long start = System.currentTimeMillis();
ZipUtil.extract(zipFile, tmpFolder, null);
FileUtil.delete(zipFile);
LOG.info("Unzip compilation caches took: " + (System.currentTimeMillis() - start));
subTaskIndicator.finished();
extractIndicatorManager.finished(this);
myTmpCacheFolder = tmpFolder;
return LoaderStatus.COMPLETE;
}
catch (ProcessCanceledException | IOException e) {
if (e instanceof IOException) LOG.warn("Failed unzip downloaded compilation caches", e);
FileUtil.delete(zipFile);
FileUtil.delete(tmpFolder);
}
return LoaderStatus.FAILED;
}
@Override
public void rollback() {
if (myTmpCacheFolder != null && myTmpCacheFolder.exists()) {
FileUtil.delete(myTmpCacheFolder);
LOG.debug("JPS cache loader rolled back");
}
}
@Override
public void apply(@NotNull SegmentedProgressIndicatorManager indicatorManager) {
if (myTmpCacheFolder == null) {
LOG.warn("Nothing to apply, download results are empty");
return;
}
File newTimestampFolder = new File(myTmpCacheFolder, TIMESTAMPS_FOLDER_NAME);
if (newTimestampFolder.exists()) FileUtil.delete(newTimestampFolder);
File currentDirForBuildCache = myBuildManager.getProjectSystemDirectory(myProject);
indicatorManager.setText(this, JpsCacheBundle.message("progress.text.applying.jps.caches"));
if (currentDirForBuildCache != null) {
SegmentedProgressIndicatorManager.SubTaskProgressIndicator subTaskIndicator = indicatorManager.createSubTaskIndicator();
subTaskIndicator.setText2(JpsCacheBundle.message("progress.details.applying.downloaded.caches"));
// Copy timestamp old folder to new cache dir
File timestamps = new File(currentDirForBuildCache, TIMESTAMPS_FOLDER_NAME);
if (timestamps.exists()) {
try {
newTimestampFolder.mkdirs();
FileUtil.copyDir(timestamps, newTimestampFolder);
}
catch (IOException e) {
LOG.warn("Couldn't copy timestamps from old JPS cache", e);
}
}
// Create new empty fsStateFile
File fsStateFile = new File(myTmpCacheFolder, FS_STATE_FILE);
fsStateFile.delete();
try {
fsStateFile.createNewFile();
}
catch (IOException e) {
LOG.warn("Couldn't create new empty FsState file", e);
}
// Remove old cache dir
FileUtil.delete(currentDirForBuildCache);
myTmpCacheFolder.renameTo(currentDirForBuildCache);
subTaskIndicator.finished();
LOG.debug("JPS cache downloads finished");
}
indicatorManager.finished(this);
}
}

357
plugins/jps-cache/src/com/intellij/jps/cache/loader/JpsCompilationOutputLoader.java vendored
View File

@@ -1,357 +0,0 @@
package com.intellij.jps.cache.loader;
import com.google.common.hash.Hasher;
import com.google.common.hash.Hashing;
import com.intellij.jps.cache.JpsCacheBundle;
import com.intellij.jps.cache.client.JpsServerClient;
import com.intellij.jps.cache.model.AffectedModule;
import com.intellij.jps.cache.model.BuildTargetState;
import com.intellij.jps.cache.model.JpsLoaderContext;
import com.intellij.jps.cache.model.OutputLoadResult;
import com.intellij.jps.cache.ui.SegmentedProgressIndicatorManager;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.progress.ProcessCanceledException;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.util.ArrayUtil;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.io.ZipUtil;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.annotations.TestOnly;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
import static com.intellij.jps.cache.JpsCachesPluginUtil.EXECUTOR_SERVICE;
class JpsCompilationOutputLoader implements JpsOutputLoader<List<OutputLoadResult>> {
private static final Logger LOG = Logger.getInstance(JpsCompilationOutputLoader.class);
private static final String RESOURCES_PRODUCTION = "resources-production";
private static final String JAVA_PRODUCTION = "java-production";
private static final String RESOURCES_TEST = "resources-test";
private static final String JAVA_TEST = "java-test";
private static final String PRODUCTION = "production";
private static final String TEST = "test";
private final JpsServerClient myClient;
private final String myBuildDirPath;
private List<File> myOldModulesPaths;
private Map<File, String> myTmpFolderToModuleName;
JpsCompilationOutputLoader(@NotNull JpsServerClient client, @NotNull String buildDirPath) {
myClient = client;
myBuildDirPath = buildDirPath;
}
@Override
public int calculateDownloads(@NotNull Map<String, Map<String, BuildTargetState>> commitSourcesState,
@Nullable Map<String, Map<String, BuildTargetState>> currentSourcesState) {
return calculateAffectedModules(currentSourcesState, commitSourcesState, true).size();
}
@Override
public List<OutputLoadResult> load(@NotNull JpsLoaderContext context) {
myOldModulesPaths = null;
myTmpFolderToModuleName = null;
SegmentedProgressIndicatorManager downloadProgressManager = context.getDownloadIndicatorManager();
downloadProgressManager.setText(this, JpsCacheBundle.message("progress.text.calculating.affected.modules"));
List<AffectedModule> affectedModules = calculateAffectedModules(context.getCurrentSourcesState(),
context.getCommitSourcesState(), true);
downloadProgressManager.finished(this);
downloadProgressManager.getProgressIndicator().checkCanceled();
if (affectedModules.size() > 0) {
long start = System.currentTimeMillis();
List<OutputLoadResult> loadResults = myClient.downloadCompiledModules(downloadProgressManager, affectedModules);
LOG.info("Download of compilation outputs took: " + (System.currentTimeMillis() - start));
return loadResults;
}
return Collections.emptyList();
}
@Override
public LoaderStatus extract(@Nullable Object loadResults, @NotNull SegmentedProgressIndicatorManager extractIndicatorManager) {
if (!(loadResults instanceof List)) return LoaderStatus.FAILED;
//noinspection unchecked
List<OutputLoadResult> outputLoadResults = (List<OutputLoadResult>)loadResults;
Map<File, String> result = new ConcurrentHashMap<>();
try {
// Extracting results
long start = System.currentTimeMillis();
extractIndicatorManager.setText(this, JpsCacheBundle.message("progress.text.extracting.downloaded.results"));
List<Future<?>> futureList = ContainerUtil.map(outputLoadResults, loadResult ->
EXECUTOR_SERVICE.submit(new UnzipOutputTask(result, loadResult, extractIndicatorManager)));
for (Future<?> future : futureList) {
future.get();
}
extractIndicatorManager.finished(this);
myTmpFolderToModuleName = result;
LOG.info("Unzip compilation output took: " + (System.currentTimeMillis() - start));
return LoaderStatus.COMPLETE;
}
catch (ProcessCanceledException | InterruptedException | ExecutionException e) {
if (!(e.getCause() instanceof ProcessCanceledException)) LOG.warn("Failed unzip downloaded compilation outputs", e);
outputLoadResults.forEach(loadResult -> FileUtil.delete(loadResult.getZipFile()));
result.forEach((key, value) -> FileUtil.delete(key));
}
return LoaderStatus.FAILED;
}
@Override
public void rollback() {
if (myTmpFolderToModuleName == null) return;
myTmpFolderToModuleName.forEach((tmpFolder, __) -> {
if (tmpFolder.isDirectory() && tmpFolder.exists()) FileUtil.delete(tmpFolder);
});
LOG.info("JPS cache loader rolled back");
}
@Override
public void apply(@NotNull SegmentedProgressIndicatorManager indicatorManager) {
long start = System.currentTimeMillis();
if (myOldModulesPaths != null) {
LOG.info("Removing old compilation outputs " + myOldModulesPaths.size() + " counts");
myOldModulesPaths.forEach(file -> {
if (file.exists()) FileUtil.delete(file);
});
}
if (myTmpFolderToModuleName == null) {
LOG.debug("Nothing to apply, download results are empty");
return;
}
indicatorManager.setText(this, JpsCacheBundle.message("progress.text.applying.jps.caches"));
ContainerUtil.map(myTmpFolderToModuleName.entrySet(),
entry -> EXECUTOR_SERVICE.submit(() -> {
String moduleName = entry.getValue();
File tmpModuleFolder = entry.getKey();
SegmentedProgressIndicatorManager.SubTaskProgressIndicator subTaskIndicator =
indicatorManager.createSubTaskIndicator();
subTaskIndicator.setText2(JpsCacheBundle.message("progress.details.applying.changes.for.module", moduleName));
File currentModuleBuildDir = new File(tmpModuleFolder.getParentFile(), moduleName);
FileUtil.delete(currentModuleBuildDir);
try {
FileUtil.rename(tmpModuleFolder, currentModuleBuildDir);
LOG.debug("Module: " + moduleName + " was replaced successfully");
}
catch (IOException e) {
LOG.warn("Couldn't replace compilation output for module: " + moduleName, e);
}
subTaskIndicator.finished();
}))
.forEach(future -> {
try {
future.get();
}
catch (InterruptedException | ExecutionException e) {
LOG.info("Couldn't apply compilation output", e);
}
});
indicatorManager.finished(this);
LOG.info("Applying compilation output took: " + (System.currentTimeMillis() - start));
}
@NotNull
private List<AffectedModule> calculateAffectedModules(@Nullable Map<String, Map<String, BuildTargetState>> currentModulesState,
@NotNull Map<String, Map<String, BuildTargetState>> commitModulesState,
boolean checkExistance) {
long start = System.currentTimeMillis();
List<AffectedModule> affectedModules = new ArrayList<>();
Map<String, String> oldModulesMap = new HashMap<>();
myOldModulesPaths = new ArrayList<>();
if (currentModulesState == null) {
commitModulesState.forEach((type, map) -> {
map.forEach((name, state) -> {
affectedModules.add(new AffectedModule(type, name, state.getHash(), getBuildDirRelativeFile(state.getRelativePath())));
});
});
LOG.warn("Project doesn't contain metadata, force to download " + affectedModules.size() + " modules.");
List<AffectedModule> result = mergeAffectedModules(affectedModules, commitModulesState);
long total = System.currentTimeMillis() - start;
LOG.info("Compilation output affected for the " + result.size() + " modules. Computation took " + total + "ms");
return result;
}
// Add new build types
Set<String> newBuildTypes = new HashSet<>(commitModulesState.keySet());
newBuildTypes.removeAll(currentModulesState.keySet());
newBuildTypes.forEach(type -> {
commitModulesState.get(type).forEach((name, state) -> {
affectedModules.add(new AffectedModule(type, name, state.getHash(), getBuildDirRelativeFile(state.getRelativePath())));
});
});
// Calculate old paths for remove
Set<String> oldBuildTypes = new HashSet<>(currentModulesState.keySet());
oldBuildTypes.removeAll(commitModulesState.keySet());
oldBuildTypes.forEach(type -> {
currentModulesState.get(type).forEach((name, state) -> {
oldModulesMap.put(name, state.getRelativePath());
});
});
commitModulesState.forEach((type, map) -> {
Map<String, BuildTargetState> currentTypeState = currentModulesState.get(type);
// New build type already added above
if (currentTypeState == null) return;
// Add new build modules
Set<String> newBuildModules = new HashSet<>(map.keySet());
newBuildModules.removeAll(currentTypeState.keySet());
newBuildModules.forEach(name -> {
BuildTargetState state = map.get(name);
affectedModules.add(new AffectedModule(type, name, state.getHash(), getBuildDirRelativeFile(state.getRelativePath())));
});
// Calculate old modules paths for remove
Set<String> oldBuildModules = new HashSet<>(currentTypeState.keySet());
oldBuildModules.removeAll(map.keySet());
oldBuildModules.forEach(name -> {
BuildTargetState state = currentTypeState.get(name);
oldModulesMap.put(name, state.getRelativePath());
});
// In another case compare modules inside the same build type
map.forEach((name, state) -> {
BuildTargetState currentTargetState = currentTypeState.get(name);
if (currentTargetState == null || !state.equals(currentTargetState)) {
affectedModules.add(new AffectedModule(type, name, state.getHash(), getBuildDirRelativeFile(state.getRelativePath())));
return;
}
File outFile = getBuildDirRelativeFile(state.getRelativePath());
if (checkExistance && (!outFile.exists() || ArrayUtil.isEmpty(outFile.listFiles()))) {
affectedModules.add(new AffectedModule(type, name, state.getHash(), outFile));
}
});
});
// Check that old modules not exist in other build types
myOldModulesPaths = oldModulesMap.entrySet().stream().filter(entry -> {
for (Map.Entry<String, Map<String, BuildTargetState>> commitEntry : commitModulesState.entrySet()) {
BuildTargetState targetState = commitEntry.getValue().get(entry.getKey());
if (targetState != null && targetState.getRelativePath().equals(entry.getValue())) return false;
}
return true;
}).map(entry -> getBuildDirRelativeFile(entry.getValue()))
.collect(Collectors.toList());
List<AffectedModule> result = mergeAffectedModules(affectedModules, commitModulesState);
long total = System.currentTimeMillis() - start;
LOG.info("Compilation output affected for the " + result.size() + " modules. Computation took " + total + "ms");
return result;
}
@NotNull
private static List<AffectedModule> mergeAffectedModules(List<AffectedModule> affectedModules,
@NotNull Map<String, Map<String, BuildTargetState>> commitModulesState) {
Set<AffectedModule> result = new HashSet<>();
affectedModules.forEach(affectedModule -> {
if (affectedModule.getType().equals(JAVA_PRODUCTION)) {
BuildTargetState targetState = commitModulesState.get(RESOURCES_PRODUCTION).get(affectedModule.getName());
if (targetState == null) {
result.add(affectedModule);
return;
}
String hash = calculateStringHash(affectedModule.getHash() + targetState.getHash());
result.add(new AffectedModule(PRODUCTION, affectedModule.getName(), hash, affectedModule.getOutPath()));
}
else if (affectedModule.getType().equals(RESOURCES_PRODUCTION)) {
BuildTargetState targetState = commitModulesState.get(JAVA_PRODUCTION).get(affectedModule.getName());
if (targetState == null) {
result.add(affectedModule);
return;
}
String hash = calculateStringHash(targetState.getHash() + affectedModule.getHash());
result.add(new AffectedModule(PRODUCTION, affectedModule.getName(), hash, affectedModule.getOutPath()));
}
else if (affectedModule.getType().equals(JAVA_TEST)) {
BuildTargetState targetState = commitModulesState.get(RESOURCES_TEST).get(affectedModule.getName());
if (targetState == null) {
result.add(affectedModule);
return;
}
String hash = calculateStringHash(affectedModule.getHash() + targetState.getHash());
result.add(new AffectedModule(TEST, affectedModule.getName(), hash, affectedModule.getOutPath()));
}
else if (affectedModule.getType().equals(RESOURCES_TEST)) {
BuildTargetState targetState = commitModulesState.get(JAVA_TEST).get(affectedModule.getName());
if (targetState == null) {
result.add(affectedModule);
return;
}
String hash = calculateStringHash(targetState.getHash() + affectedModule.getHash());
result.add(new AffectedModule(TEST, affectedModule.getName(), hash, affectedModule.getOutPath()));
}
else {
result.add(affectedModule);
}
});
return new ArrayList<>(result);
}
private File getBuildDirRelativeFile(String buildDirRelativePath) {
return new File(buildDirRelativePath.replace("$BUILD_DIR$", myBuildDirPath));
}
private static String calculateStringHash(String content) {
Hasher hasher = Hashing.murmur3_128().newHasher();
return hasher.putString(content, StandardCharsets.UTF_8).hash().toString();
}
@TestOnly
List<File> getOldModulesPaths() {
return myOldModulesPaths;
}
@TestOnly
List<AffectedModule> getAffectedModules(@Nullable Map<String, Map<String, BuildTargetState>> currentModulesState,
@NotNull Map<String, Map<String, BuildTargetState>> commitModulesState, boolean checkExistence) {
return calculateAffectedModules(currentModulesState, commitModulesState, checkExistence);
}
private static final class UnzipOutputTask implements Runnable {
private final OutputLoadResult loadResult;
private final Map<File, String> result;
private final SegmentedProgressIndicatorManager extractIndicatorManager;
private UnzipOutputTask(Map<File, String> result,
OutputLoadResult loadResult,
SegmentedProgressIndicatorManager extractIndicatorManager) {
this.result = result;
this.loadResult = loadResult;
this.extractIndicatorManager = extractIndicatorManager;
}
@Override
public void run() {
AffectedModule affectedModule = loadResult.getModule();
File outPath = affectedModule.getOutPath();
try {
SegmentedProgressIndicatorManager.SubTaskProgressIndicator subTaskIndicator = extractIndicatorManager.createSubTaskIndicator();
extractIndicatorManager.getProgressIndicator().checkCanceled();
subTaskIndicator.setText2(
JpsCacheBundle.message("progress.details.extracting.compilation.outputs.for.module", affectedModule.getName()));
LOG.debug("Downloaded JPS compiled module from: " + loadResult.getDownloadUrl());
File tmpFolder = new File(outPath.getParent(), outPath.getName() + "_tmp");
File zipFile = loadResult.getZipFile();
ZipUtil.extract(zipFile, tmpFolder, null);
FileUtil.delete(zipFile);
result.put(tmpFolder, affectedModule.getName());
subTaskIndicator.finished();
}
catch (IOException e) {
LOG.warn("Couldn't extract download result for module: " + affectedModule.getName(), e);
}
}
}
}

78
plugins/jps-cache/src/com/intellij/jps/cache/loader/JpsMetadataLoader.java vendored
View File

@@ -1,78 +0,0 @@
package com.intellij.jps.cache.loader;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import com.intellij.compiler.server.BuildManager;
import com.intellij.jps.cache.client.JpsServerClient;
import com.intellij.jps.cache.model.BuildTargetState;
import com.intellij.openapi.application.PathManager;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.util.io.FileUtil;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.lang.reflect.Type;
import java.util.Map;
public class JpsMetadataLoader {
private static final Logger LOG = Logger.getInstance(JpsMetadataLoader.class);
private static final String SOURCES_STATE_FILE_NAME = "target_sources_state.json";
private final BuildManager myBuildManager;
private final JpsServerClient myClient;
private final Project myProject;
private final Type myTokenType;
private final Gson myGson;
public JpsMetadataLoader(@NotNull Project project, @NotNull JpsServerClient client) {
myClient = client;
myGson = new Gson();
myProject = project;
myBuildManager = BuildManager.getInstance();
myTokenType = new TypeToken<Map<String, Map<String, BuildTargetState>>>() {}.getType();
}
@Nullable
Map<String, Map<String, BuildTargetState>> loadMetadataForCommit(@NotNull String metadataId) {
// On server commitId is the same as metadataId this data simply located in different folders
File tmpFolder = new File(PathManager.getTempPath());
File metadataFile = myClient.downloadMetadataById(metadataId, tmpFolder);
if (metadataFile == null) return null;
try (BufferedReader bufferedReader = new BufferedReader(new FileReader(metadataFile))) {
return myGson.fromJson(bufferedReader, myTokenType);
}
catch (IOException e) {
LOG.warn("Couldn't parse content of file " + metadataFile.getName(), e);
}
finally {
if (metadataFile.exists()) FileUtil.delete(metadataFile);
}
return null;
}
@Nullable
Map<String, Map<String, BuildTargetState>> loadCurrentProjectMetadata() {
File currentBuildCacheFolder = myBuildManager.getProjectSystemDirectory(myProject);
File currentSourceState = new File(currentBuildCacheFolder, SOURCES_STATE_FILE_NAME);
if (!currentSourceState.exists()) return null;
try (BufferedReader bufferedReader = new BufferedReader(new FileReader(currentSourceState))) {
return myGson.fromJson(bufferedReader, myTokenType);
}
catch (IOException e) {
LOG.warn("Couldn't parse current project metadata " + currentSourceState.getName(), e);
}
return null;
}
void dropCurrentProjectMetadata() {
File currentBuildCacheFolder = myBuildManager.getProjectSystemDirectory(myProject);
File currentSourceState = new File(currentBuildCacheFolder, SOURCES_STATE_FILE_NAME);
if (currentSourceState.exists()) FileUtil.delete(currentSourceState);
}
}

24
plugins/jps-cache/src/com/intellij/jps/cache/loader/JpsOutputLoader.java vendored
View File

@@ -1,24 +0,0 @@
package com.intellij.jps.cache.loader;
import com.intellij.jps.cache.model.BuildTargetState;
import com.intellij.jps.cache.model.JpsLoaderContext;
import com.intellij.jps.cache.ui.SegmentedProgressIndicatorManager;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.util.Map;
interface JpsOutputLoader<T> {
T load(@NotNull JpsLoaderContext context);
LoaderStatus extract(@Nullable Object loadResults, @NotNull SegmentedProgressIndicatorManager extractIndicatorManager);
void rollback();
void apply(@NotNull SegmentedProgressIndicatorManager indicatorManager);
default int calculateDownloads(@NotNull Map<String, Map<String, BuildTargetState>> commitSourcesState,
@Nullable Map<String, Map<String, BuildTargetState>> currentSourcesState) {
return 1;
}
enum LoaderStatus {
COMPLETE, FAILED
}
}

375
plugins/jps-cache/src/com/intellij/jps/cache/loader/JpsOutputLoaderManager.java vendored
View File

@@ -1,375 +0,0 @@
package com.intellij.jps.cache.loader;
import com.intellij.compiler.CompilerWorkspaceConfiguration;
import com.intellij.compiler.server.BuildManager;
import com.intellij.compiler.server.PortableCachesLoadListener;
import com.intellij.ide.util.PropertiesComponent;
import com.intellij.jps.cache.JpsCacheBundle;
import com.intellij.jps.cache.JpsCacheStartupActivity;
import com.intellij.jps.cache.client.JpsServerClient;
import com.intellij.jps.cache.git.GitCommitsIterator;
import com.intellij.jps.cache.git.GitRepositoryUtil;
import com.intellij.jps.cache.loader.JpsOutputLoader.LoaderStatus;
import com.intellij.jps.cache.model.BuildTargetState;
import com.intellij.jps.cache.model.JpsLoaderContext;
import com.intellij.jps.cache.ui.SegmentedProgressIndicatorManager;
import com.intellij.notification.NotificationAction;
import com.intellij.notification.NotificationType;
import com.intellij.openapi.Disposable;
import com.intellij.openapi.application.ApplicationManager;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.progress.ProcessCanceledException;
import com.intellij.openapi.progress.ProgressIndicator;
import com.intellij.openapi.progress.ProgressManager;
import com.intellij.openapi.progress.Task;
import com.intellij.openapi.progress.impl.BackgroundableProcessIndicator;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.util.Pair;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.util.containers.ContainerUtil;
import org.jdom.Element;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.jps.model.serialization.JDomSerializationUtil;
import org.jetbrains.jps.model.serialization.JpsLoaderBase;
import org.jetbrains.jps.model.serialization.PathMacroUtil;
import org.jetbrains.jps.util.JpsPathUtil;
import java.io.File;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicBoolean;
import static com.intellij.execution.process.ProcessIOExecutorService.INSTANCE;
import static com.intellij.jps.cache.statistics.JpsCacheUsagesCollector.*;
import static com.intellij.jps.cache.ui.JpsLoaderNotifications.*;
import static org.jetbrains.jps.model.serialization.java.JpsJavaModelSerializerExtension.OUTPUT_TAG;
import static org.jetbrains.jps.model.serialization.java.JpsJavaModelSerializerExtension.URL_ATTRIBUTE;
public class JpsOutputLoaderManager implements Disposable {
private static final Logger LOG = Logger.getInstance(JpsOutputLoaderManager.class);
private static final String LATEST_COMMIT_ID = "JpsOutputLoaderManager.latestCommitId";
private static final double SEGMENT_SIZE = 0.33;
private final AtomicBoolean hasRunningTask;
private final CompilerWorkspaceConfiguration myWorkspaceConfiguration;
private List<JpsOutputLoader<?>> myJpsOutputLoadersLoaders;
private final JpsMetadataLoader myMetadataLoader;
private final JpsServerClient myServerClient;
private final String myBuildOutDir;
private final Project myProject;
@Override
public void dispose() { }
@NotNull
public static JpsOutputLoaderManager getInstance(@NotNull Project project) {
return project.getService(JpsOutputLoaderManager.class);
}
public JpsOutputLoaderManager(@NotNull Project project) {
myProject = project;
hasRunningTask = new AtomicBoolean();
myBuildOutDir = getBuildOutDir(myProject);
myServerClient = JpsServerClient.getServerClient();
myMetadataLoader = new JpsMetadataLoader(project, myServerClient);
myWorkspaceConfiguration = CompilerWorkspaceConfiguration.getInstance(myProject);
// Configure build manager
BuildManager buildManager = BuildManager.getInstance();
//if (!buildManager.isGeneratePortableCachesEnabled()) buildManager.setGeneratePortableCachesEnabled(true);
}
public void load(boolean isForceUpdate, boolean verbose) {
Task.Backgroundable task = new Task.Backgroundable(myProject, JpsCacheBundle.message("progress.title.updating.compiler.caches")) {
@Override
public void run(@NotNull ProgressIndicator indicator) {
Pair<String, Integer> commitInfo = getNearestCommit(isForceUpdate, verbose);
if (commitInfo != null) {
assert myProject != null;
myProject.getMessageBus().syncPublisher(PortableCachesLoadListener.TOPIC).loadingStarted();
// Drop JPS metadata to force plugin for downloading all compilation outputs
if (isForceUpdate) {
myMetadataLoader.dropCurrentProjectMetadata();
File outDir = new File(myBuildOutDir);
if (outDir.exists()) {
indicator.setText(JpsCacheBundle.message("progress.text.clean.output.directories"));
FileUtil.delete(outDir);
}
LOG.info("Compilation output folder empty");
}
startLoadingForCommit(commitInfo.first);
}
hasRunningTask.set(false);
}
};
if (!canRunNewLoading()) return;
BackgroundableProcessIndicator processIndicator = new BackgroundableProcessIndicator(task);
processIndicator.setIndeterminate(false);
ProgressManager.getInstance().runProcessWithProgressAsynchronously(task, processIndicator);
}
public void notifyAboutNearestCache() {
Pair<String, Integer> commitInfo = getNearestCommit(false, false);
if (commitInfo == null) return;
String notificationContent = commitInfo.second == 1
? JpsCacheBundle.message("notification.content.caches.are.for.current.commit")
: JpsCacheBundle
.message("notification.content.caches.are.for.commit.commits.prior.to.yours", commitInfo.second - 1);
ApplicationManager.getApplication().invokeLater(() -> {
STANDARD
.createNotification(JpsCacheBundle.message("notification.title.compiler.caches.available"), notificationContent,
NotificationType.INFORMATION)
.addAction(NotificationAction.createSimpleExpiring(
JpsCacheBundle.message("action.NotificationAction.JpsOutputLoaderManager.text.update.caches"),
() -> {
DOWNLOAD_THROUGH_NOTIFICATION_EVENT_ID.log();
load(false, false);
}))
.notify(myProject);
});
}
@Nullable
private Pair<String, Integer> getNearestCommit(boolean isForceUpdate, boolean verbose) {
Map<String, Set<String>> availableCommitsPerRemote = myServerClient.getCacheKeysPerRemote(myProject);
String previousCommitId = PropertiesComponent.getInstance().getValue(LATEST_COMMIT_ID);
List<GitCommitsIterator> repositoryList = GitRepositoryUtil.getCommitsIterator(myProject, availableCommitsPerRemote.keySet());
String commitId = "";
int commitsBehind = 0;
Set<String> availableCommitsForRemote = new HashSet<>();
for (GitCommitsIterator commitsIterator : repositoryList) {
availableCommitsForRemote = availableCommitsPerRemote.get(commitsIterator.getRemote());
if (availableCommitsForRemote.contains(commitId)) continue;
commitsBehind = 0;
while (commitsIterator.hasNext() && !availableCommitsForRemote.contains(commitId)) {
commitId = commitsIterator.next();
commitsBehind++;
}
}
var group = verbose ? STANDARD : EVENT_LOG;
if (!availableCommitsForRemote.contains(commitId)) {
String warning = JpsCacheBundle.message("notification.content.not.found.any.caches.for.latest.commits.in.branch");
LOG.warn(warning);
ApplicationManager.getApplication().invokeLater(() -> {
group.createNotification(JpsCacheBundle.message("notification.title.jps.caches.downloader"), warning, NotificationType.WARNING)
.notify(myProject);
});
return null;
}
if (previousCommitId != null && commitId.equals(previousCommitId) && !isForceUpdate) {
String info = JpsCacheBundle.message("notification.content.system.contains.up.to.date.caches");
LOG.info(info);
ApplicationManager.getApplication().invokeLater(() -> {
group.createNotification(JpsCacheBundle.message("notification.title.jps.caches.downloader"), info, NotificationType.INFORMATION)
.notify(myProject);
});
return null;
}
return Pair.create(commitId, commitsBehind);
}
private void startLoadingForCommit(@NotNull String commitId) {
long startTime = System.nanoTime();
ProgressIndicator indicator = ProgressManager.getInstance().getProgressIndicator();
indicator.setText(JpsCacheBundle.message("progress.text.fetching.cache.for.commit", commitId));
// Loading metadata for commit
Map<String, Map<String, BuildTargetState>> commitSourcesState = myMetadataLoader.loadMetadataForCommit(commitId);
if (commitSourcesState == null) {
LOG.warn("Couldn't load metadata for commit: " + commitId);
myProject.getMessageBus().syncPublisher(PortableCachesLoadListener.TOPIC).loadingFinished(false);
return;
}
// Calculate downloads
Map<String, Map<String, BuildTargetState>> currentSourcesState = myMetadataLoader.loadCurrentProjectMetadata();
int totalDownloads =
getLoaders(myProject).stream().mapToInt(loader -> loader.calculateDownloads(commitSourcesState, currentSourcesState)).sum();
indicator.setFraction(0.01);
try {
// Computation with loaders results. If at least one of them failed rollback all job
initLoaders(commitId, indicator, totalDownloads, commitSourcesState, currentSourcesState).thenAccept(loaderStatus -> {
LOG.info("Loading finished with " + loaderStatus + " status");
try {
SegmentedProgressIndicatorManager indicatorManager =
new SegmentedProgressIndicatorManager(indicator, totalDownloads, SEGMENT_SIZE);
CompletableFuture.allOf(getLoaders(myProject).stream()
.map(loader -> applyChanges(loaderStatus, loader, indicator, indicatorManager))
.toArray(CompletableFuture[]::new))
.thenRun(() -> saveStateAndNotify(loaderStatus, commitId, startTime))
.get();
}
catch (InterruptedException | ExecutionException e) {
LOG.warn("Unexpected exception rollback all progress", e);
onFail();
getLoaders(myProject).forEach(loader -> loader.rollback());
myProject.getMessageBus().syncPublisher(PortableCachesLoadListener.TOPIC).loadingFinished(false);
indicator.setText(JpsCacheBundle.message("progress.text.rolling.back.downloaded.caches"));
}
}).handle((result, ex) -> handleExceptions(result, ex, indicator)).get();
}
catch (InterruptedException | ExecutionException e) {
LOG.warn("Couldn't fetch jps compilation caches", e);
onFail();
myProject.getMessageBus().syncPublisher(PortableCachesLoadListener.TOPIC).loadingFinished(false);
}
}
@Nullable
public static String getBuildOutDir(@NotNull Project project) {
VirtualFile projectFile = project.getProjectFile();
String projectBasePath = project.getBasePath();
if (projectFile == null || projectBasePath == null) {
LOG.warn("Project files doesn't exist");
return null;
}
String fileExtension = projectFile.getExtension();
if (fileExtension != null && fileExtension.equals("irp")) {
LOG.warn("File base project not supported");
return null;
}
Path configFile = Paths.get(FileUtil.toCanonicalPath(projectFile.getPath()));
Element componentTag = JDomSerializationUtil.findComponent(JpsLoaderBase.tryLoadRootElement(configFile), "ProjectRootManager");
if (componentTag == null) {
LOG.warn("Component tag in config file doesn't exist");
return null;
}
Element output = componentTag.getChild(OUTPUT_TAG);
if (output == null) {
LOG.warn("Output tag in config file doesn't exist");
return null;
}
String url = output.getAttributeValue(URL_ATTRIBUTE);
if (url == null) {
LOG.warn("URL attribute in output tag doesn't exist");
return null;
}
return JpsPathUtil.urlToPath(url).replace("$" + PathMacroUtil.PROJECT_DIR_MACRO_NAME + "$", projectBasePath);
}
private synchronized boolean canRunNewLoading() {
if (hasRunningTask.get()) {
LOG.warn("Jps cache loading already in progress, can't start the new one");
return false;
}
if (myBuildOutDir == null) {
LOG.warn("Build output dir is not configured for the project");
return false;
}
if (myWorkspaceConfiguration.MAKE_PROJECT_ON_SAVE) {
LOG.warn("Project automatic build should be disabled, it can affect portable caches");
return false;
}
if (!JpsCacheStartupActivity.isLineEndingsConfiguredCorrectly()) {
LOG.warn("Git line-endings not configured correctly for the project");
return false;
}
hasRunningTask.set(true);
return true;
}
private <T> CompletableFuture<LoaderStatus> initLoaders(String commitId, ProgressIndicator indicator, int totalDownloads,
Map<String, Map<String, BuildTargetState>> commitSourcesState,
Map<String, Map<String, BuildTargetState>> currentSourcesState) {
List<JpsOutputLoader<?>> loaders = getLoaders(myProject);
// Create indicator with predefined segment size
SegmentedProgressIndicatorManager downloadIndicatorManager =
new SegmentedProgressIndicatorManager(indicator, totalDownloads, SEGMENT_SIZE);
SegmentedProgressIndicatorManager extractIndicatorManager =
new SegmentedProgressIndicatorManager(indicator, totalDownloads, SEGMENT_SIZE);
JpsLoaderContext loaderContext =
JpsLoaderContext.createNewContext(commitId, downloadIndicatorManager, commitSourcesState, currentSourcesState);
// Start loaders with own context
List<CompletableFuture<LoaderStatus>> completableFutures = ContainerUtil.map(loaders, loader ->
CompletableFuture.supplyAsync(() -> loader.extract(loader.load(loaderContext), extractIndicatorManager), INSTANCE));
// Reduce loaders statuses into the one
CompletableFuture<LoaderStatus> initialFuture = completableFutures.get(0);
if (completableFutures.size() > 1) {
for (int i = 1; i < completableFutures.size(); i++) {
initialFuture = initialFuture.thenCombine(completableFutures.get(i), JpsOutputLoaderManager::combine);
}
}
return initialFuture;
}
private static CompletableFuture<Void> applyChanges(LoaderStatus loaderStatus, JpsOutputLoader<?> loader, ProgressIndicator indicator,
SegmentedProgressIndicatorManager indicatorManager) {
if (loaderStatus == LoaderStatus.FAILED) {
indicator.setText(JpsCacheBundle.message("progress.text.rolling.back"));
return CompletableFuture.runAsync(() -> loader.rollback(), INSTANCE);
}
return CompletableFuture.runAsync(() -> loader.apply(indicatorManager), INSTANCE);
}
private void saveStateAndNotify(LoaderStatus loaderStatus, String commitId, long startTime) {
if (loaderStatus == LoaderStatus.FAILED) {
onFail();
myProject.getMessageBus().syncPublisher(PortableCachesLoadListener.TOPIC).loadingFinished(false);
return;
}
PropertiesComponent.getInstance().setValue(LATEST_COMMIT_ID, commitId);
BuildManager.getInstance().clearState(myProject);
long endTime = System.nanoTime() - startTime;
ApplicationManager.getApplication().invokeLater(() -> {
STANDARD
.createNotification(JpsCacheBundle.message("notification.title.compiler.caches.loader"),
JpsCacheBundle.message("notification.content.update.compiler.caches.completed.successfully.in.s",
endTime / 1_000_000_000),
NotificationType.INFORMATION)
.notify(myProject);
});
DOWNLOAD_DURATION_EVENT_ID.log(endTime);
myProject.getMessageBus().syncPublisher(PortableCachesLoadListener.TOPIC).loadingFinished(true);
LOG.info("Loading finished");
}
private Void handleExceptions(Void result, Throwable ex, ProgressIndicator indicator) {
if (ex != null) {
Throwable cause = ex.getCause();
if (cause instanceof ProcessCanceledException) {
LOG.info("Jps caches download canceled");
}
else {
LOG.warn("Couldn't fetch jps compilation caches", ex);
onFail();
}
getLoaders(myProject).forEach(loader -> loader.rollback());
myProject.getMessageBus().syncPublisher(PortableCachesLoadListener.TOPIC).loadingFinished(false);
indicator.setText(JpsCacheBundle.message("progress.text.rolling.back.downloaded.caches"));
}
return result;
}
private List<JpsOutputLoader<?>> getLoaders(@NotNull Project project) {
if (myJpsOutputLoadersLoaders != null) return myJpsOutputLoadersLoaders;
myJpsOutputLoadersLoaders = Arrays.asList(new JpsCacheLoader(myServerClient, project),
new JpsCompilationOutputLoader(myServerClient, myBuildOutDir));
return myJpsOutputLoadersLoaders;
}
private static LoaderStatus combine(LoaderStatus firstStatus, LoaderStatus secondStatus) {
if (firstStatus == LoaderStatus.FAILED || secondStatus == LoaderStatus.FAILED) return LoaderStatus.FAILED;
return LoaderStatus.COMPLETE;
}
private void onFail() {
ApplicationManager.getApplication().invokeLater(() -> {
ATTENTION.createNotification(JpsCacheBundle.message("notification.title.compiler.caches.loader"),
JpsCacheBundle.message("notification.content.update.compiler.caches.failed"), NotificationType.WARNING)
.notify(myProject);
});
}
}

57
plugins/jps-cache/src/com/intellij/jps/cache/model/AffectedModule.java vendored
View File

@@ -1,57 +0,0 @@
package com.intellij.jps.cache.model;
import java.io.File;
public class AffectedModule {
private final String type;
private final String name;
private final String hash;
private final File outPath;
public AffectedModule(String type, String name, String hash, File outPath) {
this.type = type;
this.name = name;
this.hash = hash;
this.outPath = outPath;
}
public String getType() {
return type;
}
public String getName() {
return name;
}
public String getHash() {
return hash;
}
public File getOutPath() {
return outPath;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
AffectedModule module = (AffectedModule)o;
if (type != null ? !type.equals(module.type) : module.type != null) return false;
if (name != null ? !name.equals(module.name) : module.name != null) return false;
if (hash != null ? !hash.equals(module.hash) : module.hash != null) return false;
if (outPath != null ? !outPath.equals(module.outPath) : module.outPath != null) return false;
return true;
}
@Override
public int hashCode() {
int result = type != null ? type.hashCode() : 0;
result = 31 * result + (name != null ? name.hashCode() : 0);
result = 31 * result + (hash != null ? hash.hashCode() : 0);
result = 31 * result + (outPath != null ? outPath.hashCode() : 0);
return result;
}
}

38
plugins/jps-cache/src/com/intellij/jps/cache/model/BuildTargetState.java vendored
View File

@@ -1,38 +0,0 @@
package com.intellij.jps.cache.model;
import java.util.Objects;
public final class BuildTargetState {
private final String hash;
private final String relativePath;
private BuildTargetState(String hash, String relativePath) {
this.hash = hash;
this.relativePath = relativePath;
}
public String getHash() {
return hash;
}
public String getRelativePath() {
return relativePath;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
BuildTargetState state = (BuildTargetState)o;
if (!Objects.equals(hash, state.hash)) return false;
if (!Objects.equals(relativePath, state.relativePath)) return false;
return true;
}
@Override
public int hashCode() {
int result = hash != null ? hash.hashCode() : 0;
result = 31 * result + (relativePath != null ? relativePath.hashCode() : 0);
return result;
}
}

49
plugins/jps-cache/src/com/intellij/jps/cache/model/JpsLoaderContext.java vendored
View File

@@ -1,49 +0,0 @@
package com.intellij.jps.cache.model;
import com.intellij.jps.cache.ui.SegmentedProgressIndicatorManager;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.util.Map;
public final class JpsLoaderContext {
private final String commitId;
private final SegmentedProgressIndicatorManager downloadIndicatorManager;
private final Map<String, Map<String, BuildTargetState>> commitSourcesState;
private final Map<String, Map<String, BuildTargetState>> currentSourcesState;
private JpsLoaderContext(@NotNull String commitId, @NotNull SegmentedProgressIndicatorManager downloadIndicatorManager,
@NotNull Map<String, Map<String, BuildTargetState>> commitSourcesState,
@Nullable Map<String, Map<String, BuildTargetState>> currentSourcesState) {
this.commitId = commitId;
this.downloadIndicatorManager = downloadIndicatorManager;
this.commitSourcesState = commitSourcesState;
this.currentSourcesState = currentSourcesState;
}
@NotNull
public String getCommitId() {
return commitId;
}
@NotNull
public SegmentedProgressIndicatorManager getDownloadIndicatorManager() {
return downloadIndicatorManager;
}
@NotNull
public Map<String, Map<String, BuildTargetState>> getCommitSourcesState() {
return commitSourcesState;
}
@Nullable
public Map<String, Map<String, BuildTargetState>> getCurrentSourcesState() {
return currentSourcesState;
}
public static JpsLoaderContext createNewContext(@NotNull String commitId, @NotNull SegmentedProgressIndicatorManager downloadIndicatorManager,
@NotNull Map<String, Map<String, BuildTargetState>> commitSourcesState,
@Nullable Map<String, Map<String, BuildTargetState>> currentSourcesState) {
return new JpsLoaderContext(commitId, downloadIndicatorManager, commitSourcesState, currentSourcesState);
}
}

27
plugins/jps-cache/src/com/intellij/jps/cache/model/OutputLoadResult.java vendored
View File

@@ -1,27 +0,0 @@
package com.intellij.jps.cache.model;
import java.io.File;
public class OutputLoadResult {
private File zipFile;
private String downloadUrl;
private AffectedModule module;
public OutputLoadResult(File zipFile, String downloadUrl, AffectedModule module) {
this.zipFile = zipFile;
this.downloadUrl = downloadUrl;
this.module = module;
}
public File getZipFile() {
return zipFile;
}
public String getDownloadUrl() {
return downloadUrl;
}
public AffectedModule getModule() {
return module;
}
}

21
plugins/jps-cache/src/com/intellij/jps/cache/statistics/JpsCacheUsagesCollector.java vendored
View File

@@ -1,21 +0,0 @@
// Copyright 2000-2021 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.intellij.jps.cache.statistics;
import com.intellij.internal.statistic.eventLog.EventLogGroup;
import com.intellij.internal.statistic.eventLog.events.EventFields;
import com.intellij.internal.statistic.eventLog.events.EventId;
import com.intellij.internal.statistic.eventLog.events.EventId1;
import com.intellij.internal.statistic.service.fus.collectors.CounterUsagesCollector;
public class JpsCacheUsagesCollector extends CounterUsagesCollector {
private static final EventLogGroup GROUP = new EventLogGroup("jps.cache", 3);
public static final EventId DOWNLOAD_THROUGH_NOTIFICATION_EVENT_ID = GROUP.registerEvent("download.through.notification");
public static final EventId1<Long> DOWNLOAD_CACHE_SIZE_EVENT_ID = GROUP.registerEvent("caches.downloaded", EventFields.Long("download_cache_size"));
public static final EventId1<Long> DOWNLOAD_BINARY_SIZE_EVENT_ID = GROUP.registerEvent("caches.downloaded", EventFields.Long("download_binary_size"));
public static final EventId1<Long> DOWNLOAD_DURATION_EVENT_ID = GROUP.registerEvent("caches.downloaded", EventFields.Long("duration"));
@Override
public EventLogGroup getGroup() {
return GROUP;
}
}

13
plugins/jps-cache/src/com/intellij/jps/cache/ui/JpsLoaderNotifications.java vendored
View File

@@ -1,13 +0,0 @@
package com.intellij.jps.cache.ui;
import com.intellij.notification.NotificationDisplayType;
import com.intellij.notification.NotificationGroup;
public final class JpsLoaderNotifications {
public static final NotificationGroup ATTENTION = new NotificationGroup("Compile Output Loader: Attention",
NotificationDisplayType.STICKY_BALLOON, true);
public static final NotificationGroup STANDARD = new NotificationGroup("Compile Output Loader: Standard",
NotificationDisplayType.BALLOON, true);
public static final NotificationGroup EVENT_LOG = new NotificationGroup("Compile Output Loader: Event Log",
NotificationDisplayType.NONE, true);
}

129
plugins/jps-cache/src/com/intellij/jps/cache/ui/SegmentedProgressIndicatorManager.java vendored
View File

@@ -1,129 +0,0 @@
package com.intellij.jps.cache.ui;
import com.intellij.openapi.progress.ProgressIndicator;
import com.intellij.openapi.progress.util.ProgressWrapper;
import com.intellij.openapi.util.NlsContexts;
import com.intellij.util.containers.hash.LinkedHashMap;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
public class SegmentedProgressIndicatorManager {
private static final LinkedHashMap<SubTaskProgressIndicator, @NlsContexts.ProgressDetails String> myText2Stack = new LinkedHashMap<>();
private static final LinkedHashMap<Object, @NlsContexts.ProgressText String> myTextStack = new LinkedHashMap<>();
private static final Object myLock = new Object();
private final ProgressIndicator myProgressIndicator;
private final boolean myProgressWasIndeterminate;
private final double mySegmentSize;
private final int myTasksCount;
public SegmentedProgressIndicatorManager(ProgressIndicator progressIndicator) {
this(progressIndicator, 1, 1);
}
public SegmentedProgressIndicatorManager(ProgressIndicator progressIndicator, int tasksCount, double segmentSize) {
myProgressIndicator = progressIndicator;
myProgressWasIndeterminate = progressIndicator.isIndeterminate();
myProgressIndicator.setIndeterminate(false);
mySegmentSize = segmentSize;
myTasksCount = tasksCount;
myText2Stack.clear();
myTextStack.clear();
}
public SubTaskProgressIndicator createSubTaskIndicator() {
assert myTasksCount != 0;
return new SubTaskProgressIndicator(this);
}
public void updateFraction(double value) {
double fractionValue = value / myTasksCount * mySegmentSize;
synchronized (myProgressIndicator) {
myProgressIndicator.setFraction(myProgressIndicator.getFraction() + fractionValue);
}
}
public void setText(@NotNull Object obj, @Nullable @NlsContexts.ProgressText String text) {
if (text != null) {
synchronized (myLock) {
myTextStack.put(obj, text);
}
myProgressIndicator.setText(text);
}
else {
String prev;
synchronized (myLock) {
myTextStack.remove(obj);
prev = myTextStack.getLastValue();
}
if (prev != null) {
myProgressIndicator.setText(prev);
}
}
}
public void setText2(@NotNull SubTaskProgressIndicator subTask, @Nullable @NlsContexts.ProgressDetails String text) {
if (text != null) {
synchronized (myLock) {
myText2Stack.put(subTask, text);
}
myProgressIndicator.setText2(text);
}
else {
String prev;
synchronized (myLock) {
myText2Stack.remove(subTask);
prev = myText2Stack.getLastValue();
}
if (prev != null) {
myProgressIndicator.setText2(prev);
}
}
}
public void finished(Object obj) {
setText(obj, null);
myProgressIndicator.setIndeterminate(myProgressWasIndeterminate);
}
public ProgressIndicator getProgressIndicator() {
return myProgressIndicator;
}
public static final class SubTaskProgressIndicator extends ProgressWrapper {
private final SegmentedProgressIndicatorManager myProgressManager;
private double myFraction;
private SubTaskProgressIndicator(SegmentedProgressIndicatorManager progressManager) {
super(progressManager.myProgressIndicator, true);
myProgressManager = progressManager;
myFraction = 0;
}
@Override
public void setFraction(double newValue) {
double diffFraction = newValue - myFraction;
myProgressManager.updateFraction(diffFraction);
myFraction = newValue;
}
@Override
public void setText2(String text) {
myProgressManager.setText2(this, text);
}
@Override
public void setText(String text) {
myProgressManager.setText(this, text);
}
@Override
public double getFraction() {
return myFraction;
}
public void finished() {
setFraction(1);
myProgressManager.setText2(this, null);
}
}
}

30
plugins/jps-cache/testData/caseFive.json
View File

@@ -1,30 +0,0 @@
{
"java-test": {
"intellij.cidr.externalSystem": {
"hash": "2278b175facf7afb99432408b82489c2",
"relativePath": "$BUILD_DIR$/test/intellij.cidr.externalSystem"
}
},
"resources-test": {
"intellij.platform.ssh.integrationTests": {
"hash": "71c661e849ef948135bdfa5b4d105a7c",
"relativePath": "$BUILD_DIR$/test/intellij.platform.ssh.integrationTests"
}
},
"java-production": {
"intellij.cidr.externalSystem": {
"hash": "9bd11b1670357c60b4f0520755597ae7",
"relativePath": "$BUILD_DIR$/production/intellij.cidr.externalSystem"
}
},
"resources-production": {
"intellij.cidr.externalSystem": {
"hash": "9bd11b17f0357c60b4f0520755597ae7",
"relativePath": "$BUILD_DIR$/production/intellij.cidr.externalSystem"
},
"intellij.sh": {
"hash": "9bd11b17f0357c60b4f0520755597ae7",
"relativePath": "$BUILD_DIR$/production/intellij.sh"
}
}
}

26
plugins/jps-cache/testData/caseFour.json
View File

@@ -1,26 +0,0 @@
{
"java-test": {
"intellij.cidr.externalSystem": {
"hash": "2278b175facf7afb99432408b82489c2",
"relativePath": "$BUILD_DIR$/test/intellij.cidr.externalSystem"
}
},
"resources-test": {
"intellij.platform.ssh.integrationTests": {
"hash": "71c661e849ef948135bdfa5b4d105a7c",
"relativePath": "$BUILD_DIR$/test/intellij.platform.ssh.integrationTests"
}
},
"java-production": {
"intellij.cidr.externalSystem": {
"hash": "9bd11b1670357c60b4f0520755597ae7",
"relativePath": "$BUILD_DIR$/production/intellij.cidr.externalSystem"
}
},
"resources-production": {
"intellij.cidr.externalSystem": {
"hash": "9bd11b17f0357c60b4f0520755597ae7",
"relativePath": "$BUILD_DIR$/production/intellij.cidr.externalSystem"
}
}
}

26
plugins/jps-cache/testData/caseOne.json
View File

@@ -1,26 +0,0 @@
{
"java-test": {
"intellij.cidr.externalSystem": {
"hash": "2278b175facf7afb99432408b82489c2",
"relativePath": "$BUILD_DIR$/test/intellij.cidr.externalSystem"
}
},
"resources-test": {
"intellij.platform.ssh.integrationTests": {
"hash": "71c661e849ef948135bdfa5b4d105a7c",
"relativePath": "$BUILD_DIR$/test/intellij.platform.ssh.integrationTests"
}
},
"java-production": {
"intellij.statsCollector": {
"hash": "499dff8ad66e7a4604d4dde46bc79fc5",
"relativePath": "$BUILD_DIR$/production/intellij.statsCollector"
}
},
"resources-production": {
"intellij.cidr.externalSystem": {
"hash": "9bd11b17f0357c60b4f0520755597ae7",
"relativePath": "$BUILD_DIR$/production/intellij.cidr.externalSystem"
}
}
}

30
plugins/jps-cache/testData/caseSix.json
View File

@@ -1,30 +0,0 @@
{
"java-test": {
"intellij.cidr.externalSystem": {
"hash": "2278b175facf7afb99432408b82489c2",
"relativePath": "$BUILD_DIR$/test/intellij.cidr.externalSystem"
}
},
"resources-test": {
"intellij.cidr.externalSystem": {
"hash": "2278b175facf7afb99432408b82489c2",
"relativePath": "$BUILD_DIR$/test/intellij.cidr.externalSystem"
}
},
"java-production": {
"intellij.cidr.externalSystem": {
"hash": "9bd11b1670357c60b4f0520755597ae7",
"relativePath": "$BUILD_DIR$/production/intellij.cidr.externalSystem"
}
},
"resources-production": {
"intellij.cidr.externalSystem": {
"hash": "9bd11b17f0357c60b4f0520755597ae7",
"relativePath": "$BUILD_DIR$/production/intellij.cidr.externalSystem"
},
"intellij.sh": {
"hash": "9bd11b17f0357c60b4f0520755597ae7",
"relativePath": "$BUILD_DIR$/production/intellij.sh"
}
}
}

32
plugins/jps-cache/testData/caseThree.json
View File

@@ -1,32 +0,0 @@
{
"java-test": {
"intellij.cidr.externalSystem": {
"hash": "2278b175facf7afb99432408b82489c2",
"relativePath": "$BUILD_DIR$/test/intellij.cidr.externalSystem"
}
},
"resources-test": {
"intellij.platform.ssh.integrationTests": {
"hash": "71c661e849ef948135bdfa5b4d105a7c",
"relativePath": "$BUILD_DIR$/test/intellij.platform.ssh.integrationTests"
}
},
"java-production": {
"intellij.statsCollector": {
"hash": "499dff8ad66e7a4344d4dde46bc79fc5",
"relativePath": "$BUILD_DIR$/production/intellij.statsCollector"
}
},
"resources-production": {
"intellij.cidr.externalSystem": {
"hash": "9bd11b17f0357c60b4f0520755597ae7",
"relativePath": "$BUILD_DIR$/production/intellij.cidr.externalSystem"
}
},
"artifacts": {
"intellij.cidr.externalSystem": {
"hash": "9bd11b17f0357c60b4f0520755597ae7",
"relativePath": "$BUILD_DIR$/production/intellij.cidr.externalSystem"
}
}
}

26
plugins/jps-cache/testData/caseTwo.json
View File

@@ -1,26 +0,0 @@
{
"java-test": {
"intellij.cidr.externalSystem": {
"hash": "2278b175facf7afb99432408b82489c2",
"relativePath": "$BUILD_DIR$/test/intellij.cidr.externalSystem"
}
},
"resources-test": {
"intellij.platform.ssh.integrationTests": {
"hash": "71c661e849ef948135bdfa5b4d105a7c",
"relativePath": "$BUILD_DIR$/test/intellij.platform.ssh.integrationTests"
}
},
"java-production": {
"intellij.statsCollector": {
"hash": "499dff8ad66e7a4344d4dde46bc79fc5",
"relativePath": "$BUILD_DIR$/production/intellij.statsCollector"
}
},
"resources-production": {
"intellij.cidr.externalSystem": {
"hash": "9bd11b17f0357c60b4f0520755597ae7",
"relativePath": "$BUILD_DIR$/production/intellij.cidr.externalSystem"
}
}
}

32
plugins/jps-cache/testData/removeOne.json
View File

@@ -1,32 +0,0 @@
{
"java-test": {
"intellij.cidr.externalSystem": {
"hash": "2278b175facf7afb99432408b82489c2",
"relativePath": "$BUILD_DIR$/test/intellij.cidr.externalSystem"
}
},
"resources-test": {
"intellij.platform.ssh.integrationTests": {
"hash": "71c661e849ef948135bdfa5b4d105a7c",
"relativePath": "$BUILD_DIR$/test/intellij.platform.ssh.integrationTests"
}
},
"java-production": {
"intellij.statsCollector": {
"hash": "499dff8ad66e7a4344d4dde46bc79fc5",
"relativePath": "$BUILD_DIR$/production/intellij.statsCollector"
}
},
"resources-production": {
"intellij.cidr.externalSystem": {
"hash": "9bd11b17f0357c60b4f0520755597ae7",
"relativePath": "$BUILD_DIR$/production/intellij.cidr.externalSystem"
}
},
"artifacts": {
"intellij.cidr.externalSystem": {
"hash": "9bd11b17f0357c60b4f0520755597ae7",
"relativePath": "$BUILD_DIR$/production/intellij.cidr"
}
}
}

26
plugins/jps-cache/testData/removeThree.json
View File

@@ -1,26 +0,0 @@
{
"java-test": {
"intellij.cidr.externalSystem": {
"hash": "2278b175facf7afb99432408b82489c2",
"relativePath": "$BUILD_DIR$/test/intellij.cidr.externalSystem"
}
},
"resources-test": {
"intellij.platform.ssh.integration": {
"hash": "71c661e849ef948135bdfa5b4d105a7c",
"relativePath": "$BUILD_DIR$/test/intellij.platform.ssh.integrationTests"
}
},
"java-production": {
"intellij.statsCollector": {
"hash": "499dff8ad66e7a4344d4dde46bc79fc5",
"relativePath": "$BUILD_DIR$/production/intellij.statsCollector"
}
},
"artifacts": {
"intellij.cidr.externalSystem": {
"hash": "9bd11b17f0357c60b4f0520755597ae7",
"relativePath": "$BUILD_DIR$/production/intellij.cidr.externalSystem"
}
}
}

32
plugins/jps-cache/testData/removeTwo.json
View File

@@ -1,32 +0,0 @@
{
"java-test": {
"intellij.cidr.externalSystem": {
"hash": "2278b175facf7afb99432408b82489c2",
"relativePath": "$BUILD_DIR$/test/intellij.cidr.externalSystem"
}
},
"resources-test": {
"intellij.platform.ssh.integration": {
"hash": "71c661e849ef948135bdfa5b4d105a7c",
"relativePath": "$BUILD_DIR$/test/intellij.platform.ssh.integrationTests"
}
},
"java-production": {
"intellij.statsCollector": {
"hash": "499dff8ad66e7a4344d4dde46bc79fc5",
"relativePath": "$BUILD_DIR$/production/intellij.statsCollector"
}
},
"resources-production": {
"intellij.cidr.externalSystem": {
"hash": "9bd11b17f0357c60b4f0520755597ae7",
"relativePath": "$BUILD_DIR$/production/intellij.cidr.externalSystem"
}
},
"artifacts": {
"intellij.cidr.externalSystem": {
"hash": "9bd11b17f0357c60b4f0520755597ae7",
"relativePath": "$BUILD_DIR$/production/intellij.cidr.externalSystem"
}
}
}

131
plugins/jps-cache/tests/com/intellij/jps/cache/loader/JpsCompilationOutputLoaderTest.java vendored
View File

@@ -1,131 +0,0 @@
package com.intellij.jps.cache.loader;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import com.intellij.jps.cache.client.JpsServerClient;
import com.intellij.jps.cache.model.AffectedModule;
import com.intellij.jps.cache.model.BuildTargetState;
import com.intellij.openapi.application.PluginPathManager;
import com.intellij.testFramework.fixtures.BasePlatformTestCase;
import com.intellij.util.ConcurrencyUtil;
import com.intellij.util.containers.ContainerUtil;
import org.jetbrains.annotations.NotNull;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.lang.reflect.Type;
import java.util.List;
import java.util.Map;
public class JpsCompilationOutputLoaderTest extends BasePlatformTestCase {
private static final String PRODUCTION = "production";
private static final String TEST = "test";
private JpsCompilationOutputLoader compilationOutputLoader;
private Type myTokenType;
private Gson myGson;
@Override
public void setUp() throws Exception {
super.setUp();
compilationOutputLoader = new JpsCompilationOutputLoader(JpsServerClient.getServerClient(), "/intellij/out/classes");
myGson = new Gson();
myTokenType = new TypeToken<Map<String, Map<String, BuildTargetState>>>() {}.getType();
}
public void testCurrentModelStateNull() throws IOException {
List<AffectedModule> affectedModules = compilationOutputLoader.getAffectedModules(null, loadModelFromFile("caseOne.json"), false);
assertSize(4, affectedModules);
// 836 production
assertSize(2, ContainerUtil.filter(affectedModules, module -> module.getType().contains(PRODUCTION)));
// 407 test
assertSize(2, ContainerUtil.filter(affectedModules, module -> module.getType().contains(TEST)));
}
public void testChangedStatsCollectorModule() throws IOException {
List<AffectedModule> affectedModules =
compilationOutputLoader.getAffectedModules(loadModelFromFile("caseOne.json"), loadModelFromFile("caseTwo.json"), false);
assertSize(1, affectedModules);
AffectedModule affectedModule = affectedModules.get(0);
assertEquals("java-production", affectedModule.getType());
assertEquals("intellij.statsCollector", affectedModule.getName());
}
public void testNewType() throws IOException {
List<AffectedModule> affectedModules = compilationOutputLoader.getAffectedModules(loadModelFromFile("caseTwo.json"),
loadModelFromFile("caseThree.json"), false);
assertSize(1, affectedModules);
AffectedModule affectedModule = affectedModules.get(0);
assertEquals("artifacts", affectedModule.getType());
assertEquals("intellij.cidr.externalSystem", affectedModule.getName());
}
public void testChangedProductionModule() throws IOException {
List<AffectedModule> affectedModules = compilationOutputLoader.getAffectedModules(loadModelFromFile("caseTwo.json"),
loadModelFromFile("caseFour.json"), false);
assertSize(1, affectedModules);
AffectedModule affectedModule = affectedModules.get(0);
assertEquals(PRODUCTION, affectedModule.getType());
assertEquals("intellij.cidr.externalSystem", affectedModule.getName());
}
public void testNewBuildModule() throws IOException {
List<AffectedModule> affectedModules = compilationOutputLoader.getAffectedModules(loadModelFromFile("caseFour.json"),
loadModelFromFile("caseFive.json"), false);
assertSize(1, affectedModules);
AffectedModule affectedModule = affectedModules.get(0);
assertEquals("resources-production", affectedModule.getType());
assertEquals("intellij.sh", affectedModule.getName());
}
public void testTargetFolderNotExist() throws IOException {
List<AffectedModule> affectedModules = compilationOutputLoader.getAffectedModules(loadModelFromFile("caseFour.json"),
loadModelFromFile("caseFive.json"), true);
assertSize(4, affectedModules);
List<String> types = ContainerUtil.map(affectedModules, AffectedModule::getType);
List<String> names = ContainerUtil.map(affectedModules, AffectedModule::getName);
assertSameElements(types, "java-test", "production", "resources-test", "resources-production");
assertSameElements(names, "intellij.cidr.externalSystem", "intellij.platform.ssh.integrationTests", "intellij.sh");
}
public void testChangedTest() throws IOException {
List<AffectedModule> affectedModules = compilationOutputLoader.getAffectedModules(loadModelFromFile("caseFive.json"),
loadModelFromFile("caseSix.json"), false);
assertSize(1, affectedModules);
AffectedModule affectedModule = affectedModules.get(0);
assertEquals(TEST, affectedModule.getType());
assertEquals("intellij.cidr.externalSystem", affectedModule.getName());
}
public void testRemoveBuildType() throws IOException {
compilationOutputLoader.getAffectedModules(loadModelFromFile("removeOne.json"), loadModelFromFile("caseOne.json"), false);
List<File> oldModulesPaths = compilationOutputLoader.getOldModulesPaths();
assertSize(1, oldModulesPaths);
assertEquals("intellij.cidr", oldModulesPaths.get(0).getName());
}
public void testRemoveModuleNotExistingInOtherBuildTypes() throws IOException {
compilationOutputLoader.getAffectedModules(loadModelFromFile("removeTwo.json"), loadModelFromFile("removeOne.json"), false);
List<File> oldModulesPaths = compilationOutputLoader.getOldModulesPaths();
assertSize(1, oldModulesPaths);
assertEquals("intellij.platform.ssh.integrationTests", oldModulesPaths.get(0).getName());
}
public void testRemoveModuleExistingInOtherBuildTypes() throws IOException {
compilationOutputLoader.getAffectedModules(loadModelFromFile("removeThree.json"), loadModelFromFile("removeTwo.json"), false);
List<File> oldModulesPaths = compilationOutputLoader.getOldModulesPaths();
assertSize(0, oldModulesPaths);
}
private Map<String, Map<String, BuildTargetState>> loadModelFromFile(String fileName) throws IOException {
try (BufferedReader bufferedReader = new BufferedReader(new FileReader(getTestDataFile(fileName)))) {
return myGson.fromJson(bufferedReader, myTokenType);
}
}
private static File getTestDataFile(@NotNull String fileName) {
return new File(PluginPathManager.getPluginHomePath("jps-cache") + "/testData/", fileName);
}
}