MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-02-04 15:06:56 +07:00
Code Issues Packages Projects Releases Wiki Activity

IDEA-297797 add option to delete old caches asynchronously during build

Browse Source 
By default, the IDE deletes old caches synchronously that makes the build process twice longer (8 vs 4 minutes). The downside is that the deletion might not be completed if the build process is fast enough. In this case, garbage will pile up under 'system/plugins'. That's why this option is disabled by default.

Additionally, we try to use rsync on *nix systems that works faster than Java NIO or even rm. When the build process finishes, rsync continues working as its process is inherited by the root process. This might not be expected by allows to make things done.

GitOrigin-RevId: f864b0b703a898bec2267e2d19cb5536483c89c0
This commit is contained in:
Artem Khvastunov
2022-10-21 10:58:52 +02:00
committed by intellij-monorepo-bot
parent c1dc30db5d
commit 944ca4e316
11 changed files with 232 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
java/compiler/impl/src/com/intellij/compiler/cache/CompilerCacheConfigurator.java vendored
View File

@@ -1,4 +1,4 @@
// Copyright 2000-2021 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. // Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.compiler.cache; package com.intellij.compiler.cache;
import com.intellij.compiler.CompilerConfigurationSettings; import com.intellij.compiler.CompilerConfigurationSettings;
@@ -42,6 +42,7 @@ public class CompilerCacheConfigurator {
builder.setDeletionSpeed(CompilerCacheLoadingSettings.getApproximateDeletionSpeed()); builder.setDeletionSpeed(CompilerCacheLoadingSettings.getApproximateDeletionSpeed());
builder.setForceDownload(CompilerCacheLoadingSettings.getForceUpdateValue()); builder.setForceDownload(CompilerCacheLoadingSettings.getForceUpdateValue());
builder.setDisableDownload(CompilerCacheLoadingSettings.getDisableUpdateValue()); builder.setDisableDownload(CompilerCacheLoadingSettings.getDisableUpdateValue());
builder.setCleanupAsynchronously(CompilerCacheLoadingSettings.getCleanupAsynchronouslyValue());
builder.setMaxDownloadDuration(CompilerCacheLoadingSettings.getMaxDownloadDuration()); builder.setMaxDownloadDuration(CompilerCacheLoadingSettings.getMaxDownloadDuration());
return builder.build(); return builder.build();
} }

12
java/compiler/impl/src/com/intellij/compiler/cache/CompilerCacheLoadingSettings.java vendored
View File

@@ -9,6 +9,7 @@ public class CompilerCacheLoadingSettings {
private static final String FORCE_UPDATE = "JpsOutputLoaderManager.forceUpdate"; private static final String FORCE_UPDATE = "JpsOutputLoaderManager.forceUpdate";
private static final String DISABLE_UPDATE = "JpsOutputLoaderManager.disableUpdate"; private static final String DISABLE_UPDATE = "JpsOutputLoaderManager.disableUpdate";
private static final String CLEANUP_ASYNCHRONOUSLY = "JpsOutputLoaderManager.cleanupAsynchronously";
private static final String MAX_DOWNLOAD_DURATION = "JpsOutputLoaderManager.maxDownloadDuration"; private static final String MAX_DOWNLOAD_DURATION = "JpsOutputLoaderManager.maxDownloadDuration";
private static final String APPROXIMATE_DELETION_SPEED = "JpsOutputLoaderManager.deletionBytesPerSec"; private static final String APPROXIMATE_DELETION_SPEED = "JpsOutputLoaderManager.deletionBytesPerSec";
private static final String APPROXIMATE_DECOMPRESSION_SPEED = "JpsOutputLoaderManager.decompressionBytesPerSec"; private static final String APPROXIMATE_DECOMPRESSION_SPEED = "JpsOutputLoaderManager.decompressionBytesPerSec";
@@ -35,6 +36,17 @@ public class CompilerCacheLoadingSettings {
return disableUpdate; return disableUpdate;
} }
public static void saveCleanupAsynchronouslyValue(boolean cleanupAsynchronously) {
PropertiesComponent.getInstance().setValue(CLEANUP_ASYNCHRONOUSLY, cleanupAsynchronously);
LOG.info("Saving cleanup asynchronously value: " + cleanupAsynchronously);
}
public static boolean getCleanupAsynchronouslyValue() {
boolean cleanupAsynchronously = PropertiesComponent.getInstance().getBoolean(CLEANUP_ASYNCHRONOUSLY, false);
LOG.info("Getting cleanup asynchronously value: " + cleanupAsynchronously);
return cleanupAsynchronously;
}
public static void saveMaxDownloadDuration(int maxDownloadDuration) { public static void saveMaxDownloadDuration(int maxDownloadDuration) {
PropertiesComponent.getInstance().setValue(MAX_DOWNLOAD_DURATION, String.valueOf(maxDownloadDuration)); PropertiesComponent.getInstance().setValue(MAX_DOWNLOAD_DURATION, String.valueOf(maxDownloadDuration));
LOG.info("Saving max download duration: " + maxDownloadDuration); LOG.info("Saving max download duration: " + maxDownloadDuration);

6
java/compiler/impl/src/com/intellij/compiler/cache/ui/CompilerCacheConfigurable.java vendored
View File

@@ -23,11 +23,14 @@ public class CompilerCacheConfigurable implements SearchableConfigurable, Config
private JBCheckBox myCbDisableCacheDownload; private JBCheckBox myCbDisableCacheDownload;
private JTextField myMaxDownloadDurationField; private JTextField myMaxDownloadDurationField;
private JBLabel myMaxDownloadDurationDescription; private JBLabel myMaxDownloadDurationDescription;
private JBCheckBox myCbCleanupAsynchronously;
private JBLabel myCleanupAsynchronouslyDescription;
public CompilerCacheConfigurable(final Project project) { public CompilerCacheConfigurable(final Project project) {
myProject = project; myProject = project;
myForceUpdateDescription.setText("<html>Turn off the heuristic that determines when it's faster to download cache or build locally and force downloads the nearest cache at each build</html>"); //NON-NLS myForceUpdateDescription.setText("<html>Turn off the heuristic that determines when it's faster to download cache or build locally and force downloads the nearest cache at each build</html>"); //NON-NLS
myMaxDownloadDurationDescription.setText("<html>Set maximum applicable time of caches download. If the approximate time of work will be higher local build will be executed.</html>"); //NON-NLS myMaxDownloadDurationDescription.setText("<html>Set maximum applicable time of caches download. If the approximate time of work will be higher local build will be executed.</html>"); //NON-NLS
myCleanupAsynchronouslyDescription.setText("<html>Removes old cache directories asynchronously. This option might produce garbage under 'system/plugins' and spawn long-living rsync processes.</html>"); //NON-NLS
} }
@Override @Override
@@ -50,6 +53,7 @@ public class CompilerCacheConfigurable implements SearchableConfigurable, Config
public boolean isModified() { public boolean isModified() {
boolean isModified = ComparingUtils.isModified(myCbForceUpdateCaches, CompilerCacheLoadingSettings.getForceUpdateValue()); boolean isModified = ComparingUtils.isModified(myCbForceUpdateCaches, CompilerCacheLoadingSettings.getForceUpdateValue());
isModified |= ComparingUtils.isModified(myCbDisableCacheDownload, CompilerCacheLoadingSettings.getDisableUpdateValue()); isModified |= ComparingUtils.isModified(myCbDisableCacheDownload, CompilerCacheLoadingSettings.getDisableUpdateValue());
isModified |= ComparingUtils.isModified(myCbCleanupAsynchronously, CompilerCacheLoadingSettings.getCleanupAsynchronouslyValue());
isModified |= ComparingUtils.isModified(myMaxDownloadDurationField, String.valueOf(CompilerCacheLoadingSettings.getMaxDownloadDuration())); isModified |= ComparingUtils.isModified(myMaxDownloadDurationField, String.valueOf(CompilerCacheLoadingSettings.getMaxDownloadDuration()));
return isModified; return isModified;
} }
@@ -59,6 +63,7 @@ public class CompilerCacheConfigurable implements SearchableConfigurable, Config
try { try {
CompilerCacheLoadingSettings.saveForceUpdateValue(myCbForceUpdateCaches.isSelected()); CompilerCacheLoadingSettings.saveForceUpdateValue(myCbForceUpdateCaches.isSelected());
CompilerCacheLoadingSettings.saveDisableUpdateValue(myCbDisableCacheDownload.isSelected()); CompilerCacheLoadingSettings.saveDisableUpdateValue(myCbDisableCacheDownload.isSelected());
CompilerCacheLoadingSettings.saveCleanupAsynchronouslyValue(myCbCleanupAsynchronously.isSelected());
CompilerCacheLoadingSettings.saveMaxDownloadDuration(Integer.parseInt(myMaxDownloadDurationField.getText())); CompilerCacheLoadingSettings.saveMaxDownloadDuration(Integer.parseInt(myMaxDownloadDurationField.getText()));
} }
finally { finally {
@@ -72,6 +77,7 @@ public class CompilerCacheConfigurable implements SearchableConfigurable, Config
public void reset() { public void reset() {
myCbForceUpdateCaches.setSelected(CompilerCacheLoadingSettings.getForceUpdateValue()); myCbForceUpdateCaches.setSelected(CompilerCacheLoadingSettings.getForceUpdateValue());
myCbDisableCacheDownload.setSelected(CompilerCacheLoadingSettings.getDisableUpdateValue()); myCbDisableCacheDownload.setSelected(CompilerCacheLoadingSettings.getDisableUpdateValue());
myCbCleanupAsynchronously.setSelected(CompilerCacheLoadingSettings.getCleanupAsynchronouslyValue());
myMaxDownloadDurationField.setText(String.valueOf(CompilerCacheLoadingSettings.getMaxDownloadDuration())); myMaxDownloadDurationField.setText(String.valueOf(CompilerCacheLoadingSettings.getMaxDownloadDuration()));
} }
} }

26
java/compiler/impl/src/com/intellij/compiler/cache/ui/CompilerCacheOptionsPanel.form vendored
View File

@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<form xmlns="http://www.intellij.com/uidesigner/form/" version="1" bind-to-class="com.intellij.compiler.cache.ui.CompilerCacheConfigurable"> <form xmlns="http://www.intellij.com/uidesigner/form/" version="1" bind-to-class="com.intellij.compiler.cache.ui.CompilerCacheConfigurable">
<grid id="49284" binding="myPanel" layout-manager="GridLayoutManager" row-count="7" column-count="1" same-size-horizontally="false" same-size-vertically="false" hgap="-1" vgap="-1"> <grid id="49284" binding="myPanel" layout-manager="GridLayoutManager" row-count="9" column-count="1" same-size-horizontally="false" same-size-vertically="false" hgap="-1" vgap="-1">
<margin top="0" left="0" bottom="0" right="0"/> <margin top="0" left="0" bottom="0" right="0"/>
<constraints> <constraints>
<xy x="37" y="129" width="737" height="280"/> <xy x="37" y="129" width="737" height="280"/>
@@ -36,7 +36,7 @@
</component> </component>
<component id="f9bc" class="com.intellij.ui.components.JBLabel" binding="myMaxDownloadDurationDescription"> <component id="f9bc" class="com.intellij.ui.components.JBLabel" binding="myMaxDownloadDurationDescription">
<constraints> <constraints>
<grid row="5" column="0" row-span="1" col-span="1" vsize-policy="3" hsize-policy="3" anchor="8" fill="0" indent="0" use-parent-layout="false"/> <grid row="7" column="0" row-span="1" col-span="1" vsize-policy="3" hsize-policy="3" anchor="8" fill="0" indent="0" use-parent-layout="false"/>
</constraints> </constraints>
<properties> <properties>
<componentStyle value="SMALL"/> <componentStyle value="SMALL"/>
@@ -61,7 +61,7 @@
<grid id="61cec" layout-manager="GridLayoutManager" row-count="1" column-count="4" same-size-horizontally="false" same-size-vertically="false" hgap="-1" vgap="-1"> <grid id="61cec" layout-manager="GridLayoutManager" row-count="1" column-count="4" same-size-horizontally="false" same-size-vertically="false" hgap="-1" vgap="-1">
<margin top="0" left="0" bottom="0" right="0"/> <margin top="0" left="0" bottom="0" right="0"/>
<constraints> <constraints>
<grid row="4" column="0" row-span="1" col-span="1" vsize-policy="3" hsize-policy="3" anchor="0" fill="3" indent="0" use-parent-layout="false"> <grid row="6" column="0" row-span="1" col-span="1" vsize-policy="3" hsize-policy="3" anchor="0" fill="3" indent="0" use-parent-layout="false">
<preferred-size width="883" height="34"/> <preferred-size width="883" height="34"/>
</grid> </grid>
</constraints> </constraints>
@@ -101,9 +101,27 @@
</grid> </grid>
<vspacer id="e338e"> <vspacer id="e338e">
<constraints> <constraints>
<grid row="6" column="0" row-span="1" col-span="1" vsize-policy="6" hsize-policy="1" anchor="0" fill="2" indent="0" use-parent-layout="false"/> <grid row="8" column="0" row-span="1" col-span="1" vsize-policy="6" hsize-policy="1" anchor="0" fill="2" indent="0" use-parent-layout="false"/>
</constraints> </constraints>
</vspacer> </vspacer>
<component id="8bb6c" class="com.intellij.ui.components.JBCheckBox" binding="myCbCleanupAsynchronously">
<constraints>
<grid row="4" column="0" row-span="1" col-span="1" vsize-policy="3" hsize-policy="3" anchor="8" fill="0" indent="0" use-parent-layout="false"/>
</constraints>
<properties>
<text resource-bundle="messages/JavaCompilerBundle" key="compiler.cache.option.cleanup.asynchronously"/>
</properties>
</component>
<component id="4757a" class="com.intellij.ui.components.JBLabel" binding="myCleanupAsynchronouslyDescription">
<constraints>
<grid row="5" column="0" row-span="1" col-span="1" vsize-policy="1" hsize-policy="3" anchor="8" fill="0" indent="2" use-parent-layout="false"/>
</constraints>
<properties>
<componentStyle value="SMALL"/>
<fontColor value="BRIGHTER"/>
<text value="" noi18n="true"/>
</properties>
</component>
</children> </children>
</grid> </grid>
</form> </form>

1
java/compiler/openapi/resources/messages/JavaCompilerBundle.properties
View File

@@ -334,6 +334,7 @@ notification.group.compiler=Build finished
compiler.cache.description=Compiler Cache compiler.cache.description=Compiler Cache
compiler.cache.option.force.update=Force update caches compiler.cache.option.force.update=Force update caches
compiler.cache.option.disable.update=Disable cache download compiler.cache.option.disable.update=Disable cache download
compiler.cache.option.cleanup.asynchronously=Cleanup aynchronously
compiler.cache.option.max.download.time=Maximum duration of caches download: compiler.cache.option.max.download.time=Maximum duration of caches download:
compiler.cache.option.max.download.time.units= mins compiler.cache.option.max.download.time.units= mins
notification.title.jps.caches.downloader=Jps caches downloader notification.title.jps.caches.downloader=Jps caches downloader

88
jps/jps-builders/gen/org/jetbrains/jps/api/CmdlineRemoteProto.java
View File

@@ -1,3 +1,5 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
// Generated by the protocol buffer compiler. DO NOT EDIT! // Generated by the protocol buffer compiler. DO NOT EDIT!
// source: cmdline_remote_proto.proto // source: cmdline_remote_proto.proto
@@ -5577,6 +5579,17 @@ public final class CmdlineRemoteProto {
* @return The maxDownloadDuration. * @return The maxDownloadDuration.
*/ */
int getMaxDownloadDuration(); int getMaxDownloadDuration();
/**
* <code>required bool cleanup_asynchronously = 10;</code>
* @return Whether the cleanupAsynchronously field is set.
*/
boolean hasCleanupAsynchronously();
/**
* <code>required bool cleanup_asynchronously = 10;</code>
* @return The cleanupAsynchronously.
*/
boolean getCleanupAsynchronously();
} }
/** /**
* Protobuf type {@code org.jetbrains.jpsservice.Message.ControllerMessage.CacheDownloadSettings} * Protobuf type {@code org.jetbrains.jpsservice.Message.ControllerMessage.CacheDownloadSettings}
@@ -5996,6 +6009,40 @@ public final class CmdlineRemoteProto {
maxDownloadDuration_ = 0; maxDownloadDuration_ = 0;
} }
public static final int CLEANUP_ASYNCHRONOUSLY_FIELD_NUMBER = 10;
private boolean cleanupAsynchronously_;
/**
* <code>required bool cleanup_asynchronously = 10;</code>
* @return Whether the cleanupAsynchronously field is set.
*/
@java.lang.Override
public boolean hasCleanupAsynchronously() {
return ((bitField0_ & 0x00000100) != 0);
}
/**
* <code>required bool cleanup_asynchronously = 10;</code>
* @return The cleanupAsynchronously.
*/
@java.lang.Override
public boolean getCleanupAsynchronously() {
return cleanupAsynchronously_;
}
/**
* <code>required bool cleanup_asynchronously = 10;</code>
* @param value The cleanupAsynchronously to set.
*/
private void setCleanupAsynchronously(boolean value) {
bitField0_ |= 0x00000100;
cleanupAsynchronously_ = value;
}
/**
* <code>required bool cleanup_asynchronously = 10;</code>
*/
private void clearCleanupAsynchronously() {
bitField0_ = (bitField0_ & ~0x00000100);
cleanupAsynchronously_ = false;
}
public static org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.CacheDownloadSettings parseFrom( public static org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.CacheDownloadSettings parseFrom(
java.nio.ByteBuffer data) java.nio.ByteBuffer data)
throws com.google.protobuf.InvalidProtocolBufferException { throws com.google.protobuf.InvalidProtocolBufferException {
@@ -6519,6 +6566,42 @@ public final class CmdlineRemoteProto {
return this; return this;
} }
/**
* <code>required bool cleanup_asynchronously = 10;</code>
* @return Whether the cleanupAsynchronously field is set.
*/
@java.lang.Override
public boolean hasCleanupAsynchronously() {
return instance.hasCleanupAsynchronously();
}
/**
* <code>required bool cleanup_asynchronously = 10;</code>
* @return The cleanupAsynchronously.
*/
@java.lang.Override
public boolean getCleanupAsynchronously() {
return instance.getCleanupAsynchronously();
}
/**
* <code>required bool cleanup_asynchronously = 10;</code>
* @param value The cleanupAsynchronously to set.
* @return This builder for chaining.
*/
public Builder setCleanupAsynchronously(boolean value) {
copyOnWrite();
instance.setCleanupAsynchronously(value);
return this;
}
/**
* <code>required bool cleanup_asynchronously = 10;</code>
* @return This builder for chaining.
*/
public Builder clearCleanupAsynchronously() {
copyOnWrite();
instance.clearCleanupAsynchronously();
return this;
}
// @@protoc_insertion_point(builder_scope:org.jetbrains.jpsservice.Message.ControllerMessage.CacheDownloadSettings) // @@protoc_insertion_point(builder_scope:org.jetbrains.jpsservice.Message.ControllerMessage.CacheDownloadSettings)
} }
private byte memoizedIsInitialized = 2; private byte memoizedIsInitialized = 2;
@@ -6547,11 +6630,12 @@ public final class CmdlineRemoteProto {
"forceDownload_", "forceDownload_",
"disableDownload_", "disableDownload_",
"maxDownloadDuration_", "maxDownloadDuration_",
"cleanupAsynchronously_",
}; };
java.lang.String info = java.lang.String info =
"\u0001\t\u0000\u0001\u0001\t\t\u0001\u0000\b\u00012\u0002\u1508\u0000\u0003\u1504" + "\u0001\n\u0000\u0001\u0001\n\n\u0001\u0000\t\u00012\u0002\u1508\u0000\u0003\u1504" +
"\u0001\u0004\u1508\u0002\u0005\u1502\u0003\u0006\u1502\u0004\u0007\u1507\u0005\b" + "\u0001\u0004\u1508\u0002\u0005\u1502\u0003\u0006\u1502\u0004\u0007\u1507\u0005\b" +
"\u1507\u0006\t\u1504\u0007"; "\u1507\u0006\t\u1504\u0007\n\u1507\b";
return newMessageInfo(DEFAULT_INSTANCE, info, objects); return newMessageInfo(DEFAULT_INSTANCE, info, objects);
} }
// fall through // fall through

1
jps/jps-builders/proto/cmdline_remote_proto.proto
View File

@@ -86,6 +86,7 @@ message Message {
required bool force_download = 7; required bool force_download = 7;
required bool disable_download = 8; required bool disable_download = 8;
required int32 max_download_duration = 9; required int32 max_download_duration = 9;
required bool cleanup_asynchronously = 10;
} }
required Type type = 1; required Type type = 1;

86
jps/jps-builders/src/org/jetbrains/jps/cache/JpsCachesLoaderUtil.java vendored
View File

@@ -1,9 +1,23 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.cache; package org.jetbrains.jps.cache;
import com.intellij.openapi.application.PathManager;
import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.SystemInfo;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.util.concurrency.AppExecutorUtil; import com.intellij.util.concurrency.AppExecutorUtil;
import org.jetbrains.annotations.NotNull;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static com.intellij.execution.process.ProcessIOExecutorService.INSTANCE; import static com.intellij.execution.process.ProcessIOExecutorService.INSTANCE;
@@ -20,4 +34,76 @@ public final class JpsCachesLoaderUtil {
LOG.info("Executor service will be configured with " + threadsCount + " threads"); LOG.info("Executor service will be configured with " + threadsCount + " threads");
return threadsCount; return threadsCount;
} }
public static void delete(@NotNull File dir, boolean asynchronously) {
if (!asynchronously) {
FileUtil.delete(dir);
return;
}
LOG.info("Deleting asynchronously... " + dir.getPath());
try {
File temp = getEmptyTempDir();
Path moved = Files.move(dir.toPath(), temp.toPath());
EXECUTOR_SERVICE.execute(() -> delete(moved));
}
catch (IOException e) {
LOG.warn("Unable to move directory: " + e.getMessage());
FileUtil.delete(dir);
}
}
public static void runCleanUpAsynchronously() {
LOG.info("Running clean-up asynchronously...");
Path pluginTemp = Path.of(PathManager.getPluginTempPath());
try (Stream<Path> stream = Files.list(pluginTemp)) {
List<Path> files = stream.filter(file -> file.getFileName().toString().startsWith(getPrefix())).collect(Collectors.toList());
if (!files.isEmpty()) {
EXECUTOR_SERVICE.execute(() -> files.forEach(JpsCachesLoaderUtil::delete));
}
}
catch (IOException e) {
LOG.warn("Unable to run clean-up task: " + e.getMessage());
}
}
private static void delete(@NotNull Path dir) {
if (SystemInfo.isUnix) {
File empty = getEmptyTempDir();
if (empty.mkdir()) {
try {
// https://unix.stackexchange.com/a/79656/47504
List<String> command = new ArrayList<>();
command.add("rsync");
command.add("-a");
command.add("--delete");
command.add(empty.getPath() + "/");
command.add(dir + "/");
ProcessBuilder builder = new ProcessBuilder(command);
Process process = builder.start();
process.waitFor();
int exitCode = process.exitValue();
LOG.info("rsync exited with " + exitCode);
}
catch (IOException | InterruptedException e) {
LOG.warn("rsync failed: " + e.getMessage());
}
finally {
FileUtil.delete(empty);
}
}
}
FileUtil.delete(dir.toFile());
}
private static @NotNull File getEmptyTempDir() {
File pluginTemp = new File(PathManager.getPluginTempPath());
String prefix = getPrefix() + UUID.randomUUID();
return FileUtil.findSequentNonexistentFile(pluginTemp, prefix, "");
}
private static @NotNull String getPrefix() {
return LOADER_TMP_FOLDER_NAME + "-";
}
} }

8
jps/jps-builders/src/org/jetbrains/jps/cache/loader/JpsCacheLoader.java vendored
View File

@@ -1,3 +1,4 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.cache.loader; package org.jetbrains.jps.cache.loader;
import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.diagnostic.Logger;
@@ -7,6 +8,7 @@ import com.intellij.util.io.Decompressor;
import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.Nullable;
import org.jetbrains.jps.builders.JpsBuildBundle; import org.jetbrains.jps.builders.JpsBuildBundle;
import org.jetbrains.jps.cache.JpsCachesLoaderUtil;
import org.jetbrains.jps.cache.client.JpsServerClient; import org.jetbrains.jps.cache.client.JpsServerClient;
import org.jetbrains.jps.cache.model.JpsLoaderContext; import org.jetbrains.jps.cache.model.JpsLoaderContext;
import org.jetbrains.jps.cache.statistics.JpsCacheLoadingSystemStats; import org.jetbrains.jps.cache.statistics.JpsCacheLoadingSystemStats;
@@ -22,12 +24,14 @@ class JpsCacheLoader implements JpsOutputLoader<File> {
private static final String FS_STATE_FILE = "fs_state.dat"; private static final String FS_STATE_FILE = "fs_state.dat";
private final File myBuildCacheFolder; private final File myBuildCacheFolder;
private final JpsServerClient myClient; private final JpsServerClient myClient;
private final boolean isCleanupAsynchronously;
private JpsLoaderContext myContext; private JpsLoaderContext myContext;
private File myTmpCacheFolder; private File myTmpCacheFolder;
JpsCacheLoader(@NotNull JpsServerClient client, @NotNull String myProjectPath) { JpsCacheLoader(@NotNull JpsServerClient client, @NotNull String myProjectPath, boolean cleanupAsynchronously) {
myBuildCacheFolder = Utils.getDataStorageRoot(myProjectPath); myBuildCacheFolder = Utils.getDataStorageRoot(myProjectPath);
myClient = client; myClient = client;
isCleanupAsynchronously = cleanupAsynchronously;
} }
@Nullable @Nullable
@@ -128,7 +132,7 @@ class JpsCacheLoader implements JpsOutputLoader<File> {
} }
// Remove old cache dir // Remove old cache dir
FileUtil.delete(myBuildCacheFolder); JpsCachesLoaderUtil.delete(myBuildCacheFolder, isCleanupAsynchronously);
myTmpCacheFolder.renameTo(myBuildCacheFolder); myTmpCacheFolder.renameTo(myBuildCacheFolder);
//subTaskIndicator.finished(); //subTaskIndicator.finished();
LOG.debug("JPS cache downloads finished"); LOG.debug("JPS cache downloads finished");

10
jps/jps-builders/src/org/jetbrains/jps/cache/loader/JpsOutputLoaderManager.java vendored
View File

@@ -1,9 +1,9 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.cache.loader; package org.jetbrains.jps.cache.loader;
import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.progress.ProcessCanceledException; import com.intellij.openapi.progress.ProcessCanceledException;
import com.intellij.openapi.util.Pair; import com.intellij.openapi.util.Pair;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.text.StringUtil; import com.intellij.openapi.util.text.StringUtil;
import com.intellij.util.containers.ContainerUtil; import com.intellij.util.containers.ContainerUtil;
import io.netty.channel.Channel; import io.netty.channel.Channel;
@@ -14,6 +14,7 @@ import org.jetbrains.jps.api.CmdlineRemoteProto;
import org.jetbrains.jps.builders.BuildTargetType; import org.jetbrains.jps.builders.BuildTargetType;
import org.jetbrains.jps.builders.JpsBuildBundle; import org.jetbrains.jps.builders.JpsBuildBundle;
import org.jetbrains.jps.builders.java.JavaBuilderUtil; import org.jetbrains.jps.builders.java.JavaBuilderUtil;
import org.jetbrains.jps.cache.JpsCachesLoaderUtil;
import org.jetbrains.jps.cache.client.JpsNettyClient; import org.jetbrains.jps.cache.client.JpsNettyClient;
import org.jetbrains.jps.cache.client.JpsServerAuthUtil; import org.jetbrains.jps.cache.client.JpsServerAuthUtil;
import org.jetbrains.jps.cache.client.JpsServerClient; import org.jetbrains.jps.cache.client.JpsServerClient;
@@ -58,6 +59,7 @@ public class JpsOutputLoaderManager {
private final JpsServerClient myServerClient; private final JpsServerClient myServerClient;
private boolean isCacheDownloaded; private boolean isCacheDownloaded;
private final boolean isForceCachesDownload; private final boolean isForceCachesDownload;
private final boolean isCleanupAsynchronously;
private final String myBuildOutDir; private final String myBuildOutDir;
private final String myProjectPath; private final String myProjectPath;
private final String myCommitHash; private final String myCommitHash;
@@ -82,6 +84,7 @@ public class JpsOutputLoaderManager {
myCommitsCountBetweenCompilation = cacheDownloadSettings.getCommitsCountLatestBuild(); myCommitsCountBetweenCompilation = cacheDownloadSettings.getCommitsCountLatestBuild();
myMaxDownloadDuration = cacheDownloadSettings.getMaxDownloadDuration() * 60; myMaxDownloadDuration = cacheDownloadSettings.getMaxDownloadDuration() * 60;
isForceCachesDownload = cacheDownloadSettings.getForceDownload(); isForceCachesDownload = cacheDownloadSettings.getForceDownload();
isCleanupAsynchronously = cacheDownloadSettings.getCleanupAsynchronously();
JpsServerAuthUtil.setRequestHeaders(cacheDownloadSettings.getAuthHeadersMap()); JpsServerAuthUtil.setRequestHeaders(cacheDownloadSettings.getAuthHeadersMap());
JpsCacheLoadingSystemStats.setDeletionSpeed(cacheDownloadSettings.getDeletionSpeed()); JpsCacheLoadingSystemStats.setDeletionSpeed(cacheDownloadSettings.getDeletionSpeed());
JpsCacheLoadingSystemStats.setDecompressionSpeed(cacheDownloadSettings.getDecompressionSpeed()); JpsCacheLoadingSystemStats.setDecompressionSpeed(cacheDownloadSettings.getDecompressionSpeed());
@@ -91,6 +94,7 @@ public class JpsOutputLoaderManager {
@NotNull List<CmdlineRemoteProto.Message.ControllerMessage.ParametersMessage.TargetTypeBuildScope> scopes, @NotNull List<CmdlineRemoteProto.Message.ControllerMessage.ParametersMessage.TargetTypeBuildScope> scopes,
@NotNull Runnable beforeDownload) { @NotNull Runnable beforeDownload) {
if (!canRunNewLoading()) return; if (!canRunNewLoading()) return;
if (isCleanupAsynchronously) JpsCachesLoaderUtil.runCleanUpAsynchronously();
if (isForceCachesDownload || isDownloadQuickerThanLocalBuild(buildRunner, myCommitsCountBetweenCompilation, scopes)) { if (isForceCachesDownload || isDownloadQuickerThanLocalBuild(buildRunner, myCommitsCountBetweenCompilation, scopes)) {
LOG.info("Before download task execution..."); LOG.info("Before download task execution...");
beforeDownload.run(); beforeDownload.run();
@@ -102,7 +106,7 @@ public class JpsOutputLoaderManager {
if (outDir.exists()) { if (outDir.exists()) {
LOG.info("Start removing old caches before downloading"); LOG.info("Start removing old caches before downloading");
myNettyClient.sendDescriptionStatusMessage(JpsBuildBundle.message("progress.text.removing.old.caches")); myNettyClient.sendDescriptionStatusMessage(JpsBuildBundle.message("progress.text.removing.old.caches"));
FileUtil.delete(outDir); JpsCachesLoaderUtil.delete(outDir, isCleanupAsynchronously);
} }
LOG.info("Compilation output folder empty"); LOG.info("Compilation output folder empty");
} }
@@ -328,7 +332,7 @@ public class JpsOutputLoaderManager {
private List<JpsOutputLoader<?>> getLoaders() { private List<JpsOutputLoader<?>> getLoaders() {
if (myJpsOutputLoadersLoaders != null) return myJpsOutputLoadersLoaders; if (myJpsOutputLoadersLoaders != null) return myJpsOutputLoadersLoaders;
myJpsOutputLoadersLoaders = Arrays.asList(new JpsCacheLoader(myServerClient, myProjectPath), myJpsOutputLoadersLoaders = Arrays.asList(new JpsCacheLoader(myServerClient, myProjectPath, isCleanupAsynchronously),
new JpsCompilationOutputLoader(myServerClient, myBuildOutDir)); new JpsCompilationOutputLoader(myServerClient, myBuildOutDir));
return myJpsOutputLoadersLoaders; return myJpsOutputLoadersLoaders;
} }

4
jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildSession.java
View File

@@ -168,7 +168,9 @@ final class BuildSession implements Runnable, CanceledStatus {
myCacheLoadManager = null; myCacheLoadManager = null;
if (ProjectStamps.PORTABLE_CACHES && myCacheDownloadSettings != null) { if (ProjectStamps.PORTABLE_CACHES && myCacheDownloadSettings != null) {
LOG.info("Cache download settings: disableDownload=" + myCacheDownloadSettings.getDisableDownload() + "; forceUpdate=" + myCacheDownloadSettings.getForceDownload()); LOG.info("Cache download settings: disableDownload=" + myCacheDownloadSettings.getDisableDownload() +
"; forceUpdate=" + myCacheDownloadSettings.getForceDownload() +
"; cleanupAsynchronously=" + myCacheDownloadSettings.getCleanupAsynchronously());
if (myCacheDownloadSettings.getDisableDownload()) { if (myCacheDownloadSettings.getDisableDownload()) {
LOG.info("Cache download is disabled"); LOG.info("Cache download is disabled");
} else { } else {