mirror of
https://gitflic.ru/project/openide/openide.git
synced 2026-03-22 06:50:54 +07:00
RUBY-31136 add option to WslSync for creating stubs for filtered out files.
(cherry picked from commit fdadb26083074b6e510df5c8b66bb3ee13bdb8c3) IJ-CR-105378 GitOrigin-RevId: b5d0d16b1bba3802191775d9400b23205242238f
This commit is contained in:
Maxat Mansurov
committed by
intellij-monorepo-bot
intellij-monorepo-bot
parent
60ab598213
commit
29da655cee
Binary file not shown.
Binary file not shown.
@@ -25,6 +25,8 @@
|
||||
// skip the hash calculation step.
|
||||
// -f FILTER
|
||||
// filters the files using the given FILTER. May be specified multiple times.
|
||||
// -s
|
||||
// report files for stubbing e.g. files that exists, but were filtered out (explicitly or implicitly).
|
||||
//
|
||||
// Description:
|
||||
// Calculate hashes (unless `-n`) for all files in the given DIR.
|
||||
@@ -47,10 +49,11 @@
|
||||
// Filters within each OPERATOR group are processed in the order of appearance in the command line.
|
||||
//
|
||||
// Output format:
|
||||
// [FILE_PATH]:[HASH]
|
||||
// [FILE_PATH]\0[HASH]
|
||||
// where HASH is little-endian 8 byte (64 bit) integer
|
||||
// [FILE_PATH];[LINK_LEN][LINK]
|
||||
// [LINK_PATH]\1[LINK_LEN][LINK]
|
||||
// where LINK_LEN is 4 byte (32 bit) signed int
|
||||
// [STUB_PATH]\2
|
||||
|
||||
//#define WSLHASH_DEBUG 1
|
||||
#ifdef WSLHASH_DEBUG
|
||||
@@ -69,6 +72,9 @@
|
||||
#define FLT_NAME_LEN_MAX (1 + FLT_MATCHER_LEN_MAX + FLT_PATTERN_LEN_MAX + 2) // OPERATOR + MATCHER + PATTERN + delims
|
||||
#define FLT_SCAN_FMT "%c:%" STRINGIFY(FLT_MATCHER_LEN_MAX) "s:%" STRINGIFY(FLT_PATTERN_LEN_MAX) "s"
|
||||
|
||||
#define FILE_SEPARATOR 0
|
||||
#define LINK_SEPARATOR 1
|
||||
#define STUB_SEPARATOR 2
|
||||
|
||||
struct wslhash_filter_t {
|
||||
char name[FLT_NAME_LEN_MAX + 1]; // full filter name (OPERATOR:MATCHER:PATTERN).
|
||||
@@ -93,6 +99,7 @@ struct wslhash_options_t {
|
||||
size_t includes_len;
|
||||
|
||||
int skip_hash;
|
||||
int report_stubs;
|
||||
};
|
||||
|
||||
|
||||
@@ -148,8 +155,8 @@ static int is_dir(const char *path) {
|
||||
return S_ISDIR(stat_info.st_mode);
|
||||
}
|
||||
|
||||
static const char * filename(const char* fpath) {
|
||||
const char* last_slash = strrchr(fpath, '/');
|
||||
static const char *filename(const char *fpath) {
|
||||
const char *last_slash = strrchr(fpath, '/');
|
||||
return (last_slash != NULL) ? last_slash + 1 : fpath;
|
||||
}
|
||||
|
||||
@@ -158,16 +165,20 @@ static int
|
||||
process_file(const char *fpath, const struct stat *sb, int tflag, __attribute__((unused)) struct FTW *ftwbuf) {
|
||||
DEBUG_PRINTF("Processing file: %s\n", fpath);
|
||||
if (tflag != FTW_F && tflag != FTW_SL) {
|
||||
DEBUG_PRINTF("Skipping file: %s\n", fpath);
|
||||
DEBUG_PRINTF("Skipping: %s\n", fpath);
|
||||
return 0; // Not a file
|
||||
}
|
||||
if (tflag == FTW_F && !is_filename_ok(filename(fpath))) {
|
||||
DEBUG_PRINTF("Excluding file: %s\n", fpath);
|
||||
return 0;
|
||||
}
|
||||
const char *fpath_relative = fpath + g_options.root_dir_len + 1; // remove first "/"
|
||||
if (tflag == FTW_F) {
|
||||
printf("%s:", fpath_relative);
|
||||
if (!is_filename_ok(filename(fpath))) {
|
||||
DEBUG_PRINTF("Excluding file: %s\n", fpath);
|
||||
if (g_options.report_stubs) {
|
||||
printf("%s%c", fpath_relative, STUB_SEPARATOR);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
printf("%s%c", fpath_relative, FILE_SEPARATOR);
|
||||
if (sb->st_size == 0 || g_options.skip_hash) {
|
||||
// No need to calculate hash for empty file
|
||||
fwrite(EMPTY, sizeof(EMPTY), 1, stdout);
|
||||
@@ -196,7 +207,7 @@ process_file(const char *fpath, const struct stat *sb, int tflag, __attribute__(
|
||||
} else {
|
||||
char real_path[PATH_MAX] = {0};
|
||||
if (realpath(fpath, real_path) != NULL && is_dir(real_path)) {
|
||||
printf("%s;", fpath_relative);
|
||||
printf("%s%c", fpath_relative, LINK_SEPARATOR);
|
||||
const int32_t len = (int32_t) strlen(real_path);
|
||||
fwrite(&len, sizeof(int32_t), 1, stdout);
|
||||
fputs(real_path, stdout);
|
||||
@@ -283,8 +294,11 @@ static void parse_filter(const char *arg) {
|
||||
|
||||
static void parse_args(int argc, char *argv[]) {
|
||||
int c;
|
||||
while ((c = getopt(argc, argv, "nf:")) != -1) {
|
||||
while ((c = getopt(argc, argv, "nsf:")) != -1) {
|
||||
switch (c) {
|
||||
case 's':
|
||||
g_options.report_stubs = 1;
|
||||
break;
|
||||
case 'n':
|
||||
g_options.skip_hash = 1;
|
||||
break;
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
package com.intellij.execution.wsl
|
||||
|
||||
import com.intellij.execution.wsl.sync.*
|
||||
import com.intellij.execution.wsl.sync.WslHashFilters.Companion.EMPTY_FILTERS
|
||||
import com.intellij.execution.wsl.sync.WslHashFilters.WslHashFiltersBuilder
|
||||
import com.intellij.execution.wsl.sync.WslHashMatcher.Factory.basename
|
||||
import com.intellij.execution.wsl.sync.WslHashMatcher.Factory.extension
|
||||
@@ -64,7 +63,7 @@ class WslSyncTest(private val linToWin: Boolean) {
|
||||
for (source in sources) {
|
||||
storage.createSymLinks(mapOf(Pair(source, target)))
|
||||
}
|
||||
return storage.getHashesAndLinks(false).second
|
||||
return storage.calculateSyncData().links
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,13 +89,13 @@ class WslSyncTest(private val linToWin: Boolean) {
|
||||
if (linToWin) {
|
||||
val dir = linuxDirRule.dir
|
||||
wslRule.wsl.executeOnWsl(1000, "mkdir", "${dir}/target")
|
||||
storage = LinuxFileStorage(dir, wslRule.wsl, EMPTY_FILTERS)
|
||||
storage = LinuxFileStorage(dir, wslRule.wsl)
|
||||
}
|
||||
else {
|
||||
val dir = winDirRule.newDirectoryPath()
|
||||
val targetDir = dir.resolve("target")
|
||||
targetDir.createDirectory()
|
||||
storage = WindowsFileStorage(dir, wslRule.wsl, EMPTY_FILTERS)
|
||||
storage = WindowsFileStorage(dir, wslRule.wsl)
|
||||
}
|
||||
|
||||
val links = createAndGetLinks(storage, FilePathRelativeToDir("target"), *sources)
|
||||
@@ -112,8 +111,8 @@ class WslSyncTest(private val linToWin: Boolean) {
|
||||
val to: FileStorage<*, *>
|
||||
val winRoot = winDirRule.newDirectoryPath()
|
||||
val linRoot = linuxDirRule.dir
|
||||
val win = WindowsFileStorage(winRoot, wslRule.wsl, EMPTY_FILTERS)
|
||||
val lin = LinuxFileStorage(linRoot, wslRule.wsl, EMPTY_FILTERS)
|
||||
val win = WindowsFileStorage(winRoot, wslRule.wsl)
|
||||
val lin = LinuxFileStorage(linRoot, wslRule.wsl)
|
||||
if (!linToWin) {
|
||||
winRoot.resolve("target dir").createDirectory().resolve("file.txt").createFile()
|
||||
winRoot.resolve("dir_to_ignore").createDirectory()
|
||||
@@ -132,13 +131,13 @@ class WslSyncTest(private val linToWin: Boolean) {
|
||||
}
|
||||
to.createSymLinks(mapOf(Pair(FilePathRelativeToDir("dir_to_ignore/foo"), FilePathRelativeToDir("target dir"))))
|
||||
WslSync.syncWslFolders(linRoot, winRoot, wslRule.wsl, linToWinCopy = linToWin)
|
||||
val links = to.getHashesAndLinks(false).second
|
||||
val links = to.calculateSyncData().links
|
||||
for (source in sources) {
|
||||
Assert.assertEquals("target dir", links[source]?.asWindowsPath)
|
||||
}
|
||||
to.createSymLinks(mapOf(Pair(FilePathRelativeToDir("remove_me"), FilePathRelativeToDir("target dir"))))
|
||||
WslSync.syncWslFolders(linRoot, winRoot, wslRule.wsl, linToWinCopy = linToWin)
|
||||
Assert.assertEquals(null, to.getHashesAndLinks(false).second[FilePathRelativeToDir("remove_me")])
|
||||
Assert.assertEquals(null, to.calculateSyncData().links[FilePathRelativeToDir("remove_me")])
|
||||
for (source in sources) {
|
||||
Assert.assertEquals("target dir", links[source]?.asWindowsPath)
|
||||
}
|
||||
@@ -252,7 +251,8 @@ class WslSyncTest(private val linToWin: Boolean) {
|
||||
.exclude(*extensions("dll", "gz", "zip"),
|
||||
basename("debug"),
|
||||
fullname("idea.log"))
|
||||
.build()
|
||||
.build(),
|
||||
false
|
||||
)
|
||||
}
|
||||
|
||||
@@ -266,6 +266,7 @@ class WslSyncTest(private val linToWin: Boolean) {
|
||||
basename("debug"),
|
||||
fullname("idea.log"))
|
||||
.build(),
|
||||
false
|
||||
)
|
||||
}
|
||||
|
||||
@@ -281,12 +282,44 @@ class WslSyncTest(private val linToWin: Boolean) {
|
||||
.include(extension("tar.gz"),
|
||||
fullname("debug.in"))
|
||||
.build(),
|
||||
false
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun syncWithExcludesAndStubs() {
|
||||
doSyncAndAssertFilePresence(
|
||||
setOf(),
|
||||
setOf("файл.dll", "файл.gz", "файл.zip", "debug", "debug.out", "idea.log",
|
||||
"файл.py", "файл.tar.gz", "файл.java", "файл-dll", "ddebug", "debug.in", "idea.log.bck"),
|
||||
WslHashFiltersBuilder()
|
||||
.exclude(*extensions("dll", "gz", "zip"),
|
||||
basename("debug"),
|
||||
fullname("idea.log"))
|
||||
.build(),
|
||||
true
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun syncWithIncludesAndStubs() {
|
||||
doSyncAndAssertFilePresence(
|
||||
setOf(),
|
||||
setOf("файл.dll", "файл.gz", "файл.zip", "debug", "debug.out", "idea.log",
|
||||
"файл.py", "файл.tar.gz", "файл.java", "файл-dll", "ddebug", "debug.in", "idea.log.bck"),
|
||||
WslHashFiltersBuilder()
|
||||
.include(*extensions("dll", "gz", "zip"),
|
||||
basename("debug"),
|
||||
fullname("idea.log"))
|
||||
.build(),
|
||||
true
|
||||
)
|
||||
}
|
||||
|
||||
private fun doSyncAndAssertFilePresence(fileNamesToIgnore: Set<String>,
|
||||
fileNamesToSync: Set<String>,
|
||||
filters: WslHashFilters) {
|
||||
filters: WslHashFilters,
|
||||
useStubs: Boolean) {
|
||||
val windowsDir = winDirRule.newDirectoryPath()
|
||||
val allFileNames = fileNamesToSync + fileNamesToIgnore
|
||||
|
||||
@@ -297,7 +330,7 @@ class WslSyncTest(private val linToWin: Boolean) {
|
||||
srcDir.resolve(fileName).writeText("hello $fileName")
|
||||
}
|
||||
|
||||
WslSync.syncWslFolders(linuxDirRule.dir, windowsDir, wslRule.wsl, linToWin, filters)
|
||||
WslSync.syncWslFolders(linuxDirRule.dir, windowsDir, wslRule.wsl, linToWin, filters, useStubs)
|
||||
|
||||
for (fileName in allFileNames) {
|
||||
val file = dstDir.resolve(fileName)
|
||||
@@ -306,6 +339,9 @@ class WslSyncTest(private val linToWin: Boolean) {
|
||||
}
|
||||
else {
|
||||
Assert.assertTrue("File ${file} must be copied", file.exists())
|
||||
if (useStubs && !filters.isFileNameOk(file.fileName.toString())) {
|
||||
Assert.assertEquals("File ${file} must be stubbed", "", file.readText().trim())
|
||||
}
|
||||
}
|
||||
}
|
||||
Assert.assertEquals("Not all files synced", fileNamesToSync.size, dstDir.toFile().list()!!.size)
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
package com.intellij.execution.wsl.sync
|
||||
|
||||
import com.intellij.execution.wsl.AbstractWslDistribution
|
||||
import com.intellij.execution.wsl.sync.WslHashFilters.Companion.EMPTY_FILTERS
|
||||
|
||||
|
||||
/**
|
||||
@@ -12,8 +13,7 @@ import com.intellij.execution.wsl.AbstractWslDistribution
|
||||
*/
|
||||
abstract class FileStorage<MyFileType, OtherSideFileType>(
|
||||
protected val dir: MyFileType,
|
||||
protected val distro: AbstractWslDistribution,
|
||||
protected val filters: WslHashFilters
|
||||
protected val distro: AbstractWslDistribution
|
||||
) {
|
||||
|
||||
/**
|
||||
@@ -22,11 +22,14 @@ abstract class FileStorage<MyFileType, OtherSideFileType>(
|
||||
abstract fun createSymLinks(links: Map<FilePathRelativeToDir, FilePathRelativeToDir>)
|
||||
|
||||
/**
|
||||
* List of [WslHashRecord] (file + hash) and map of `source->target` links.
|
||||
* [skipHashCalculation] saves time by skipping hash (hence [WslHashRecord.hash] is 0).
|
||||
* [filters] determine which files to process.
|
||||
* [skipHash] saves time by skipping hash (hence each [WslHashRecord.hash] in [WslSyncData.hashes] is 0).
|
||||
* Such records can't be used for sync, but only to copy all files
|
||||
* [useStubs] specifies whether files for stubbing will be reported.
|
||||
*/
|
||||
abstract fun getHashesAndLinks(skipHashCalculation: Boolean): Pair<List<WslHashRecord>, Map<FilePathRelativeToDir, FilePathRelativeToDir>>
|
||||
abstract fun calculateSyncData(filters: WslHashFilters = EMPTY_FILTERS,
|
||||
skipHash: Boolean = false,
|
||||
useStubs: Boolean = false): WslSyncData
|
||||
|
||||
/**
|
||||
* is [dir] empty
|
||||
@@ -35,6 +38,7 @@ abstract class FileStorage<MyFileType, OtherSideFileType>(
|
||||
abstract fun removeFiles(filesToRemove: Collection<FilePathRelativeToDir>)
|
||||
abstract fun createTempFile(): MyFileType
|
||||
abstract fun removeTempFile(file: MyFileType)
|
||||
abstract fun createStubs(files: Collection<FilePathRelativeToDir>)
|
||||
|
||||
/**
|
||||
* tar [files] and copy to [destTar]
|
||||
|
||||
@@ -6,8 +6,8 @@ import com.intellij.execution.processTools.getBareExecutionResult
|
||||
import com.intellij.execution.processTools.getResultStdoutStr
|
||||
import com.intellij.execution.wsl.*
|
||||
import com.intellij.openapi.diagnostic.Logger
|
||||
import com.intellij.openapi.util.Ref
|
||||
import com.intellij.util.TimeoutUtil
|
||||
import com.intellij.util.containers.ContainerUtil.append
|
||||
import com.intellij.util.io.delete
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import java.io.InputStream
|
||||
@@ -19,14 +19,15 @@ import kotlin.io.path.writeText
|
||||
private val LOGGER = Logger.getInstance(LinuxFileStorage::class.java)
|
||||
|
||||
|
||||
class LinuxFileStorage(dir: LinuxFilePath, distro: AbstractWslDistribution, filters: WslHashFilters)
|
||||
: FileStorage<LinuxFilePath, WindowsFilePath>(dir.trimEnd('/') + '/', distro, filters) {
|
||||
class LinuxFileStorage(dir: LinuxFilePath, distro: AbstractWslDistribution)
|
||||
: FileStorage<LinuxFilePath, WindowsFilePath>(dir.trimEnd('/') + '/', distro) {
|
||||
|
||||
// Linux side only works with UTF of 7-bit ASCII which is also supported by UTF and WSL doesn't support other charsets
|
||||
|
||||
private val CHARSET = Charsets.UTF_8
|
||||
private val FILE_SEPARATOR = CHARSET.encode(":").get()
|
||||
private val LINK_SEPARATOR = CHARSET.encode(";").get()
|
||||
|
||||
private val FILE_SEPARATOR: Byte = 0
|
||||
private val LINK_SEPARATOR: Byte = 1
|
||||
private val STUB_SEPARATOR: Byte = 2
|
||||
|
||||
override fun createSymLinks(links: Map<FilePathRelativeToDir, FilePathRelativeToDir>) {
|
||||
val script = createTmpWinFile(distro)
|
||||
@@ -39,27 +40,38 @@ class LinuxFileStorage(dir: LinuxFilePath, distro: AbstractWslDistribution, filt
|
||||
script.first.delete()
|
||||
}
|
||||
|
||||
override fun getHashesAndLinks(skipHashCalculation: Boolean): Pair<List<WslHashRecord>, Map<FilePathRelativeToDir, FilePathRelativeToDir>> {
|
||||
val hashes = ArrayList<WslHashRecord>(AVG_NUM_FILES)
|
||||
val links = HashMap<FilePathRelativeToDir, FilePathRelativeToDir>(AVG_NUM_FILES)
|
||||
override fun calculateSyncData(filters: WslHashFilters, skipHash: Boolean, useStubs: Boolean): WslSyncData {
|
||||
val dataRef = Ref<WslSyncData>()
|
||||
val time = TimeoutUtil.measureExecutionTime<Throwable> {
|
||||
val wslHashArgs = if (skipHashCalculation) append(filters.toArgs(), "-n", dir) else append(filters.toArgs(), dir)
|
||||
val wslHashArgs = listOfNotNull(if (skipHash) "-n" else null,
|
||||
if (useStubs) "-s" else null,
|
||||
*filters.toArgs().toTypedArray(),
|
||||
dir)
|
||||
val tool = distro.getTool("wslhash", *wslHashArgs.toTypedArray())
|
||||
val process = tool.createProcess()
|
||||
process.inputStream.use {
|
||||
val hashesAndLinks = getHashesInternal(it)
|
||||
hashes += hashesAndLinks.first
|
||||
links += hashesAndLinks.second
|
||||
dataRef.set(calculateSyncDataInternal(it))
|
||||
}
|
||||
runBlocking { process.getResultStdoutStr() }
|
||||
}
|
||||
LOGGER.info("Linux files calculated in $time")
|
||||
return Pair(hashes, links)
|
||||
return dataRef.get()
|
||||
}
|
||||
|
||||
|
||||
override fun createTempFile(): String = distro.runCommand("mktemp", "-u").getOrThrow()
|
||||
|
||||
override fun createStubs(files: Collection<FilePathRelativeToDir>) {
|
||||
val script = createTmpWinFile(distro)
|
||||
try {
|
||||
val scriptContent = files.joinToString("\n") { "mkdir -p \"$(dirname ${it.escapedWithDir})\" && touch ${it.escapedWithDir}" }
|
||||
script.first.writeText(scriptContent)
|
||||
runBlocking { distro.createProcess("sh", script.second).getBareExecutionResult() }
|
||||
}
|
||||
finally {
|
||||
script.first.delete()
|
||||
}
|
||||
}
|
||||
|
||||
override fun removeLinks(vararg linksToRemove: FilePathRelativeToDir) {
|
||||
this.removeFiles(linksToRemove.asList())
|
||||
}
|
||||
@@ -113,17 +125,19 @@ class LinuxFileStorage(dir: LinuxFilePath, distro: AbstractWslDistribution, filt
|
||||
}
|
||||
|
||||
/**
|
||||
* Read `wslhash` stdout and return map of [file->hash]
|
||||
* Parse output from `wslhash` and return [WslSyncData].
|
||||
*/
|
||||
private fun getHashesInternal(toolStdout: InputStream): Pair<List<WslHashRecord>, Map<FilePathRelativeToDir, FilePathRelativeToDir>> {
|
||||
private fun calculateSyncDataInternal(toolStdout: InputStream): WslSyncData {
|
||||
val hashes = ArrayList<WslHashRecord>(AVG_NUM_FILES)
|
||||
val links = HashMap<FilePathRelativeToDir, FilePathRelativeToDir>(AVG_NUM_FILES)
|
||||
val links = mutableMapOf<FilePathRelativeToDir, FilePathRelativeToDir>()
|
||||
val stubs = mutableSetOf<FilePathRelativeToDir>()
|
||||
val fileOutput = ByteBuffer.wrap(toolStdout.readAllBytes()).order(ByteOrder.LITTLE_ENDIAN)
|
||||
|
||||
// See wslhash.c: format is the following: [file_path]:[hash].
|
||||
// Hash is little-endian 8 byte (64 bit) integer
|
||||
// or [file_path];[link_len][link] where link_len is 4 byte signed int
|
||||
|
||||
// See wslhash.c.
|
||||
// Output format is the following:
|
||||
// [file_path]\0[hash], where hash is little-endian 8 byte (64 bit) integer
|
||||
// [link_path]\1[link_len][link], where link_len is 4 byte signed int
|
||||
// [stub_path]\2
|
||||
var fileStarted = 0
|
||||
val outputLimit = fileOutput.limit()
|
||||
while (fileOutput.position() < outputLimit) {
|
||||
@@ -149,9 +163,17 @@ class LinuxFileStorage(dir: LinuxFilePath, distro: AbstractWslDistribution, filt
|
||||
}
|
||||
fileStarted = prevPos + length
|
||||
}
|
||||
STUB_SEPARATOR -> {
|
||||
val prevPos = fileOutput.position()
|
||||
// 1 = separator
|
||||
val name = CHARSET.decode(fileOutput.limit(prevPos - 1).position(fileStarted)).toString()
|
||||
fileOutput.limit(outputLimit).position(prevPos)
|
||||
stubs += FilePathRelativeToDir(name)
|
||||
fileStarted = prevPos
|
||||
}
|
||||
}
|
||||
}
|
||||
return Pair(hashes, links)
|
||||
return WslSyncData(hashes, links, stubs)
|
||||
}
|
||||
|
||||
private val FilePathRelativeToDir.escapedWithDir: String get() = escapePath(dir + asUnixPath)
|
||||
|
||||
@@ -5,11 +5,11 @@ import com.intellij.execution.configurations.GeneralCommandLine
|
||||
import com.intellij.execution.util.ExecUtil
|
||||
import com.intellij.execution.wsl.AbstractWslDistribution
|
||||
import com.intellij.openapi.diagnostic.Logger
|
||||
import com.intellij.openapi.diagnostic.thisLogger
|
||||
import com.intellij.openapi.util.io.FileSystemUtil
|
||||
import com.intellij.util.TimeoutUtil
|
||||
import com.intellij.util.io.*
|
||||
import com.intellij.util.system.CpuArch
|
||||
import net.jpountz.xxhash.XXHash64
|
||||
import net.jpountz.xxhash.XXHashFactory
|
||||
import java.io.IOException
|
||||
import java.nio.channels.FileChannel
|
||||
@@ -24,27 +24,55 @@ private val LOGGER = Logger.getInstance(WindowsFileStorage::class.java)
|
||||
|
||||
private class MyFileVisitor(private val filters: WslHashFilters,
|
||||
private val rootDir: Path,
|
||||
private val processFile: (relativeToDir: FilePathRelativeToDir, file: Path, attrs: BasicFileAttributes) -> Unit) : SimpleFileVisitor<Path>() {
|
||||
private val dirLinksInt: MutableMap<FilePathRelativeToDir, FilePathRelativeToDir> = mutableMapOf()
|
||||
val dirLinks: Map<FilePathRelativeToDir, FilePathRelativeToDir> get() = dirLinksInt
|
||||
private val hashTool: XXHash64,
|
||||
private val skipHash: Boolean,
|
||||
private val useStubs: Boolean) : SimpleFileVisitor<Path>() {
|
||||
|
||||
private val _hashes: MutableList<WslHashRecord> = ArrayList(AVG_NUM_FILES)
|
||||
private val _dirLinks: MutableMap<FilePathRelativeToDir, FilePathRelativeToDir> = mutableMapOf()
|
||||
private val _stubs: MutableSet<FilePathRelativeToDir> = mutableSetOf()
|
||||
|
||||
val hashes: List<WslHashRecord> get() = _hashes
|
||||
val dirLinks: Map<FilePathRelativeToDir, FilePathRelativeToDir> get() = _dirLinks
|
||||
val stubs: Set<FilePathRelativeToDir> get() = _stubs
|
||||
|
||||
override fun postVisitDirectory(dir: Path?, exc: IOException?): FileVisitResult {
|
||||
return super.postVisitDirectory(dir, exc)
|
||||
}
|
||||
|
||||
override fun visitFile(path: Path, attrs: BasicFileAttributes): FileVisitResult {
|
||||
if (!(attrs.isRegularFile)) return FileVisitResult.CONTINUE
|
||||
if (!filters.isFileNameOk(path.fileName.toString())) {
|
||||
return FileVisitResult.CONTINUE
|
||||
}
|
||||
processFile(FilePathRelativeToDir(rootDir.relativize(path).joinToString("/").lowercase()), path, attrs)
|
||||
return FileVisitResult.CONTINUE
|
||||
}
|
||||
|
||||
fun processFile(relativeToDir: FilePathRelativeToDir, file: Path, attrs: BasicFileAttributes) {
|
||||
if (!filters.isFileNameOk(file.fileName.toString())) {
|
||||
if (useStubs) {
|
||||
_stubs.add(relativeToDir)
|
||||
}
|
||||
}
|
||||
else if (skipHash || attrs.size() == 0L) { // Empty file's hash is 0, see wslhash.c
|
||||
_hashes.add(WslHashRecord(relativeToDir, 0))
|
||||
}
|
||||
else { // Map file and read hash
|
||||
FileChannel.open(file, StandardOpenOption.READ).use {
|
||||
val buf = it.map(FileChannel.MapMode.READ_ONLY, 0, attrs.size())
|
||||
try {
|
||||
_hashes.add(WslHashRecord(relativeToDir, hashTool.hash(buf, 0))) // Seed 0 is default, see wslhash.c
|
||||
}
|
||||
finally {
|
||||
ByteBufferUtil.cleanBuffer(buf) // Unmap file: can't overwrite mapped file
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult = if (FileSystemUtil.getAttributes(
|
||||
dir.toFile())?.isSymLink == true) {
|
||||
val target = FileSystemUtil.resolveSymLink(dir.toFile())?.let { rootDir.resolve(it) }
|
||||
if (target != null && target.isDirectory() && target.startsWith(rootDir)) {
|
||||
dirLinksInt[FilePathRelativeToDir(rootDir.relativize(dir).toString())] = FilePathRelativeToDir(rootDir.relativize(target).toString())
|
||||
_dirLinks[FilePathRelativeToDir(rootDir.relativize(dir).toString())] = FilePathRelativeToDir(rootDir.relativize(target).toString())
|
||||
}
|
||||
FileVisitResult.SKIP_SUBTREE
|
||||
}
|
||||
@@ -54,8 +82,7 @@ private class MyFileVisitor(private val filters: WslHashFilters,
|
||||
}
|
||||
|
||||
class WindowsFileStorage(dir: Path,
|
||||
distro: AbstractWslDistribution,
|
||||
filters: WslHashFilters) : FileStorage<WindowsFilePath, LinuxFilePath>(dir, distro, filters) {
|
||||
distro: AbstractWslDistribution) : FileStorage<WindowsFilePath, LinuxFilePath>(dir, distro) {
|
||||
private fun runCommand(vararg commands: String) {
|
||||
val cmd = arrayOf("cmd", "/c", *commands)
|
||||
ExecUtil.execAndGetOutput(GeneralCommandLine(*cmd)).let {
|
||||
@@ -76,36 +103,21 @@ class WindowsFileStorage(dir: Path,
|
||||
}
|
||||
}
|
||||
|
||||
override fun getHashesAndLinks(skipHashCalculation: Boolean): Pair<List<WslHashRecord>, Map<FilePathRelativeToDir, FilePathRelativeToDir>> {
|
||||
val result = ArrayList<WslHashRecord>(AVG_NUM_FILES)
|
||||
override fun calculateSyncData(filters: WslHashFilters, skipHash: Boolean, useStubs: Boolean): WslSyncData {
|
||||
val arch = System.getProperty("os.arch")
|
||||
val useNativeHash = CpuArch.CURRENT == CpuArch.X86_64
|
||||
thisLogger().info("Arch $arch, using native hash: $useNativeHash")
|
||||
LOGGER.info("Arch $arch, using native hash: $useNativeHash")
|
||||
val hashTool = if (useNativeHash) XXHashFactory.nativeInstance().hash64() else XXHashFactory.safeInstance().hash64() // Native hash can access direct (mapped) buffer a little-bit faster
|
||||
val visitor = MyFileVisitor(filters, dir) { relativeToDir: FilePathRelativeToDir, file: Path, attrs: BasicFileAttributes ->
|
||||
if (skipHashCalculation || attrs.size() == 0L) { // Empty file's hash is 0, see wslhash.c
|
||||
result.add(WslHashRecord(relativeToDir, 0))
|
||||
}
|
||||
else { // Map file and read hash
|
||||
FileChannel.open(file, StandardOpenOption.READ).use {
|
||||
val buf = it.map(FileChannel.MapMode.READ_ONLY, 0, attrs.size())
|
||||
try {
|
||||
result.add(WslHashRecord(relativeToDir, hashTool.hash(buf, 0))) // Seed 0 is default, see wslhash.c
|
||||
}
|
||||
finally {
|
||||
ByteBufferUtil.cleanBuffer(buf) // Unmap file: can't overwrite mapped file
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
val visitor = MyFileVisitor(filters, dir, hashTool, skipHash, useStubs)
|
||||
val time = TimeoutUtil.measureExecutionTime<Throwable> {
|
||||
Files.walkFileTree(dir, visitor)
|
||||
}
|
||||
LOGGER.info("Windows files calculated in $time")
|
||||
return Pair(result, visitor.dirLinks)
|
||||
return WslSyncData(visitor.hashes, visitor.dirLinks, visitor.stubs)
|
||||
}
|
||||
|
||||
override fun isEmpty(): Boolean = dir.notExists() || dir.listDirectoryEntries().isEmpty()
|
||||
|
||||
override fun removeFiles(filesToRemove: Collection<FilePathRelativeToDir>) {
|
||||
if (filesToRemove.isEmpty()) return
|
||||
for (file in filesToRemove) {
|
||||
@@ -117,6 +129,16 @@ class WindowsFileStorage(dir: Path,
|
||||
}
|
||||
|
||||
override fun createTempFile(): Path = createTmpWinFile(distro).first
|
||||
|
||||
override fun createStubs(files: Collection<FilePathRelativeToDir>) {
|
||||
for (file in files) {
|
||||
val filePath = dir.resolve(file.asWindowsPath)
|
||||
if (!filePath.exists()) {
|
||||
filePath.createFile()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun removeLinks(vararg linksToRemove: FilePathRelativeToDir) {
|
||||
for (link in linksToRemove) {
|
||||
runCommand("rmdir", dir.resolve(link.asWindowsPath).toString())
|
||||
|
||||
@@ -25,36 +25,38 @@ private const val MIN_CHUNK_SIZE = 1000
|
||||
private val LOGGER = Logger.getInstance(WslSync::class.java)
|
||||
|
||||
class WslSync<SourceFile, DestFile> private constructor(private val source: FileStorage<SourceFile, DestFile>,
|
||||
private val dest: FileStorage<DestFile, SourceFile>) {
|
||||
|
||||
private val dest: FileStorage<DestFile, SourceFile>,
|
||||
private val filters: WslHashFilters,
|
||||
private val useStubs: Boolean) {
|
||||
|
||||
companion object {
|
||||
|
||||
/**
|
||||
* Makes [windowsDir] reflect [linuxDir] (or vice versa depending on [linToWinCopy]) on [distribution] much like rsync.
|
||||
* Redundant files deleted, new/changed files copied.
|
||||
* Set [onlyExtensions] if you only care about certain extensions.
|
||||
* Direction depends on [linToWinCopy]
|
||||
* Synchronizes the given [windowsDir] and [linuxDir] (inside [distro]).
|
||||
* [linToWinCopy] determines the sync direction.
|
||||
* [filters] allow you to specify which files to include/exclude.
|
||||
* [useStubs] dictates whether empty stubs should be created for filtered out files.
|
||||
*/
|
||||
@JvmOverloads
|
||||
fun syncWslFolders(linuxDir: String,
|
||||
windowsDir: Path,
|
||||
distribution: AbstractWslDistribution,
|
||||
distro: AbstractWslDistribution,
|
||||
linToWinCopy: Boolean = true,
|
||||
filters: WslHashFilters = EMPTY_FILTERS) {
|
||||
filters: WslHashFilters = EMPTY_FILTERS,
|
||||
useStubs: Boolean = false) {
|
||||
LOGGER.info("Sync " + if (linToWinCopy) "$linuxDir -> $windowsDir" else "$windowsDir -> $linuxDir")
|
||||
val win = WindowsFileStorage(windowsDir, distribution, filters)
|
||||
val lin = LinuxFileStorage(linuxDir, distribution, filters)
|
||||
val win = WindowsFileStorage(windowsDir, distro)
|
||||
val lin = LinuxFileStorage(linuxDir, distro)
|
||||
if (linToWinCopy) {
|
||||
WslSync(lin, win)
|
||||
WslSync(lin, win, filters, useStubs)
|
||||
}
|
||||
else {
|
||||
WslSync(win, lin)
|
||||
WslSync(win, lin, filters, useStubs)
|
||||
val execFile = windowsDir.resolve("exec.txt")
|
||||
if (execFile.exists()) {
|
||||
// TODO: Support non top level files
|
||||
for(fileToMarkExec in execFile.readText().split(Regex("\\s+")).map { it.trim() }) {
|
||||
lin.markExec(fileToMarkExec)
|
||||
for (fileToMarkExec in execFile.readText().split(Regex("\\s+")).map { it.trim() }) {
|
||||
lin.markExec(fileToMarkExec)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -64,37 +66,48 @@ class WslSync<SourceFile, DestFile> private constructor(private val source: File
|
||||
init {
|
||||
if (dest.isEmpty()) { //Shortcut: no need to sync anything, just copy everything
|
||||
LOGGER.info("Destination folder is empty, will copy all files")
|
||||
val hashesAndLinks = source.getHashesAndLinks(true)
|
||||
copyFilesInParallel(hashesAndLinks.first.map { it.file })
|
||||
copyAllLinks(hashesAndLinks.second)
|
||||
val syncData = source.calculateSyncData(filters, true, useStubs)
|
||||
copyFilesInParallel(syncData.hashes.map { it.file })
|
||||
syncLinks(syncData.links)
|
||||
syncStubs(syncData.stubs)
|
||||
}
|
||||
else {
|
||||
syncFoldersInternal()
|
||||
}
|
||||
}
|
||||
|
||||
private fun copyAllLinks(toCreate: Map<FilePathRelativeToDir, FilePathRelativeToDir>,
|
||||
current: Map<FilePathRelativeToDir, FilePathRelativeToDir> = emptyMap()) {
|
||||
val linksToCreate = toCreate.filterNot { current[it.key] == it.value }
|
||||
val linksToRemove = current.filterNot { toCreate[it.key] == it.value }.keys
|
||||
private fun syncLinks(sourceLinks: Map<FilePathRelativeToDir, FilePathRelativeToDir>,
|
||||
destStubs: Map<FilePathRelativeToDir, FilePathRelativeToDir> = emptyMap()) {
|
||||
val linksToCreate = sourceLinks.filterNot { destStubs[it.key] == it.value }
|
||||
val linksToRemove = destStubs.filterNot { sourceLinks[it.key] == it.value }.keys
|
||||
|
||||
LOGGER.info("Will create ${linksToCreate.size} links and remove ${linksToRemove.size}")
|
||||
dest.removeLinks(*linksToRemove.toTypedArray())
|
||||
dest.createSymLinks(linksToCreate)
|
||||
}
|
||||
|
||||
private fun syncFoldersInternal() {
|
||||
val sourceHashesFuture = supplyAsync({
|
||||
source.getHashesAndLinks(false)
|
||||
}, ProcessIOExecutorService.INSTANCE)
|
||||
val destHashesFuture = supplyAsync({
|
||||
dest.getHashesAndLinks(false)
|
||||
}, ProcessIOExecutorService.INSTANCE)
|
||||
private fun syncStubs(sourceStubs: Set<FilePathRelativeToDir>,
|
||||
destStubs: Set<FilePathRelativeToDir> = emptySet()) {
|
||||
val stubsToCreate = sourceStubs.minus(destStubs)
|
||||
val stubsToRemove = destStubs.minus(sourceStubs)
|
||||
|
||||
val sourceHashAndLinks = sourceHashesFuture.get()
|
||||
val sourceHashes: MutableMap<FilePathRelativeToDir, WslHashRecord> = sourceHashAndLinks.first.associateBy { it.fileLowerCase }.toMutableMap()
|
||||
val destHashAndLinks = destHashesFuture.get()
|
||||
val destHashes: List<WslHashRecord> = destHashAndLinks.first
|
||||
LOGGER.info("Will create ${stubsToCreate.size} links and remove ${stubsToRemove.size}")
|
||||
dest.createStubs(stubsToCreate)
|
||||
dest.removeFiles(stubsToRemove)
|
||||
}
|
||||
|
||||
private fun syncFoldersInternal() {
|
||||
val sourceSyncDataFuture = supplyAsync({
|
||||
source.calculateSyncData(filters, false, useStubs)
|
||||
}, ProcessIOExecutorService.INSTANCE)
|
||||
val destSyncDataFuture = supplyAsync({
|
||||
dest.calculateSyncData(filters, false, useStubs)
|
||||
}, ProcessIOExecutorService.INSTANCE)
|
||||
|
||||
val sourceSyncData = sourceSyncDataFuture.get()
|
||||
val sourceHashes = sourceSyncData.hashes.associateBy { it.fileLowerCase }.toMutableMap()
|
||||
val destSyncData = destSyncDataFuture.get()
|
||||
val destHashes = destSyncData.hashes
|
||||
|
||||
val destFilesToRemove = ArrayList<FilePathRelativeToDir>(AVG_NUM_FILES)
|
||||
for (destRecord in destHashes) {
|
||||
@@ -113,7 +126,8 @@ class WslSync<SourceFile, DestFile> private constructor(private val source: File
|
||||
|
||||
copyFilesInParallel(sourceHashes.values.map { it.file })
|
||||
dest.removeFiles(destFilesToRemove)
|
||||
copyAllLinks(sourceHashAndLinks.second, destHashAndLinks.second)
|
||||
syncLinks(sourceSyncData.links, destSyncData.links)
|
||||
syncStubs(sourceSyncData.stubs, destSyncData.stubs)
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
package com.intellij.execution.wsl.sync
|
||||
|
||||
/**
|
||||
* [hashes] is a list of [WslHashRecord] (file + hash)
|
||||
* [links] is a map of `source->target` links.
|
||||
* [stubs] is a list of target files to stub.
|
||||
*/
|
||||
data class WslSyncData(val hashes: List<WslHashRecord> = listOf(),
|
||||
val links: Map<FilePathRelativeToDir, FilePathRelativeToDir> = mapOf(),
|
||||
val stubs: Set<FilePathRelativeToDir> = setOf())
|
||||
Reference in New Issue
Block a user