diff --git a/bin/win/aarch64/wslhash b/bin/win/aarch64/wslhash index eb6b435e1f53..a5a2cd7ddca9 100644 Binary files a/bin/win/aarch64/wslhash and b/bin/win/aarch64/wslhash differ diff --git a/bin/win/amd64/wslhash b/bin/win/amd64/wslhash index aba1d48f8f12..d7fcf1eebe26 100644 Binary files a/bin/win/amd64/wslhash and b/bin/win/amd64/wslhash differ diff --git a/native/WslTools/wslhash.c b/native/WslTools/wslhash.c index 99246fc383bf..92613c5c3203 100644 --- a/native/WslTools/wslhash.c +++ b/native/WslTools/wslhash.c @@ -25,6 +25,8 @@ // skip the hash calculation step. // -f FILTER // filters the files using the given FILTER. May be specified multiple times. +// -s +// report files for stubbing e.g. files that exists, but were filtered out (explicitly or implicitly). // // Description: // Calculate hashes (unless `-n`) for all files in the given DIR. @@ -47,10 +49,11 @@ // Filters within each OPERATOR group are processed in the order of appearance in the command line. // // Output format: -// [FILE_PATH]:[HASH] +// [FILE_PATH]\0[HASH] // where HASH is little-endian 8 byte (64 bit) integer -// [FILE_PATH];[LINK_LEN][LINK] +// [LINK_PATH]\1[LINK_LEN][LINK] // where LINK_LEN is 4 byte (32 bit) signed int +// [STUB_PATH]\2 //#define WSLHASH_DEBUG 1 #ifdef WSLHASH_DEBUG @@ -69,6 +72,9 @@ #define FLT_NAME_LEN_MAX (1 + FLT_MATCHER_LEN_MAX + FLT_PATTERN_LEN_MAX + 2) // OPERATOR + MATCHER + PATTERN + delims #define FLT_SCAN_FMT "%c:%" STRINGIFY(FLT_MATCHER_LEN_MAX) "s:%" STRINGIFY(FLT_PATTERN_LEN_MAX) "s" +#define FILE_SEPARATOR 0 +#define LINK_SEPARATOR 1 +#define STUB_SEPARATOR 2 struct wslhash_filter_t { char name[FLT_NAME_LEN_MAX + 1]; // full filter name (OPERATOR:MATCHER:PATTERN). @@ -93,6 +99,7 @@ struct wslhash_options_t { size_t includes_len; int skip_hash; + int report_stubs; }; @@ -148,8 +155,8 @@ static int is_dir(const char *path) { return S_ISDIR(stat_info.st_mode); } -static const char * filename(const char* fpath) { - const char* last_slash = strrchr(fpath, '/'); +static const char *filename(const char *fpath) { + const char *last_slash = strrchr(fpath, '/'); return (last_slash != NULL) ? last_slash + 1 : fpath; } @@ -158,16 +165,20 @@ static int process_file(const char *fpath, const struct stat *sb, int tflag, __attribute__((unused)) struct FTW *ftwbuf) { DEBUG_PRINTF("Processing file: %s\n", fpath); if (tflag != FTW_F && tflag != FTW_SL) { - DEBUG_PRINTF("Skipping file: %s\n", fpath); + DEBUG_PRINTF("Skipping: %s\n", fpath); return 0; // Not a file } - if (tflag == FTW_F && !is_filename_ok(filename(fpath))) { - DEBUG_PRINTF("Excluding file: %s\n", fpath); - return 0; - } const char *fpath_relative = fpath + g_options.root_dir_len + 1; // remove first "/" if (tflag == FTW_F) { - printf("%s:", fpath_relative); + if (!is_filename_ok(filename(fpath))) { + DEBUG_PRINTF("Excluding file: %s\n", fpath); + if (g_options.report_stubs) { + printf("%s%c", fpath_relative, STUB_SEPARATOR); + } + return 0; + } + + printf("%s%c", fpath_relative, FILE_SEPARATOR); if (sb->st_size == 0 || g_options.skip_hash) { // No need to calculate hash for empty file fwrite(EMPTY, sizeof(EMPTY), 1, stdout); @@ -196,7 +207,7 @@ process_file(const char *fpath, const struct stat *sb, int tflag, __attribute__( } else { char real_path[PATH_MAX] = {0}; if (realpath(fpath, real_path) != NULL && is_dir(real_path)) { - printf("%s;", fpath_relative); + printf("%s%c", fpath_relative, LINK_SEPARATOR); const int32_t len = (int32_t) strlen(real_path); fwrite(&len, sizeof(int32_t), 1, stdout); fputs(real_path, stdout); @@ -283,8 +294,11 @@ static void parse_filter(const char *arg) { static void parse_args(int argc, char *argv[]) { int c; - while ((c = getopt(argc, argv, "nf:")) != -1) { + while ((c = getopt(argc, argv, "nsf:")) != -1) { switch (c) { + case 's': + g_options.report_stubs = 1; + break; case 'n': g_options.skip_hash = 1; break; diff --git a/platform/platform-tests/testSrc/com/intellij/execution/wsl/WslSyncTest.kt b/platform/platform-tests/testSrc/com/intellij/execution/wsl/WslSyncTest.kt index 58d9ce029e3f..046ed9ecb9f3 100644 --- a/platform/platform-tests/testSrc/com/intellij/execution/wsl/WslSyncTest.kt +++ b/platform/platform-tests/testSrc/com/intellij/execution/wsl/WslSyncTest.kt @@ -2,7 +2,6 @@ package com.intellij.execution.wsl import com.intellij.execution.wsl.sync.* -import com.intellij.execution.wsl.sync.WslHashFilters.Companion.EMPTY_FILTERS import com.intellij.execution.wsl.sync.WslHashFilters.WslHashFiltersBuilder import com.intellij.execution.wsl.sync.WslHashMatcher.Factory.basename import com.intellij.execution.wsl.sync.WslHashMatcher.Factory.extension @@ -64,7 +63,7 @@ class WslSyncTest(private val linToWin: Boolean) { for (source in sources) { storage.createSymLinks(mapOf(Pair(source, target))) } - return storage.getHashesAndLinks(false).second + return storage.calculateSyncData().links } } @@ -90,13 +89,13 @@ class WslSyncTest(private val linToWin: Boolean) { if (linToWin) { val dir = linuxDirRule.dir wslRule.wsl.executeOnWsl(1000, "mkdir", "${dir}/target") - storage = LinuxFileStorage(dir, wslRule.wsl, EMPTY_FILTERS) + storage = LinuxFileStorage(dir, wslRule.wsl) } else { val dir = winDirRule.newDirectoryPath() val targetDir = dir.resolve("target") targetDir.createDirectory() - storage = WindowsFileStorage(dir, wslRule.wsl, EMPTY_FILTERS) + storage = WindowsFileStorage(dir, wslRule.wsl) } val links = createAndGetLinks(storage, FilePathRelativeToDir("target"), *sources) @@ -112,8 +111,8 @@ class WslSyncTest(private val linToWin: Boolean) { val to: FileStorage<*, *> val winRoot = winDirRule.newDirectoryPath() val linRoot = linuxDirRule.dir - val win = WindowsFileStorage(winRoot, wslRule.wsl, EMPTY_FILTERS) - val lin = LinuxFileStorage(linRoot, wslRule.wsl, EMPTY_FILTERS) + val win = WindowsFileStorage(winRoot, wslRule.wsl) + val lin = LinuxFileStorage(linRoot, wslRule.wsl) if (!linToWin) { winRoot.resolve("target dir").createDirectory().resolve("file.txt").createFile() winRoot.resolve("dir_to_ignore").createDirectory() @@ -132,13 +131,13 @@ class WslSyncTest(private val linToWin: Boolean) { } to.createSymLinks(mapOf(Pair(FilePathRelativeToDir("dir_to_ignore/foo"), FilePathRelativeToDir("target dir")))) WslSync.syncWslFolders(linRoot, winRoot, wslRule.wsl, linToWinCopy = linToWin) - val links = to.getHashesAndLinks(false).second + val links = to.calculateSyncData().links for (source in sources) { Assert.assertEquals("target dir", links[source]?.asWindowsPath) } to.createSymLinks(mapOf(Pair(FilePathRelativeToDir("remove_me"), FilePathRelativeToDir("target dir")))) WslSync.syncWslFolders(linRoot, winRoot, wslRule.wsl, linToWinCopy = linToWin) - Assert.assertEquals(null, to.getHashesAndLinks(false).second[FilePathRelativeToDir("remove_me")]) + Assert.assertEquals(null, to.calculateSyncData().links[FilePathRelativeToDir("remove_me")]) for (source in sources) { Assert.assertEquals("target dir", links[source]?.asWindowsPath) } @@ -252,7 +251,8 @@ class WslSyncTest(private val linToWin: Boolean) { .exclude(*extensions("dll", "gz", "zip"), basename("debug"), fullname("idea.log")) - .build() + .build(), + false ) } @@ -266,6 +266,7 @@ class WslSyncTest(private val linToWin: Boolean) { basename("debug"), fullname("idea.log")) .build(), + false ) } @@ -281,12 +282,44 @@ class WslSyncTest(private val linToWin: Boolean) { .include(extension("tar.gz"), fullname("debug.in")) .build(), + false + ) + } + + @Test + fun syncWithExcludesAndStubs() { + doSyncAndAssertFilePresence( + setOf(), + setOf("файл.dll", "файл.gz", "файл.zip", "debug", "debug.out", "idea.log", + "файл.py", "файл.tar.gz", "файл.java", "файл-dll", "ddebug", "debug.in", "idea.log.bck"), + WslHashFiltersBuilder() + .exclude(*extensions("dll", "gz", "zip"), + basename("debug"), + fullname("idea.log")) + .build(), + true + ) + } + + @Test + fun syncWithIncludesAndStubs() { + doSyncAndAssertFilePresence( + setOf(), + setOf("файл.dll", "файл.gz", "файл.zip", "debug", "debug.out", "idea.log", + "файл.py", "файл.tar.gz", "файл.java", "файл-dll", "ddebug", "debug.in", "idea.log.bck"), + WslHashFiltersBuilder() + .include(*extensions("dll", "gz", "zip"), + basename("debug"), + fullname("idea.log")) + .build(), + true ) } private fun doSyncAndAssertFilePresence(fileNamesToIgnore: Set, fileNamesToSync: Set, - filters: WslHashFilters) { + filters: WslHashFilters, + useStubs: Boolean) { val windowsDir = winDirRule.newDirectoryPath() val allFileNames = fileNamesToSync + fileNamesToIgnore @@ -297,7 +330,7 @@ class WslSyncTest(private val linToWin: Boolean) { srcDir.resolve(fileName).writeText("hello $fileName") } - WslSync.syncWslFolders(linuxDirRule.dir, windowsDir, wslRule.wsl, linToWin, filters) + WslSync.syncWslFolders(linuxDirRule.dir, windowsDir, wslRule.wsl, linToWin, filters, useStubs) for (fileName in allFileNames) { val file = dstDir.resolve(fileName) @@ -306,6 +339,9 @@ class WslSyncTest(private val linToWin: Boolean) { } else { Assert.assertTrue("File ${file} must be copied", file.exists()) + if (useStubs && !filters.isFileNameOk(file.fileName.toString())) { + Assert.assertEquals("File ${file} must be stubbed", "", file.readText().trim()) + } } } Assert.assertEquals("Not all files synced", fileNamesToSync.size, dstDir.toFile().list()!!.size) diff --git a/platform/wsl-impl/src/com/intellij/execution/wsl/sync/FileStorage.kt b/platform/wsl-impl/src/com/intellij/execution/wsl/sync/FileStorage.kt index 747776375722..acaa2588ac91 100644 --- a/platform/wsl-impl/src/com/intellij/execution/wsl/sync/FileStorage.kt +++ b/platform/wsl-impl/src/com/intellij/execution/wsl/sync/FileStorage.kt @@ -2,6 +2,7 @@ package com.intellij.execution.wsl.sync import com.intellij.execution.wsl.AbstractWslDistribution +import com.intellij.execution.wsl.sync.WslHashFilters.Companion.EMPTY_FILTERS /** @@ -12,8 +13,7 @@ import com.intellij.execution.wsl.AbstractWslDistribution */ abstract class FileStorage( protected val dir: MyFileType, - protected val distro: AbstractWslDistribution, - protected val filters: WslHashFilters + protected val distro: AbstractWslDistribution ) { /** @@ -22,11 +22,14 @@ abstract class FileStorage( abstract fun createSymLinks(links: Map) /** - * List of [WslHashRecord] (file + hash) and map of `source->target` links. - * [skipHashCalculation] saves time by skipping hash (hence [WslHashRecord.hash] is 0). + * [filters] determine which files to process. + * [skipHash] saves time by skipping hash (hence each [WslHashRecord.hash] in [WslSyncData.hashes] is 0). * Such records can't be used for sync, but only to copy all files + * [useStubs] specifies whether files for stubbing will be reported. */ - abstract fun getHashesAndLinks(skipHashCalculation: Boolean): Pair, Map> + abstract fun calculateSyncData(filters: WslHashFilters = EMPTY_FILTERS, + skipHash: Boolean = false, + useStubs: Boolean = false): WslSyncData /** * is [dir] empty @@ -35,6 +38,7 @@ abstract class FileStorage( abstract fun removeFiles(filesToRemove: Collection) abstract fun createTempFile(): MyFileType abstract fun removeTempFile(file: MyFileType) + abstract fun createStubs(files: Collection) /** * tar [files] and copy to [destTar] diff --git a/platform/wsl-impl/src/com/intellij/execution/wsl/sync/LinuxFileStorage.kt b/platform/wsl-impl/src/com/intellij/execution/wsl/sync/LinuxFileStorage.kt index f26d3623ade2..bc0d936d2d57 100644 --- a/platform/wsl-impl/src/com/intellij/execution/wsl/sync/LinuxFileStorage.kt +++ b/platform/wsl-impl/src/com/intellij/execution/wsl/sync/LinuxFileStorage.kt @@ -6,8 +6,8 @@ import com.intellij.execution.processTools.getBareExecutionResult import com.intellij.execution.processTools.getResultStdoutStr import com.intellij.execution.wsl.* import com.intellij.openapi.diagnostic.Logger +import com.intellij.openapi.util.Ref import com.intellij.util.TimeoutUtil -import com.intellij.util.containers.ContainerUtil.append import com.intellij.util.io.delete import kotlinx.coroutines.runBlocking import java.io.InputStream @@ -19,14 +19,15 @@ import kotlin.io.path.writeText private val LOGGER = Logger.getInstance(LinuxFileStorage::class.java) -class LinuxFileStorage(dir: LinuxFilePath, distro: AbstractWslDistribution, filters: WslHashFilters) - : FileStorage(dir.trimEnd('/') + '/', distro, filters) { +class LinuxFileStorage(dir: LinuxFilePath, distro: AbstractWslDistribution) + : FileStorage(dir.trimEnd('/') + '/', distro) { // Linux side only works with UTF of 7-bit ASCII which is also supported by UTF and WSL doesn't support other charsets - private val CHARSET = Charsets.UTF_8 - private val FILE_SEPARATOR = CHARSET.encode(":").get() - private val LINK_SEPARATOR = CHARSET.encode(";").get() + + private val FILE_SEPARATOR: Byte = 0 + private val LINK_SEPARATOR: Byte = 1 + private val STUB_SEPARATOR: Byte = 2 override fun createSymLinks(links: Map) { val script = createTmpWinFile(distro) @@ -39,27 +40,38 @@ class LinuxFileStorage(dir: LinuxFilePath, distro: AbstractWslDistribution, filt script.first.delete() } - override fun getHashesAndLinks(skipHashCalculation: Boolean): Pair, Map> { - val hashes = ArrayList(AVG_NUM_FILES) - val links = HashMap(AVG_NUM_FILES) + override fun calculateSyncData(filters: WslHashFilters, skipHash: Boolean, useStubs: Boolean): WslSyncData { + val dataRef = Ref() val time = TimeoutUtil.measureExecutionTime { - val wslHashArgs = if (skipHashCalculation) append(filters.toArgs(), "-n", dir) else append(filters.toArgs(), dir) + val wslHashArgs = listOfNotNull(if (skipHash) "-n" else null, + if (useStubs) "-s" else null, + *filters.toArgs().toTypedArray(), + dir) val tool = distro.getTool("wslhash", *wslHashArgs.toTypedArray()) val process = tool.createProcess() process.inputStream.use { - val hashesAndLinks = getHashesInternal(it) - hashes += hashesAndLinks.first - links += hashesAndLinks.second + dataRef.set(calculateSyncDataInternal(it)) } runBlocking { process.getResultStdoutStr() } } LOGGER.info("Linux files calculated in $time") - return Pair(hashes, links) + return dataRef.get() } - override fun createTempFile(): String = distro.runCommand("mktemp", "-u").getOrThrow() + override fun createStubs(files: Collection) { + val script = createTmpWinFile(distro) + try { + val scriptContent = files.joinToString("\n") { "mkdir -p \"$(dirname ${it.escapedWithDir})\" && touch ${it.escapedWithDir}" } + script.first.writeText(scriptContent) + runBlocking { distro.createProcess("sh", script.second).getBareExecutionResult() } + } + finally { + script.first.delete() + } + } + override fun removeLinks(vararg linksToRemove: FilePathRelativeToDir) { this.removeFiles(linksToRemove.asList()) } @@ -113,17 +125,19 @@ class LinuxFileStorage(dir: LinuxFilePath, distro: AbstractWslDistribution, filt } /** - * Read `wslhash` stdout and return map of [file->hash] + * Parse output from `wslhash` and return [WslSyncData]. */ - private fun getHashesInternal(toolStdout: InputStream): Pair, Map> { + private fun calculateSyncDataInternal(toolStdout: InputStream): WslSyncData { val hashes = ArrayList(AVG_NUM_FILES) - val links = HashMap(AVG_NUM_FILES) + val links = mutableMapOf() + val stubs = mutableSetOf() val fileOutput = ByteBuffer.wrap(toolStdout.readAllBytes()).order(ByteOrder.LITTLE_ENDIAN) - // See wslhash.c: format is the following: [file_path]:[hash]. - // Hash is little-endian 8 byte (64 bit) integer - // or [file_path];[link_len][link] where link_len is 4 byte signed int - + // See wslhash.c. + // Output format is the following: + // [file_path]\0[hash], where hash is little-endian 8 byte (64 bit) integer + // [link_path]\1[link_len][link], where link_len is 4 byte signed int + // [stub_path]\2 var fileStarted = 0 val outputLimit = fileOutput.limit() while (fileOutput.position() < outputLimit) { @@ -149,9 +163,17 @@ class LinuxFileStorage(dir: LinuxFilePath, distro: AbstractWslDistribution, filt } fileStarted = prevPos + length } + STUB_SEPARATOR -> { + val prevPos = fileOutput.position() + // 1 = separator + val name = CHARSET.decode(fileOutput.limit(prevPos - 1).position(fileStarted)).toString() + fileOutput.limit(outputLimit).position(prevPos) + stubs += FilePathRelativeToDir(name) + fileStarted = prevPos + } } } - return Pair(hashes, links) + return WslSyncData(hashes, links, stubs) } private val FilePathRelativeToDir.escapedWithDir: String get() = escapePath(dir + asUnixPath) diff --git a/platform/wsl-impl/src/com/intellij/execution/wsl/sync/WindowsFileStorage.kt b/platform/wsl-impl/src/com/intellij/execution/wsl/sync/WindowsFileStorage.kt index 0533a0d3980d..289f119f4848 100644 --- a/platform/wsl-impl/src/com/intellij/execution/wsl/sync/WindowsFileStorage.kt +++ b/platform/wsl-impl/src/com/intellij/execution/wsl/sync/WindowsFileStorage.kt @@ -5,11 +5,11 @@ import com.intellij.execution.configurations.GeneralCommandLine import com.intellij.execution.util.ExecUtil import com.intellij.execution.wsl.AbstractWslDistribution import com.intellij.openapi.diagnostic.Logger -import com.intellij.openapi.diagnostic.thisLogger import com.intellij.openapi.util.io.FileSystemUtil import com.intellij.util.TimeoutUtil import com.intellij.util.io.* import com.intellij.util.system.CpuArch +import net.jpountz.xxhash.XXHash64 import net.jpountz.xxhash.XXHashFactory import java.io.IOException import java.nio.channels.FileChannel @@ -24,27 +24,55 @@ private val LOGGER = Logger.getInstance(WindowsFileStorage::class.java) private class MyFileVisitor(private val filters: WslHashFilters, private val rootDir: Path, - private val processFile: (relativeToDir: FilePathRelativeToDir, file: Path, attrs: BasicFileAttributes) -> Unit) : SimpleFileVisitor() { - private val dirLinksInt: MutableMap = mutableMapOf() - val dirLinks: Map get() = dirLinksInt + private val hashTool: XXHash64, + private val skipHash: Boolean, + private val useStubs: Boolean) : SimpleFileVisitor() { + + private val _hashes: MutableList = ArrayList(AVG_NUM_FILES) + private val _dirLinks: MutableMap = mutableMapOf() + private val _stubs: MutableSet = mutableSetOf() + + val hashes: List get() = _hashes + val dirLinks: Map get() = _dirLinks + val stubs: Set get() = _stubs + override fun postVisitDirectory(dir: Path?, exc: IOException?): FileVisitResult { return super.postVisitDirectory(dir, exc) } override fun visitFile(path: Path, attrs: BasicFileAttributes): FileVisitResult { if (!(attrs.isRegularFile)) return FileVisitResult.CONTINUE - if (!filters.isFileNameOk(path.fileName.toString())) { - return FileVisitResult.CONTINUE - } processFile(FilePathRelativeToDir(rootDir.relativize(path).joinToString("/").lowercase()), path, attrs) return FileVisitResult.CONTINUE } + fun processFile(relativeToDir: FilePathRelativeToDir, file: Path, attrs: BasicFileAttributes) { + if (!filters.isFileNameOk(file.fileName.toString())) { + if (useStubs) { + _stubs.add(relativeToDir) + } + } + else if (skipHash || attrs.size() == 0L) { // Empty file's hash is 0, see wslhash.c + _hashes.add(WslHashRecord(relativeToDir, 0)) + } + else { // Map file and read hash + FileChannel.open(file, StandardOpenOption.READ).use { + val buf = it.map(FileChannel.MapMode.READ_ONLY, 0, attrs.size()) + try { + _hashes.add(WslHashRecord(relativeToDir, hashTool.hash(buf, 0))) // Seed 0 is default, see wslhash.c + } + finally { + ByteBufferUtil.cleanBuffer(buf) // Unmap file: can't overwrite mapped file + } + } + } + } + override fun preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult = if (FileSystemUtil.getAttributes( dir.toFile())?.isSymLink == true) { val target = FileSystemUtil.resolveSymLink(dir.toFile())?.let { rootDir.resolve(it) } if (target != null && target.isDirectory() && target.startsWith(rootDir)) { - dirLinksInt[FilePathRelativeToDir(rootDir.relativize(dir).toString())] = FilePathRelativeToDir(rootDir.relativize(target).toString()) + _dirLinks[FilePathRelativeToDir(rootDir.relativize(dir).toString())] = FilePathRelativeToDir(rootDir.relativize(target).toString()) } FileVisitResult.SKIP_SUBTREE } @@ -54,8 +82,7 @@ private class MyFileVisitor(private val filters: WslHashFilters, } class WindowsFileStorage(dir: Path, - distro: AbstractWslDistribution, - filters: WslHashFilters) : FileStorage(dir, distro, filters) { + distro: AbstractWslDistribution) : FileStorage(dir, distro) { private fun runCommand(vararg commands: String) { val cmd = arrayOf("cmd", "/c", *commands) ExecUtil.execAndGetOutput(GeneralCommandLine(*cmd)).let { @@ -76,36 +103,21 @@ class WindowsFileStorage(dir: Path, } } - override fun getHashesAndLinks(skipHashCalculation: Boolean): Pair, Map> { - val result = ArrayList(AVG_NUM_FILES) + override fun calculateSyncData(filters: WslHashFilters, skipHash: Boolean, useStubs: Boolean): WslSyncData { val arch = System.getProperty("os.arch") val useNativeHash = CpuArch.CURRENT == CpuArch.X86_64 - thisLogger().info("Arch $arch, using native hash: $useNativeHash") + LOGGER.info("Arch $arch, using native hash: $useNativeHash") val hashTool = if (useNativeHash) XXHashFactory.nativeInstance().hash64() else XXHashFactory.safeInstance().hash64() // Native hash can access direct (mapped) buffer a little-bit faster - val visitor = MyFileVisitor(filters, dir) { relativeToDir: FilePathRelativeToDir, file: Path, attrs: BasicFileAttributes -> - if (skipHashCalculation || attrs.size() == 0L) { // Empty file's hash is 0, see wslhash.c - result.add(WslHashRecord(relativeToDir, 0)) - } - else { // Map file and read hash - FileChannel.open(file, StandardOpenOption.READ).use { - val buf = it.map(FileChannel.MapMode.READ_ONLY, 0, attrs.size()) - try { - result.add(WslHashRecord(relativeToDir, hashTool.hash(buf, 0))) // Seed 0 is default, see wslhash.c - } - finally { - ByteBufferUtil.cleanBuffer(buf) // Unmap file: can't overwrite mapped file - } - } - } - } + val visitor = MyFileVisitor(filters, dir, hashTool, skipHash, useStubs) val time = TimeoutUtil.measureExecutionTime { Files.walkFileTree(dir, visitor) } LOGGER.info("Windows files calculated in $time") - return Pair(result, visitor.dirLinks) + return WslSyncData(visitor.hashes, visitor.dirLinks, visitor.stubs) } override fun isEmpty(): Boolean = dir.notExists() || dir.listDirectoryEntries().isEmpty() + override fun removeFiles(filesToRemove: Collection) { if (filesToRemove.isEmpty()) return for (file in filesToRemove) { @@ -117,6 +129,16 @@ class WindowsFileStorage(dir: Path, } override fun createTempFile(): Path = createTmpWinFile(distro).first + + override fun createStubs(files: Collection) { + for (file in files) { + val filePath = dir.resolve(file.asWindowsPath) + if (!filePath.exists()) { + filePath.createFile() + } + } + } + override fun removeLinks(vararg linksToRemove: FilePathRelativeToDir) { for (link in linksToRemove) { runCommand("rmdir", dir.resolve(link.asWindowsPath).toString()) diff --git a/platform/wsl-impl/src/com/intellij/execution/wsl/sync/WslSync.kt b/platform/wsl-impl/src/com/intellij/execution/wsl/sync/WslSync.kt index f53a5df0f1dd..7f161503850f 100644 --- a/platform/wsl-impl/src/com/intellij/execution/wsl/sync/WslSync.kt +++ b/platform/wsl-impl/src/com/intellij/execution/wsl/sync/WslSync.kt @@ -25,36 +25,38 @@ private const val MIN_CHUNK_SIZE = 1000 private val LOGGER = Logger.getInstance(WslSync::class.java) class WslSync private constructor(private val source: FileStorage, - private val dest: FileStorage) { - + private val dest: FileStorage, + private val filters: WslHashFilters, + private val useStubs: Boolean) { companion object { /** - * Makes [windowsDir] reflect [linuxDir] (or vice versa depending on [linToWinCopy]) on [distribution] much like rsync. - * Redundant files deleted, new/changed files copied. - * Set [onlyExtensions] if you only care about certain extensions. - * Direction depends on [linToWinCopy] + * Synchronizes the given [windowsDir] and [linuxDir] (inside [distro]). + * [linToWinCopy] determines the sync direction. + * [filters] allow you to specify which files to include/exclude. + * [useStubs] dictates whether empty stubs should be created for filtered out files. */ @JvmOverloads fun syncWslFolders(linuxDir: String, windowsDir: Path, - distribution: AbstractWslDistribution, + distro: AbstractWslDistribution, linToWinCopy: Boolean = true, - filters: WslHashFilters = EMPTY_FILTERS) { + filters: WslHashFilters = EMPTY_FILTERS, + useStubs: Boolean = false) { LOGGER.info("Sync " + if (linToWinCopy) "$linuxDir -> $windowsDir" else "$windowsDir -> $linuxDir") - val win = WindowsFileStorage(windowsDir, distribution, filters) - val lin = LinuxFileStorage(linuxDir, distribution, filters) + val win = WindowsFileStorage(windowsDir, distro) + val lin = LinuxFileStorage(linuxDir, distro) if (linToWinCopy) { - WslSync(lin, win) + WslSync(lin, win, filters, useStubs) } else { - WslSync(win, lin) + WslSync(win, lin, filters, useStubs) val execFile = windowsDir.resolve("exec.txt") if (execFile.exists()) { // TODO: Support non top level files - for(fileToMarkExec in execFile.readText().split(Regex("\\s+")).map { it.trim() }) { - lin.markExec(fileToMarkExec) + for (fileToMarkExec in execFile.readText().split(Regex("\\s+")).map { it.trim() }) { + lin.markExec(fileToMarkExec) } } } @@ -64,37 +66,48 @@ class WslSync private constructor(private val source: File init { if (dest.isEmpty()) { //Shortcut: no need to sync anything, just copy everything LOGGER.info("Destination folder is empty, will copy all files") - val hashesAndLinks = source.getHashesAndLinks(true) - copyFilesInParallel(hashesAndLinks.first.map { it.file }) - copyAllLinks(hashesAndLinks.second) + val syncData = source.calculateSyncData(filters, true, useStubs) + copyFilesInParallel(syncData.hashes.map { it.file }) + syncLinks(syncData.links) + syncStubs(syncData.stubs) } else { syncFoldersInternal() } } - private fun copyAllLinks(toCreate: Map, - current: Map = emptyMap()) { - val linksToCreate = toCreate.filterNot { current[it.key] == it.value } - val linksToRemove = current.filterNot { toCreate[it.key] == it.value }.keys + private fun syncLinks(sourceLinks: Map, + destStubs: Map = emptyMap()) { + val linksToCreate = sourceLinks.filterNot { destStubs[it.key] == it.value } + val linksToRemove = destStubs.filterNot { sourceLinks[it.key] == it.value }.keys LOGGER.info("Will create ${linksToCreate.size} links and remove ${linksToRemove.size}") dest.removeLinks(*linksToRemove.toTypedArray()) dest.createSymLinks(linksToCreate) } - private fun syncFoldersInternal() { - val sourceHashesFuture = supplyAsync({ - source.getHashesAndLinks(false) - }, ProcessIOExecutorService.INSTANCE) - val destHashesFuture = supplyAsync({ - dest.getHashesAndLinks(false) - }, ProcessIOExecutorService.INSTANCE) + private fun syncStubs(sourceStubs: Set, + destStubs: Set = emptySet()) { + val stubsToCreate = sourceStubs.minus(destStubs) + val stubsToRemove = destStubs.minus(sourceStubs) - val sourceHashAndLinks = sourceHashesFuture.get() - val sourceHashes: MutableMap = sourceHashAndLinks.first.associateBy { it.fileLowerCase }.toMutableMap() - val destHashAndLinks = destHashesFuture.get() - val destHashes: List = destHashAndLinks.first + LOGGER.info("Will create ${stubsToCreate.size} links and remove ${stubsToRemove.size}") + dest.createStubs(stubsToCreate) + dest.removeFiles(stubsToRemove) + } + + private fun syncFoldersInternal() { + val sourceSyncDataFuture = supplyAsync({ + source.calculateSyncData(filters, false, useStubs) + }, ProcessIOExecutorService.INSTANCE) + val destSyncDataFuture = supplyAsync({ + dest.calculateSyncData(filters, false, useStubs) + }, ProcessIOExecutorService.INSTANCE) + + val sourceSyncData = sourceSyncDataFuture.get() + val sourceHashes = sourceSyncData.hashes.associateBy { it.fileLowerCase }.toMutableMap() + val destSyncData = destSyncDataFuture.get() + val destHashes = destSyncData.hashes val destFilesToRemove = ArrayList(AVG_NUM_FILES) for (destRecord in destHashes) { @@ -113,7 +126,8 @@ class WslSync private constructor(private val source: File copyFilesInParallel(sourceHashes.values.map { it.file }) dest.removeFiles(destFilesToRemove) - copyAllLinks(sourceHashAndLinks.second, destHashAndLinks.second) + syncLinks(sourceSyncData.links, destSyncData.links) + syncStubs(sourceSyncData.stubs, destSyncData.stubs) } /** diff --git a/platform/wsl-impl/src/com/intellij/execution/wsl/sync/WslSyncData.kt b/platform/wsl-impl/src/com/intellij/execution/wsl/sync/WslSyncData.kt new file mode 100644 index 000000000000..c3b01e159313 --- /dev/null +++ b/platform/wsl-impl/src/com/intellij/execution/wsl/sync/WslSyncData.kt @@ -0,0 +1,11 @@ +// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.execution.wsl.sync + +/** + * [hashes] is a list of [WslHashRecord] (file + hash) + * [links] is a map of `source->target` links. + * [stubs] is a list of target files to stub. + */ +data class WslSyncData(val hashes: List = listOf(), + val links: Map = mapOf(), + val stubs: Set = setOf())