PY-40581: WSL: Do not copy editable package sources to `remote_sources`.

All python packages from any remote SDK (WSL included) must be copied to ``remote_sources`` since indexing WSL (as eny remote SDK) from Windows may be slow.

This should be done each time package is installed/upgraded.

But when you have editable package, there is no reason nor possibility to copy it on each change.

WSL provides access to its filesystem using VFS (see RemoteTargetEnvironmentWithLocalVfs) and we use it.

``remote_sync.py`` reports editable package as root. But we know that it resides in module content root, so we exclude it from copying, but add it as simple mapping and content root
instead.

With this change editable packages resolved not to ``remote_sources`` but to module content
root instead.

GitOrigin-RevId: 1557950f0b1ba588e5ef7e6a767c3d9c1d85ee28

platform/lang-api/src/com/intellij/openapi/roots/WatchedRootsProvider.java

@@ -9,6 +9,11 @@ import java.util.Set;
 
 public interface WatchedRootsProvider {
   /**
+   * While implementing the method make sure that {@link com.intellij.openapi.vfs.newvfs.impl.VirtualDirectoryImpl} for each of roots
+   * has all children loaded recursively. You can do that explicitly by loading VFS and calling {@link com.intellij.openapi.vfs.newvfs.impl.VirtualDirectoryImpl#getChildren()}.
+   * This is implicitly required by the file watcher to fire events for the changing descendants of a directory.
+   * The indicator that all children are loaded is that {@link com.intellij.openapi.vfs.newvfs.impl.VirtualDirectoryImpl#allChildrenLoaded()} returns true
+   *
    * @return paths which should be monitored via {@link LocalFileSystem#addRootToWatch(String, boolean)}.
    * @see LocalFileSystem
    */

python/helpers/remote_sync.py

@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 import argparse
-import contextlib
 import json
 import os
 import sys
@@ -114,8 +113,8 @@ def open_zip(zip_path, mode):
 
 
 class RemoteSync(object):
-    def __init__(self, roots, output_dir, state_json=None):
-        self.roots = self.sanitize_roots(roots)
+    def __init__(self, roots, output_dir, state_json=None, project_roots=()):
+        self.roots, self.skipped_roots = self.sanitize_roots(roots, project_roots)
         self.output_dir = self.sanitize_output_dir(output_dir)
         self.in_state_json = state_json
         self._name_counts = defaultdict(int)
@@ -129,6 +128,8 @@ class RemoteSync(object):
             new_state = self.collect_sources_in_root(root, zip_path, old_state)
             out_state_json['roots'].append(new_state)
 
+        if self.skipped_roots:
+            out_state_json['skipped_roots'] = self.skipped_roots
         dump_json(out_state_json, os.path.join(self.output_dir, '.state.json'))
 
     def collect_sources_in_root(self, root, zip_path, old_state):
@@ -181,8 +182,9 @@ class RemoteSync(object):
     def sanitize_path(path):
         return os.path.normpath(_decode_path(path))
 
-    def sanitize_roots(self, roots):
+    def sanitize_roots(self, roots, project_roots):
         result = []
+        skipped_roots = []
         for root in roots:
             normalized = self.sanitize_path(root)
             if (not os.path.isdir(normalized) or
@@ -190,8 +192,15 @@ class RemoteSync(object):
                     not path_is_under(normalized, sys.prefix) and
                     not path_is_under(normalized, _helpers_test_root)):
                 continue
+            if any(path_is_under(normalized, p) for p in project_roots) \
+                    and not path_is_under(normalized, sys.prefix):
+                # Root is available locally and not under sys.prefix (hence not .venv)
+                # Must be editable package on the target (for example, WSL or SSH)
+                # Do not copy it, report instead
+                skipped_roots.append(normalized)
+                continue
             result.append(normalized)
-        return result
+        return result, skipped_roots
 
     def sanitize_output_dir(self, output_dir):
         normalized = self.sanitize_path(output_dir)
@@ -279,6 +288,9 @@ def main():
                         help='Directory to collect ZIP archives with sources into.')
     parser.add_argument('--state-file', type=argparse.FileType('rb'),
                         help='State of the last synchronization in JSON.')
+    parser.add_argument('--project-roots', type=ArgparseTypes.path,
+                        nargs='+', default=(),
+                        help='Exclude roots from copying, report them to stdout instead')
     decoded_sys_path = [_decode_path(p) for p in sys.path]
     parser.add_argument('--roots', metavar='PATH_LIST', dest='roots',
                         type=ArgparseTypes.path_list, default=decoded_sys_path,
@@ -299,7 +311,8 @@ def main():
 
     RemoteSync(roots=args.roots,
                output_dir=args.output_dir,
-               state_json=state_json).run()
+               state_json=state_json,
+               project_roots=set(args.project_roots)).run()
 
 
 if __name__ == '__main__':

python/helpers/tests/test_remote_sync.py

@@ -50,6 +50,35 @@ class RemoteSyncTest(HelpersTestCase):
         root3.zip
         """)
 
+    def test_project_root_excluded(self):
+        project_root = os.path.join(self.test_data_dir, 'project_root')
+        self.collect_sources(
+            ['root1', 'root2', 'project_root'],
+            project_roots={project_root}
+        )
+
+        expected_json = {'roots': [{'invalid_entries': [],
+                                    'path': 'root1',
+                                    'valid_entries': {
+                                        '__init__.py': {
+                                            'mtime': self.mtime('root1/__init__.py')}},
+                                    'zip_name': 'root1.zip'},
+                                   {'invalid_entries': [],
+                                    'path': 'root2',
+                                    'valid_entries': {
+                                        '__init__.py': {
+                                            'mtime': self.mtime('root2/__init__.py')}},
+                                    'zip_name': 'root2.zip'}],
+                         'skipped_roots': [project_root]}
+        self.assertJsonEquals(self.resolve_in_temp_dir('.state.json'),
+                              expected_json)
+
+        self.assertDirLayoutEquals(self.temp_dir, """
+        .state.json
+        root1.zip
+        root2.zip
+        """)
+
     def test_roots_with_identical_name(self):
         self.collect_sources(['root', 'dir/root'])
         self.assertDirLayoutEquals(self.temp_dir, """
@@ -527,7 +556,8 @@ class RemoteSyncTest(HelpersTestCase):
                                          universal_newlines=True)
         self.assertIn('usage: remote_sync.py', output)
 
-    def collect_sources(self, roots_inside_test_data, output_dir=None, state_json=None):
+    def collect_sources(self, roots_inside_test_data, output_dir=None, state_json=None,
+                        project_roots=()):
         if output_dir is None:
             output_dir = self.temp_dir
         self.assertTrue(
@@ -535,7 +565,8 @@ class RemoteSyncTest(HelpersTestCase):
             'Test data directory {} does not exist'.format(self.test_data_dir)
         )
         roots = [self.resolve_in_test_data(r) for r in roots_inside_test_data]
-        rsync = RemoteSync(roots, output_dir, state_json)
+        rsync = RemoteSync(roots, output_dir, state_json,
+                           [self.resolve_in_temp_dir(r) for r in project_roots])
         rsync._test_root = self.test_data_dir
         rsync.run()
 

python/src/META-INF/python-core-common.xml

@@ -28,6 +28,7 @@
   <extensions defaultExtensionNs="com.intellij">
     <iconMapper mappingFile="PythonIconMappings.json"/>
     <library.type implementation="com.jetbrains.python.library.PythonLibraryType"/>
+    <roots.watchedRootsProvider implementation="com.jetbrains.python.target.targetWithVfs.TargetVfsWatchedRootsProvider"/>
     <renameHandler implementation="com.jetbrains.python.magicLiteral.PyMagicLiteralRenameHandler"/>
     <nameSuggestionProvider implementation="com.jetbrains.python.refactoring.PyNameSuggestionProvider"/>
     <methodNavigationOffsetProvider implementation="com.jetbrains.python.codeInsight.PyMethodNavigationOffsetProvider"/>

python/src/com/jetbrains/python/packaging/PyPackageUtil.java

@@ -32,6 +32,7 @@ import com.intellij.psi.PsiFile;
 import com.intellij.psi.PsiManager;
 import com.intellij.psi.ResolveResult;
 import com.intellij.serviceContainer.AlreadyDisposedException;
+import com.intellij.util.containers.ContainerUtil;
 import com.jetbrains.python.PyBundle;
 import com.jetbrains.python.PyNames;
 import com.jetbrains.python.PyPsiPackageUtil;
@@ -43,8 +44,11 @@ import com.jetbrains.python.psi.impl.PyPsiUtils;
 import com.jetbrains.python.psi.resolve.PyResolveContext;
 import com.jetbrains.python.psi.types.TypeEvalContext;
 import com.jetbrains.python.remote.PyCredentialsContribution;
+import com.jetbrains.python.run.PythonInterpreterTargetEnvironmentFactory;
 import com.jetbrains.python.sdk.CredentialsTypeExChecker;
+import com.jetbrains.python.sdk.PySdkExtKt;
 import com.jetbrains.python.sdk.PythonSdkUtil;
+import com.jetbrains.python.target.PyTargetAwareAdditionalData;
 import one.util.streamex.StreamEx;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
@@ -557,6 +561,16 @@ public final class PyPackageUtil {
   @NotNull
   private static Set<VirtualFile> getPackagingAwareSdkRoots(@NotNull Sdk sdk) {
     final Set<VirtualFile> result = Sets.newHashSet(sdk.getRootProvider().getFiles(OrderRootType.CLASSES));
+    var targetAdditionalData = PySdkExtKt.getTargetAdditionalData(sdk);
+    if (targetAdditionalData != null) {
+      // For targets that support VFS we are interested not only in local dirs, but also for VFS on target
+      // When user changes something on WSL FS for example, we still need to trigger path updates
+      for (var remoteSourceToVfs : getRemoteSourceToVfsMapping(targetAdditionalData).entrySet()) {
+        if (result.contains(remoteSourceToVfs.getKey())) {
+          result.add(remoteSourceToVfs.getValue());
+        }
+      }
+    }
     final String skeletonsPath = PythonSdkUtil.getSkeletonsPath(PathManager.getSystemPath(), sdk.getHomePath());
     final VirtualFile skeletonsRoot = LocalFileSystem.getInstance().findFileByPath(skeletonsPath);
     result.removeIf(vf -> vf.equals(skeletonsRoot) ||
@@ -564,4 +578,29 @@ public final class PyPackageUtil {
                           PyTypeShed.INSTANCE.isInside(vf));
     return result;
   }
+
+  /**
+   * If target provides access to its FS using VFS, rerun all mappings in format [path-to-"remote_sources" -> vfs-on-target]
+   * i.e: "c:\remote_sources -> \\wsl$\..."
+   */
+  @NotNull
+  private static Map<@NotNull VirtualFile, @NotNull VirtualFile> getRemoteSourceToVfsMapping(@NotNull PyTargetAwareAdditionalData additionalData) {
+    var configuration = additionalData.getTargetEnvironmentConfiguration();
+    if (configuration == null) return Collections.emptyMap();
+    var vfsMapper = PythonInterpreterTargetEnvironmentFactory.getTargetWithMappedLocalVfs(configuration);
+    if (vfsMapper == null) return Collections.emptyMap();
+    var vfs = LocalFileSystem.getInstance();
+    var result = new HashMap<@NotNull VirtualFile, @NotNull VirtualFile>();
+    for (var remoteSourceAndVfs : ContainerUtil.map(additionalData.getPathMappings().getPathMappings(),
+                                                    m -> Pair.create(
+                                                      vfs.findFileByPath(m.getLocalRoot()),
+                                                      vfsMapper.getVfsFromTargetPath(m.getRemoteRoot())))) {
+      var remoteSourceDir = remoteSourceAndVfs.first;
+      var vfsDir = remoteSourceAndVfs.second;
+      if (remoteSourceDir != null && vfsDir != null) {
+        result.put(remoteSourceDir, vfsDir);
+      }
+    }
+    return result;
+  }
 }

python/src/com/jetbrains/python/run/PythonInterpreterTargetEnvironmentFactory.kt

@@ -12,6 +12,7 @@ import com.jetbrains.python.run.target.HelpersAwareLocalTargetEnvironmentRequest
 import com.jetbrains.python.run.target.HelpersAwareTargetEnvironmentRequest
 import com.jetbrains.python.sdk.add.target.ProjectSync
 import com.jetbrains.python.target.PyTargetAwareAdditionalData
+import com.jetbrains.python.target.targetWithVfs.TargetWithMappedLocalVfs
 import org.jetbrains.annotations.ApiStatus
 
 @ApiStatus.Experimental
@@ -37,6 +38,11 @@ interface PythonInterpreterTargetEnvironmentFactory {
    */
   fun getProjectSync(project: Project?, configuration: TargetEnvironmentConfiguration): ProjectSync?
 
+  /**
+   * Target provides access to its filesystem using VFS (like WSL)
+   */
+  fun asTargetWithMappedLocalVfs(envConfig: TargetEnvironmentConfiguration): TargetWithMappedLocalVfs? = null
+
   companion object {
     const val UNKNOWN_INTERPRETER_VERSION = "unknown interpreter"
 
@@ -86,5 +92,13 @@ interface PythonInterpreterTargetEnvironmentFactory {
 
     fun TargetEnvironmentConfiguration.isOfType(targetEnvironmentType: TargetEnvironmentType<*>): Boolean =
       typeId == targetEnvironmentType.id
+
+    /**
+     * Target provides access to its filesystem using VFS (like WSL)
+     */
+    @JvmStatic
+    fun getTargetWithMappedLocalVfs(targetEnvironmentConfiguration: TargetEnvironmentConfiguration) = EP_NAME.extensionList.asSequence().mapNotNull {
+      it.asTargetWithMappedLocalVfs(targetEnvironmentConfiguration)
+    }.firstOrNull()
   }
 }

python/src/com/jetbrains/python/sdk/PySdkExt.kt

@@ -393,10 +393,15 @@ private fun filterSuggestedPaths(suggestedPaths: Collection<String>,
 
 fun Sdk?.isTargetBased(): Boolean = this != null && targetEnvConfiguration != null
 
+/**
+ *  Additional data if sdk is target-based
+ */
+val Sdk.targetAdditionalData get():PyTargetAwareAdditionalData? = sdkAdditionalData as? PyTargetAwareAdditionalData
+
 /**
  * Returns target environment if configuration is target api based
  */
-val Sdk.targetEnvConfiguration get():TargetEnvironmentConfiguration? = (sdkAdditionalData as? PyTargetAwareAdditionalData)?.targetEnvironmentConfiguration
+val Sdk.targetEnvConfiguration get():TargetEnvironmentConfiguration? = targetAdditionalData?.targetEnvironmentConfiguration
 
 /**
  * Where "remote_sources" folder for certain SDK is stored

python/src/com/jetbrains/python/sdk/PyTargetsRemoteSourcesRefresher.kt

@@ -15,8 +15,11 @@ import com.intellij.openapi.Disposable
 import com.intellij.openapi.diagnostic.logger
 import com.intellij.openapi.progress.ProgressIndicator
 import com.intellij.openapi.project.Project
+import com.intellij.openapi.project.modules
+import com.intellij.openapi.project.rootManager
 import com.intellij.openapi.projectRoots.Sdk
 import com.intellij.openapi.util.Disposer
+import com.intellij.openapi.vfs.LocalFileSystem
 import com.intellij.remote.RemoteSdkProperties
 import com.intellij.util.PathMappingSettings
 import com.intellij.util.PathUtil
@@ -36,7 +39,7 @@ import kotlin.io.path.div
 
 private const val STATE_FILE = ".state.json"
 
-class PyTargetsRemoteSourcesRefresher(val sdk: Sdk, project: Project) {
+class PyTargetsRemoteSourcesRefresher(val sdk: Sdk, private val project: Project) {
   private val pyRequest: HelpersAwareTargetEnvironmentRequest =
     checkNotNull(PythonInterpreterTargetEnvironmentFactory.findPythonTargetInterpreter(sdk, project))
 
@@ -73,6 +76,21 @@ class PyTargetsRemoteSourcesRefresher(val sdk: Sdk, project: Project) {
     }
     execution.addParameter(downloadVolume.getTargetDownloadPath())
 
+    val targetWithVfs = sdk.targetEnvConfiguration?.let { PythonInterpreterTargetEnvironmentFactory.getTargetWithMappedLocalVfs(it) }
+    if (targetWithVfs != null) {
+      // If sdk is target that supports local VFS, there is no reason to copy editable packages to remote_sources
+      // since their paths should be available locally (to be edited)
+      // Such packages are in user content roots, so we report them to remote_sync script
+      val moduleRoots = project.modules.flatMap { it.rootManager.contentRoots.asList() }.mapNotNull { targetWithVfs.getTargetPathFromVfs(it) }
+      if (moduleRoots.isNotEmpty()) {
+        execution.addParameter("--project-roots")
+        for (root in moduleRoots) {
+          execution.addParameter(root)
+        }
+      }
+    }
+
+
     val targetIndicator = TargetProgressIndicatorAdapter(indicator)
     val environment = targetEnvRequest.prepareEnvironment(targetIndicator)
     try {
@@ -116,17 +134,35 @@ class PyTargetsRemoteSourcesRefresher(val sdk: Sdk, project: Project) {
       }
       rootZip.deleteExisting()
     }
+
+    if (targetWithVfs != null) {
+      // If target has local VFS, we map locally available roots to VFS instead of copying them to remote_sources
+      // See how ``updateSdkPaths`` is used
+      for (remoteRoot in stateFile.skippedRoots) {
+        val localPath = targetWithVfs.getVfsFromTargetPath(remoteRoot)?.path ?: continue
+        pathMappings.add(PathMappingSettings.PathMapping(localPath, remoteRoot))
+      }
+    }
     (sdk.sdkAdditionalData as? RemoteSdkProperties)?.setPathMappings(pathMappings)
+    val fs = LocalFileSystem.getInstance()
+    // "remote_sources" folder may now contain new packages
+    // since we copied them there not via VFS, we must refresh it, so Intellij knows about them
+    pathMappings.pathMappings.mapNotNull { fs.findFileByPath(it.localRoot) }.forEach { it.refresh(false, true) }
   }
 
   private class StateFile {
     var roots: List<RootInfo> = emptyList()
+
+    @SerializedName("skipped_roots")
+    var skippedRoots: List<String> = emptyList()
   }
 
   private class RootInfo {
     var path: String = ""
+
     @SerializedName("zip_name")
     var zipName: String = ""
+
     @SerializedName("invalid_entries")
     var invalidEntries: List<String> = emptyList()
   }

python/src/com/jetbrains/python/target/targetWithVfs/TargetVfsWatchedRootsProvider.kt

@@ -0,0 +1,22 @@
+// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package com.jetbrains.python.target.targetWithVfs
+
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.roots.WatchedRootsProvider
+import com.jetbrains.python.run.PythonInterpreterTargetEnvironmentFactory
+import com.jetbrains.python.sdk.targetAdditionalData
+import com.jetbrains.python.statistics.sdks
+
+class TargetVfsWatchedRootsProvider : WatchedRootsProvider {
+  override fun getRootsToWatch(project: Project): Set<String> {
+    val result = mutableSetOf<String>()
+    for (data in project.sdks.mapNotNull { it.targetAdditionalData }) {
+      val mapper = data.targetEnvironmentConfiguration?.let { PythonInterpreterTargetEnvironmentFactory.getTargetWithMappedLocalVfs(it) }
+                   ?: continue
+      for (remotePath in data.pathMappings.pathMappings.map { it.remoteRoot })
+        // children must be loaded for events to work
+        mapper.getVfsFromTargetPath(remotePath)?.let { it.children; result.add(it.path) }
+    }
+    return result
+  }
+}

python/src/com/jetbrains/python/target/targetWithVfs/TargetWithMappedLocalVfs.kt

@@ -0,0 +1,9 @@
+// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package com.jetbrains.python.target.targetWithVfs
+
+import com.intellij.openapi.vfs.VirtualFile
+
+interface TargetWithMappedLocalVfs {
+  fun getVfsFromTargetPath(targetPath: String): VirtualFile?
+  fun getTargetPathFromVfs(file: VirtualFile): String?
+}