Various validation refactorings

`readableFs` is redundant and unstable. Remove it: we will migrate to ijent anyway.
All validations are in `PathValidator.kt` now. They are used by `ManualPathEntryDialog` and sdk validation.
Lots of thread annotations added to prevent calling validation code from EDT.

In general, this change makes path validation ready for ijent: validation based on nio with slow IO access.

Validation is removed from old, non-target classes

(cherry picked from commit 185b4f7fe8cbd5d7a37dad609c8a4cb8163d6eed)

IJ-MR-112281

GitOrigin-RevId: 12c4a4f3d459d0523ef6694a9e4bb2db7a1582b7

python/python-sdk/intellij.python.sdk.iml

@@ -28,5 +28,6 @@
     <orderEntry type="library" name="commons-collections" level="project" />
     <orderEntry type="library" name="jna" level="project" />
     <orderEntry type="library" scope="TEST" name="JUnit4" level="project" />
+    <orderEntry type="library" name="caffeine" level="project" />
   </component>
 </module>

python/python-sdk/resources/messages/PySdkBundle.properties

@@ -68,4 +68,14 @@ python.configure.interpreter.action=Configure a Python interpreter\u2026
 python.configuring.interpreter.progress=Configuring a Python interpreter
 
 # Skeletons generator
-dialog.message.broken.home.path.for=Broken home path for {0}
+dialog.message.broken.home.path.for=Broken home path for {0}
+
+path.validation.wait=Validating path, Please Wait...
+path.validation.wait.path=Validating {0}, Please Wait...
+path.validation.field.empty=Path field is empty
+path.validation.cannot.execute=Cannot execute {0}
+path.validation.must.be.absolute=Path must be absolute
+path.validation.ends.with.whitespace=Path ends with a whitespace
+path.validation.file.not.found=File {0} is not found
+path.validation.invalid=Path is invalid: {0}
+path.validation.inaccessible=Path is inaccessible

python/python-sdk/src/com/jetbrains/python/pathValidation/PathValidator.kt

@@ -0,0 +1,87 @@
+// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package com.jetbrains.python.pathValidation
+
+import com.intellij.execution.Platform
+import com.intellij.openapi.ui.ValidationInfo
+import com.intellij.openapi.util.io.OSAgnosticPathUtil
+import com.intellij.util.concurrency.annotations.RequiresBackgroundThread
+import com.intellij.util.io.isDirectory
+import com.intellij.util.io.isFile
+import com.jetbrains.python.PySdkBundle
+import com.jetbrains.python.sdk.appxProduct
+import org.jetbrains.annotations.Nls
+import org.jetbrains.annotations.NonNls
+import java.io.IOException
+import java.nio.file.InvalidPathException
+import java.nio.file.Path
+import javax.swing.JComponent
+import kotlin.io.path.isExecutable
+import kotlin.io.path.listDirectoryEntries
+
+/**
+ * To be used with [validateExecutableFile] and [validateEmptyDir]
+ * @param path from path field value.
+ * @param fieldIsEmpty message to show if field is empty
+ * @param component see [ValidationInfo.component]
+ * @param platformAndRoot to validate path against
+ */
+class ValidationRequest(@NonNls internal val path: String?,
+                        @Nls val fieldIsEmpty: String = PySdkBundle.message("path.validation.field.empty"),
+                        private val platformAndRoot: PlatformAndRoot,
+                        private val component: JComponent? = null) {
+  internal fun validate(getMessage: (Path) -> @Nls String?): ValidationInfo? {
+    val message: @Nls String? = when {
+      path.isNullOrEmpty() -> fieldIsEmpty
+      !isAbsolutePath(path) -> PySdkBundle.message("path.validation.must.be.absolute")
+      path.endsWith(" ") -> PySdkBundle.message("path.validation.ends.with.whitespace")
+      else -> platformAndRoot.root?.let {
+        try {
+          val nioPath = it.resolve(path)
+          getMessage(nioPath)
+        }
+        catch (e: InvalidPathException) {
+          PySdkBundle.message("path.validation.invalid", e.message)
+        }
+        catch (e: IOException) {
+          PySdkBundle.message("path.validation.inaccessible", e.message)
+        }
+      }
+    }
+    return message?.let { ValidationInfo(it, component) }
+  }
+
+  private fun isAbsolutePath(path: String): Boolean = when (platformAndRoot.platform) {
+    Platform.UNIX -> path.startsWith("/")
+    Platform.WINDOWS -> OSAgnosticPathUtil.isAbsoluteDosPath(path)
+  }
+}
+
+/**
+ * Ensure file is executable
+ */
+@RequiresBackgroundThread
+fun validateExecutableFile(
+  request: ValidationRequest
+): ValidationInfo? = request.validate {
+  if (it.appxProduct != null) return@validate null // Nio can't be used to validate appx, assume file is valid
+  when {
+    it.isFile() -> if (it.isExecutable()) null else PySdkBundle.message("path.validation.cannot.execute", it)
+    it.isDirectory() -> PySdkBundle.message("path.validation.cannot.execute", it)
+    else -> PySdkBundle.message("path.validation.file.not.found", it)
+  }
+}
+
+/**
+ * Ensure directory either doesn't exist or empty
+ */
+@RequiresBackgroundThread
+fun validateEmptyDir(request: ValidationRequest,
+                     @Nls notADirectory: String,
+                     @Nls directoryNotEmpty: String
+): ValidationInfo? = request.validate {
+  when {
+    it.isDirectory() -> if (it.listDirectoryEntries().isEmpty()) null else directoryNotEmpty
+    it.isFile() || it.appxProduct != null -> notADirectory
+    else -> null
+  }
+}

python/python-sdk/src/com/jetbrains/python/pathValidation/PlatformAndRoot.kt

@@ -0,0 +1,33 @@
+// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package com.jetbrains.python.pathValidation
+
+import com.intellij.execution.Platform
+import com.intellij.execution.target.TargetEnvironmentConfiguration
+import com.intellij.util.concurrency.annotations.RequiresBackgroundThread
+import java.nio.file.Path
+
+/**
+ * Against what path must be validated. [root] is now nullable since no targets provide nio access, but that will be fixed soon.
+ */
+class PlatformAndRoot private constructor(val root: Path?, val platform: Platform) {
+  companion object {
+    /**
+     * Local system
+     */
+    val local: PlatformAndRoot = PlatformAndRoot(Path.of(""), Platform.current())
+
+    /**
+     * Creates [PlatformAndRoot] for [TargetEnvironmentConfiguration]. If null then returns either [local] or [platform] only depending
+     * on [defaultIsLocal]
+     */
+    @RequiresBackgroundThread
+    fun TargetEnvironmentConfiguration?.getPlatformAndRoot(defaultIsLocal: Boolean = true): PlatformAndRoot {
+      val unknownTarget = PlatformAndRoot(null, Platform.UNIX)
+      return when {
+        this == null -> if (defaultIsLocal) local else unknownTarget
+        else -> unknownTarget //Non-null target is never local
+      }
+    }
+
+  }
+}

python/python-sdk/src/com/jetbrains/python/sdk/flavors/PySdkFlavorUtil.kt

@@ -0,0 +1,16 @@
+package com.jetbrains.python.sdk.flavors
+
+import com.intellij.execution.target.TargetEnvironmentConfiguration
+import com.intellij.util.concurrency.annotations.RequiresBackgroundThread
+import com.jetbrains.python.pathValidation.PlatformAndRoot.Companion.getPlatformAndRoot
+import com.jetbrains.python.pathValidation.ValidationRequest
+import com.jetbrains.python.pathValidation.validateExecutableFile
+import org.jetbrains.annotations.Nls
+import org.jetbrains.annotations.NonNls
+
+/**
+ * Checks if file is executable. If no -- returns error
+ */
+@RequiresBackgroundThread
+internal fun getFileExecutionError(@NonNls fullPath: String, targetEnvConfig: TargetEnvironmentConfiguration?): @Nls String? =
+  validateExecutableFile(ValidationRequest(fullPath, platformAndRoot = targetEnvConfig.getPlatformAndRoot()))?.message

python/python-sdk/src/com/jetbrains/python/sdk/flavors/PythonSdkFlavor.java

@@ -1,15 +1,18 @@
 // Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
 package com.jetbrains.python.sdk.flavors;
 
-import com.google.common.collect.EvictingQueue;
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
 import com.intellij.execution.configurations.GeneralCommandLine;
 import com.intellij.execution.process.ProcessOutput;
+import com.intellij.execution.target.TargetConfigurationWithId;
 import com.intellij.execution.target.TargetEnvironmentConfiguration;
-import com.intellij.execution.target.readableFs.PathInfo;
-import com.intellij.execution.target.readableFs.TargetConfigurationReadableFs;
 import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.openapi.extensions.ExtensionPointName;
 import com.intellij.openapi.module.Module;
+import com.intellij.openapi.progress.ProgressIndicator;
+import com.intellij.openapi.progress.ProgressManager;
+import com.intellij.openapi.progress.Task;
 import com.intellij.openapi.projectRoots.Sdk;
 import com.intellij.openapi.projectRoots.SdkAdditionalData;
 import com.intellij.openapi.util.UserDataHolder;
@@ -18,6 +21,7 @@ import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.util.PatternUtil;
 import com.intellij.util.containers.ContainerUtil;
+import com.jetbrains.python.PySdkBundle;
 import com.jetbrains.python.psi.LanguageLevel;
 import com.jetbrains.python.run.CommandLinePatcher;
 import com.jetbrains.python.sdk.PyRemoteSdkAdditionalDataMarker;
@@ -25,16 +29,19 @@ import com.jetbrains.python.sdk.PySdkUtil;
 import com.jetbrains.python.sdk.PythonEnvUtil;
 import com.jetbrains.python.sdk.PythonSdkAdditionalData;
 import icons.PythonSdkIcons;
+import org.jetbrains.annotations.Nls;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
 import javax.swing.*;
 import java.io.File;
-import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.*;
+import java.util.concurrent.TimeUnit;
 import java.util.regex.Pattern;
 
+import static com.jetbrains.python.sdk.flavors.PySdkFlavorUtilKt.getFileExecutionError;
+
 
 /**
  * Flavor is a type of python interpreter stored in {@link PythonSdkAdditionalData}.
@@ -45,10 +52,13 @@ import java.util.regex.Pattern;
 public abstract class PythonSdkFlavor<D extends PyFlavorData> {
   public static final ExtensionPointName<PythonSdkFlavor<?>> EP_NAME = ExtensionPointName.create("Pythonid.pythonSdkFlavor");
   /**
-   * To prevent log pollution, we cache every {@link #isFileExecutable(String, TargetEnvironmentConfiguration)} call
+   * To prevent log pollution and slowness, we cache every {@link #isFileExecutable(String, TargetEnvironmentConfiguration)} call
    * and only log it once
    */
-  private static final Collection<String> ourBuffer = Collections.synchronizedCollection(EvictingQueue.create(10));
+  private static final Cache<@NotNull String, @NotNull Boolean> ourExecutableFiles = Caffeine.newBuilder()
+    .expireAfterWrite(10, TimeUnit.MINUTES)
+    .maximumSize(1000)
+    .build();
 
   private static final Pattern VERSION_RE = Pattern.compile("(Python \\S+).*");
   private static final Logger LOG = Logger.getInstance(PythonSdkFlavor.class);
@@ -153,30 +163,58 @@ public abstract class PythonSdkFlavor<D extends PyFlavorData> {
    * @param fullPath full path on target
    */
   protected static boolean isFileExecutable(@NotNull String fullPath, @Nullable TargetEnvironmentConfiguration targetEnvConfig) {
-    boolean executable = isFileExecutableImpl(fullPath, targetEnvConfig);
-    if (!executable) {
-      if (!ourBuffer.contains(fullPath)) {
-        ourBuffer.add(fullPath);
-        Logger.getInstance(PythonSdkFlavor.class).warn(String.format("%s is not executable", fullPath));
-      }
+    var id = getIdForCache(fullPath, targetEnvConfig);
+    Boolean executable = ourExecutableFiles.getIfPresent(id);
+    if (executable != null) {
+      return executable;
     }
-    return executable;
+    var error = getErrorIfNotExecutable(fullPath, targetEnvConfig);
+    if (error != null) {
+      Logger.getInstance(PythonSdkFlavor.class).warn(String.format("%s is not executable: %s", fullPath, error));
+    }
+    var newValue = error == null;
+    ourExecutableFiles.put(id, newValue);
+    return newValue;
   }
 
-  private static boolean isFileExecutableImpl(@NotNull String fullPath, @Nullable TargetEnvironmentConfiguration targetEnvConfig) {
-    if (targetEnvConfig == null) {
-      // Local
-      return Files.isExecutable(Path.of(fullPath));
+  @Nullable
+  @Nls
+  private static String getErrorIfNotExecutable(@NotNull String fullPath, @Nullable TargetEnvironmentConfiguration targetEnvConfig) {
+    if (SwingUtilities.isEventDispatchThread()) {
+      // Run under progress
+      // TODO: use pyModalBlocking when we merge two modules
+      return ProgressManager.getInstance()
+        .run(new Task.WithResult<@Nullable @Nls String, RuntimeException>(null, PySdkBundle.message("path.validation.wait.path", fullPath),
+                                                                          false) {
+          @Override
+          @Nls
+          @Nullable
+          protected String compute(@NotNull ProgressIndicator indicator) throws RuntimeException {
+            return getFileExecutionError(fullPath, targetEnvConfig);
+          }
+        });
     }
-    if (targetEnvConfig instanceof TargetConfigurationReadableFs) {
-      var fileInfo = ((TargetConfigurationReadableFs)targetEnvConfig).getPathInfo(fullPath);
-      if (fileInfo instanceof PathInfo.Unknown) {
-        return true; // We can't be sure if file is executable or not
-      }
-      return (fileInfo instanceof PathInfo.RegularFile) && (((PathInfo.RegularFile)fileInfo).getExecutable());
+    else {
+      return getFileExecutionError(fullPath, targetEnvConfig);
     }
-    // We can't be sure if file is executable or not
-    return true;
+  }
+
+  @NotNull
+  private static String getIdForCache(@NotNull String fullPath, @Nullable TargetEnvironmentConfiguration configuration) {
+    var builder = new StringBuilder(fullPath);
+    builder.append(" ");
+    if (configuration instanceof TargetConfigurationWithId) {
+      var typeAndTargetId = ((TargetConfigurationWithId)configuration).getTargetAndTypeId();
+      builder.append(typeAndTargetId.component1().toString());
+      builder.append(typeAndTargetId.getSecond());
+    }
+    else if (configuration != null) {
+      builder.append(configuration.getClass().getName());
+    }
+    else {
+      builder.append("local");
+    }
+    return builder.toString();
   }
 
   public static @NotNull List<PythonSdkFlavor<?>> getApplicableFlavors() {