PY-40423: List `LOCALAPPDATA to find python from store, do not use Get-AppxPackage` cmdlet.

Appx apps are installed in special folder inside of ``LOCALAPPDATA``. You can get their location via cmdlet, but this path is not accessible for user in modern versions of Win10.

The official way is to use reparse point inside of ``LOCALAPPDATA``. Since it is added to ``PATH``, it should be accessible and executable.

However, this file can't be checked with ``File.exists`` and other ``File`` tools because it is a special reparse point.

* We list ``LOCALAPPDATA`` to find Python
* Appx-specific knowledge is in WinAppxTools.kt now

GitOrigin-RevId: 9eeaf186379c30531159b9fe4a2395930d4993c0

python/src/com/jetbrains/python/sdk/flavors/WinAppxTools.kt

@@ -0,0 +1,55 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.jetbrains.python.sdk.flavors
+
+import com.intellij.openapi.util.SystemInfo
+import com.intellij.openapi.util.io.FileUtil
+import java.io.File
+import java.io.FilenameFilter
+
+/**
+ * AppX packages installed to AppX volume (see `Get-AppxDefaultVolume`).
+ * At the same time, **reparse point** is created somewhere in `%LOCALAPPDATA%`.
+ * This point has tag `IO_REPARSE_TAG_APPEXECLINK` and it also added to `PATH`
+ *
+ * Such points can't be read. Their attributes are also inaccessible. [File#exists] returns false.
+ * But when executed, they are processed by NTFS filter and redirected to their real location in AppX volume.
+ * They are also returned with parent's [File#listFiles]
+ * There is no Java API to fetch reparse data, and its structure is undocumented (although pretty simple), so we workaround it
+ */
+
+/**
+ * If file is appx reparse point, then file.exists doesn't work.
+ */
+fun mayBeAppXReparsePoint(file: File): Boolean =
+  pythonsStoreLocation?.let { storeLocation ->
+    FileUtil.isAncestor(storeLocation, file, false)
+  } == true
+
+/**
+ * Since you can't use file.exists for reparse point, this function checks if file exists
+ */
+fun appXReparsePointFileExists(file: File): Boolean {
+  return file.exists() || (mayBeAppXReparsePoint(file) && file.parentFile.list()?.contains(file.name) == true)
+}
+
+/**
+ * Appx apps are installed in [pythonsStoreLocation], each one in separate folder.
+ * But folders are inaccessible, but there are reparse points on the toplevel.
+ * This function provides list of them
+ */
+fun getAppXAppsInstalled(filenameFilter: FilenameFilter): List<File> =
+  pythonsStoreLocation?.list(filenameFilter)?.mapNotNull { File(pythonsStoreLocation, it) }
+  ?: emptyList()
+
+private val pythonsStoreLocation
+  get(): File? {
+    if (!SystemInfo.isWin10OrNewer) {
+      return null
+    }
+    val localappdata = System.getenv("LOCALAPPDATA") ?: return null
+    val appsPath = File(localappdata, "Microsoft//WindowsApps")
+    return if (appsPath.exists()) appsPath else null
+  }
+
+
+

python/src/com/jetbrains/python/sdk/flavors/WinPythonSdkFlavor.java

@@ -5,7 +5,6 @@ import com.google.common.collect.ImmutableMap;
 import com.intellij.openapi.application.ApplicationManager;
 import com.intellij.openapi.module.Module;
 import com.intellij.openapi.util.ClearableLazyValue;
-import com.intellij.openapi.util.Key;
 import com.intellij.openapi.util.SystemInfo;
 import com.intellij.openapi.util.UserDataHolder;
 import com.intellij.openapi.util.io.FileUtil;
@@ -13,9 +12,7 @@ import com.intellij.openapi.util.io.FileUtilRt;
 import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.openapi.vfs.LocalFileSystem;
 import com.intellij.openapi.vfs.VirtualFile;
-import com.intellij.openapi.vfs.VirtualFileSystem;
 import com.intellij.openapi.vfs.newvfs.NewVirtualFile;
-import com.intellij.util.ArrayUtil;
 import com.jetbrains.python.PythonHelpersLocator;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
@@ -23,6 +20,8 @@ import org.jetbrains.annotations.Nullable;
 import java.io.File;
 import java.util.*;
 
+import static com.jetbrains.python.sdk.flavors.WinAppxToolsKt.*;
+
 /**
  * This class knows how to find python in Windows Registry according to
  * <a href="https://www.python.org/dev/peps/pep-0514/">PEP 514</a>
@@ -31,7 +30,6 @@ import java.util.*;
  */
 public class WinPythonSdkFlavor extends CPythonSdkFlavor {
   @NotNull
-  private static final Key<String> APPX_PYTHON_CACHE = new Key<>("PythonFromStoreCache");
   private static final String NOTHING = "";
   private static final String[] REG_ROOTS = {"HKEY_LOCAL_MACHINE", "HKEY_CURRENT_USER"};
   private static final Map<String, String> REGISTRY_MAP =
@@ -57,44 +55,9 @@ public class WinPythonSdkFlavor extends CPythonSdkFlavor {
     Set<String> candidates = new TreeSet<>();
     findInCandidatePaths(candidates, "python.exe", "jython.bat", "pypy.exe");
     findInstallations(candidates, "python.exe", PythonHelpersLocator.getHelpersRoot().getParent());
-
-    if (SystemInfo.isWin10OrNewer) {
-      // For pythons installed from WindowsStore
-      final VirtualFile installLocation = getInstallationLocationForStoreWithCache(context);
-      if (installLocation != null) {
-        final VirtualFile pythonFromStore = installLocation.findChild("python.exe");
-        if (pythonFromStore != null) {
-          candidates.add(pythonFromStore.getPath());
-        }
-      }
-    }
-
     return candidates;
   }
 
-  @Nullable
-  private static VirtualFile getInstallationLocationForStoreWithCache(@Nullable final UserDataHolder context) {
-    final VirtualFileSystem fs = LocalFileSystem.getInstance();
-
-    if (context != null) {
-      synchronized (APPX_PYTHON_CACHE) {
-        final String result = context.getUserData(APPX_PYTHON_CACHE);
-        if (result != null) {
-          return result.equals(NOTHING) ? null : fs.refreshAndFindFileByPath(result);
-        }
-        final VirtualFile python = getInstallationLocationForStore(fs);
-        context.putUserData(APPX_PYTHON_CACHE, python != null ? python.getPath() : NOTHING);
-        return python;
-      }
-    }
-    return getInstallationLocationForStore(fs);
-  }
-
-  @Nullable
-  private static VirtualFile getInstallationLocationForStore(@NotNull final VirtualFileSystem fs) {
-    return WindowsStoreServiceKt.findInstallLocationForPackage("Python", fs);
-  }
-
   private void findInCandidatePaths(Set<String> candidates, String... exe_names) {
     for (String name : exe_names) {
       findInstallations(candidates, name, "C:\\", "C:\\Program Files\\");
@@ -102,6 +65,10 @@ public class WinPythonSdkFlavor extends CPythonSdkFlavor {
     }
 
     findInRegistry(candidates);
+
+    getAppXAppsInstalled((dir, name) -> name.equals("python.exe")).stream()
+      .findFirst()
+      .ifPresent(python -> candidates.add(python.getAbsolutePath()));
   }
 
   @Override
@@ -119,39 +86,6 @@ public class WinPythonSdkFlavor extends CPythonSdkFlavor {
     myRegistryCache.drop();
   }
 
-  /**
-   * AppX packages installed to AppX volume (see <code>Get-AppxDefaultVolume</code>).
-   * At the same time, <strong>reparse point</strong> is created somewhere in <code>%LOCALAPPDATA%</code>.
-   * This point has tag <code>IO_REPARSE_TAG_APPEXECLINK</code> and it also added to <code>PATH</code>
-   * <br/>
-   * Such points can't be read. Their attributes are also inaccessible. {@link File#exists()} returns false.
-   * But when executed, they are processed by NTFS filter and redirected to their real location in AppX volume.
-   * They are also returned with parent's {@link File#listFiles()}
-   * <br/>
-   * There is no Java API to fetch reparse data, and its structure is undocumented (although pretty simple), so we workaround it
-   */
-  private static boolean mayBeAppXReparsePoint(@NotNull final File file) {
-    if (!SystemInfo.isWin10OrNewer) {
-      return false;
-    }
-
-    final String localAppData = System.getenv("LOCALAPPDATA");
-    if (localAppData == null) {
-      return false;
-    }
-    final File localAppDataFile = new File(localAppData);
-
-    if (!FileUtil.isAncestor(localAppDataFile, file, true)) {
-      return false;
-    }
-    final File parent = file.getParentFile();
-    if (parent == null) {
-      return false;
-    }
-    final File[] files = parent.listFiles();
-    return (files != null && ArrayUtil.contains(file, files));
-  }
-
 
   void findInRegistry(@NotNull final Collection<String> candidates) {
     candidates.addAll(myRegistryCache.getValue());
@@ -220,6 +154,7 @@ public class WinPythonSdkFlavor extends CPythonSdkFlavor {
     return result;
   }
 
+
   private static void findSubdirInstallations(Collection<String> candidates, String rootDir, String dir_prefix, String exe_name) {
     VirtualFile rootVDir = LocalFileSystem.getInstance().findFileByPath(rootDir);
     if (rootVDir != null) {

python/src/com/jetbrains/python/sdk/flavors/WindowsStoreService.kt

@@ -1,70 +0,0 @@
-// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
-package com.jetbrains.python.sdk.flavors
-
-import com.intellij.openapi.diagnostic.Logger
-import com.intellij.openapi.util.SystemInfo
-import com.intellij.openapi.util.text.StringUtil
-import com.intellij.openapi.vfs.LocalFileSystem
-import com.intellij.openapi.vfs.VirtualFile
-import com.intellij.openapi.vfs.VirtualFileSystem
-import java.io.IOException
-import java.util.concurrent.TimeUnit
-
-/**
- * Powershell may not exist under PATH, so we first look for it in well-known location
- */
-private fun findPowerShell(): VirtualFile? {
-  val fs = LocalFileSystem.getInstance()
-  val winDir = System.getenv("WINDIR") ?: return null
-  val winDirFile = fs.findFileByPath(winDir) ?: return null
-  return winDirFile.findFileByRelativePath("/System32/WindowsPowerShell/v1.0/powershell.exe")
-}
-
-/**
- * On Win10 uses `Get-AppxPackage` cmdlet to fetch installation location of package by name.
- * To be used to find location of tools installed with Windows Store
- */
-fun findInstallLocationForPackage(packageName: String, fs:VirtualFileSystem): VirtualFile? {
-  if (!SystemInfo.isWin10OrNewer) {
-    return null
-  }
-  val alphaNumeric = Regex("^[a-zA-Z0-9]+$")
-  val splitLine = Regex("""^[$]_.InstallLocation\s*:\s*(.+)$""")
-  val logger = Logger.getFactory().getLoggerInstance("findPackage")
-
-  assert(packageName.isNotBlank() && packageName.matches(alphaNumeric)) { "Only alphanumeric packages are supported" }
-  val powershell = findPowerShell()?.path ?: "powershell.exe"
-  val command = "\"Get-AppxPackage | Where-Object {\$_.Name -like '*$packageName*'} | Select-Object {\$_.InstallLocation} | Format-List\""
-  val process: Process
-  try {
-    process = Runtime.getRuntime().exec(arrayOf(powershell, "-Command", command))
-  }
-  catch (e: IOException) {
-    logger.warn(e)
-    return null
-  }
-  val result = process.waitFor(5, TimeUnit.SECONDS)
-  if (!result) {
-    reportError(command, "Process still runs after timeout", process, logger)
-    return null
-  }
-  val exitValue = process.exitValue()
-  if (exitValue != 0) {
-    reportError(command, "Process exited $exitValue", process, logger)
-    return null
-  }
-  val line = process.inputStream.bufferedReader().lines().filter { it.isNotBlank() }.findFirst().orElse(null) ?: return null
-  val groupValues = splitLine.find(line)?.groupValues ?: return null
-  if (groupValues.size != 2) {
-    logger.warn("Strange output: $line")
-    return null
-  }
-  return fs.refreshAndFindFileByPath(groupValues[1])
-}
-
-private fun reportError(command: String, error: String, process: Process, logger: Logger) {
-  logger.warn(error)
-  logger.warn(command)
-  logger.warn(process.errorStream.bufferedReader().readText())
-  logger.warn(process.inputStream.bufferedReader().readText())
-}