PY-43082: Support pythons from WindowsStore, but not "fake" pythons that link to store app.

Since 2004 Win10 has "python.exe" reparse point in path. This point runs WindowsStore app, so user can install Python.
After installation this point is replaced with real python.

This change uses native app to see if reparse point links to real python or desktop

GitOrigin-RevId: 2d7962be588b5dc83bab412c6ad483fe48298bf3

native/WinElevator/AppxReparseTool/README.txt

@@ -1,19 +1,19 @@
 AppX apps are installed in "C:\Program Files\WindowsApps\".
 You can't run them directly. Instead, you must use reparse point from
-"%LOCALAPPDATA%\Microsoft\WindowsApps" (thi folder is under the %PATH%)
+"%LOCALAPPDATA%\Microsoft\WindowsApps" (this folder is under the %PATH%)
 
 Reparse point is the special structure on NTFS level that stores "reparse tag" (type) and some type-specific data.
-When user access such files, Windows redirects her to appropriate target.
+When user access such files, Windows redirects request to the appropriate target.
 So, files in "%LOCALAPPDATA%\Microsoft\WindowsApps" are reparse points to AppX apps, and AppX can only be launched via them.
 
 But for Python there can be reparse point that points to Windows store, so Store is opened when Python is not installed.
 There is no official way to tell if "python.exe" there points to AppX python or AppX "Windows Store".
 
 MS provides API (via DeviceIOControl) to read reparse point structure.
-There is also raprse point tag for "AppX link" in SDK.
-Reparse data is undocumented, but it is just a array of wide chars with some unprintable chars at the beginning.
+There is also reparse point tag for "AppX link" in SDK.
+Reparse data is undocumented, but it is just an array of wide chars with some unprintable chars at the beginning.
 
 This tool reads reparse point info and tries to fetch AppX name, so we can see if it points to Store or not.
-See https://youtrack.jetbrains.com/issue/PY-43082#focus=Comments-27-4224605.0-0
+See https://youtrack.jetbrains.com/issue/PY-43082
 
 Output is unicode 16-LE

python/src/com/jetbrains/python/sdk/flavors/WinAppxTools.kt

@@ -1,55 +1,70 @@
 // Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
 package com.jetbrains.python.sdk.flavors
 
+import com.intellij.openapi.application.PathManager
 import com.intellij.openapi.util.SystemInfo
-import com.intellij.openapi.util.io.FileUtil
 import java.io.File
 import java.io.FilenameFilter
+import java.util.concurrent.TimeUnit
 
 /**
- * AppX packages installed to AppX volume (see `Get-AppxDefaultVolume`).
- * At the same time, **reparse point** is created somewhere in `%LOCALAPPDATA%`.
- * This point has tag `IO_REPARSE_TAG_APPEXECLINK` and it also added to `PATH`
+ * AppX packages installed to AppX volume (see ``Get-AppxDefaultVolume``, ``Get-AppxPackage``).
+ * They can't be executed nor read.
  *
- * Such points can't be read. Their attributes are also inaccessible. [File#exists] returns false.
+ * At the same time, **reparse point** is created somewhere in `%LOCALAPPDATA%`.
+ * This point has tag ``IO_REPARSE_TAG_APPEXECLINK`` and it also added to `PATH`
+ *
+ * Their attributes are also inaccessible. [File#exists] returns false.
  * But when executed, they are processed by NTFS filter and redirected to their real location in AppX volume.
- * They are also returned with parent's [File#listFiles]
- * There is no Java API to fetch reparse data, and its structure is undocumented (although pretty simple), so we workaround it
+ *
+ * There may be ``python.exe`` there, but it may point to Windows Store (so it can be installed when accessed) or to the real python.
+ * There is no Java API to see reparse point destination, so we use native app (See ``appxreparse.c`` and README in the same folder).
+ * This tool returns AppX name (either ``PythonSoftwareFoundation...`` or ``DesktopAppInstaller..``).
+ * We use it to check if ``python.exe`` is real python or WindowsStore mock.
  */
 
 /**
- * If file is appx reparse point, then file.exists doesn't work.
+ * When product of AppX reparse point contains this word, that means it is a link to the store
  */
-fun mayBeAppXReparsePoint(file: File): Boolean =
-  pythonsStoreLocation?.let { storeLocation ->
-    FileUtil.isAncestor(storeLocation, file, false)
-  } == true
+private const val storeMarker = "DesktopAppInstaller"
 
 /**
- * Since you can't use file.exists for reparse point, this function checks if file exists
+ * Files in [userAppxFolder] that matches [filePattern] and contains [expectedProduct]] in their product name
  */
-fun appXReparsePointFileExists(file: File): Boolean {
-  return file.exists() || (mayBeAppXReparsePoint(file) && file.parentFile.list()?.contains(file.name) == true)
-}
+fun getAppxFiles(expectedProduct: String, filePattern: Regex): Collection<File> =
+  userAppxFolder?.listFiles(FilenameFilter { _, name -> filePattern.matches(name) })
+    ?.sortedBy { it.nameWithoutExtension }
+    ?.mapNotNull { file -> file.appxProduct?.let { product -> Pair(product, file) } }
+    ?.toMap()
+    ?.filterKeys { expectedProduct in it }
+    ?.values ?: emptyList()
 
 /**
- * Appx apps are installed in [pythonsStoreLocation], each one in separate folder.
- * But folders are inaccessible, but there are reparse points on the toplevel.
- * This function provides list of them
+ * If file is AppX reparse point link -- return it's product name
  */
-fun getAppXAppsInstalled(filenameFilter: FilenameFilter): List<File> =
-  pythonsStoreLocation?.list(filenameFilter)?.mapNotNull { File(pythonsStoreLocation, it) }
-  ?: emptyList()
-
-private val pythonsStoreLocation
-  get(): File? {
-    if (!SystemInfo.isWin10OrNewer) {
-      return null
-    }
-    val localappdata = System.getenv("LOCALAPPDATA") ?: return null
-    val appsPath = File(localappdata, "Microsoft//WindowsApps")
-    return if (appsPath.exists()) appsPath else null
+val File.appxProduct: String?
+  get() {
+    if (parentFile?.equals(userAppxFolder) != true) return null
+    val reparseTool = PathManager.findBinFile("AppxReparse.exe")!!
+    // Intellij API prohibits running external processes under Read action or on EDT, so we use java api as it is done for registry
+    val process = Runtime.getRuntime().exec(arrayOf(reparseTool.toFile().absolutePath, absolutePath))
+    // It is much faster in most cases, but since this code may run under EDT we limit it
+    if (!process.waitFor(1, TimeUnit.SECONDS)) return null
+    if (process.exitValue() != 0) return null
+    // appxreparse outputs wide chars (2 bytes), they are LE on Intel
+    val output = process.inputStream.readBytes().toString(Charsets.UTF_16LE)
+    return if (storeMarker !in output) output else null
   }
 
-
-
+/**
+ * Path to ``%LOCALAPPDATA%\Microsoft\WindowsApps``
+ */
+private val userAppxFolder =
+  if (!SystemInfo.isWin10OrNewer) {
+    null
+  }
+  else {
+    val localappdata = System.getenv("LOCALAPPDATA") ?: null
+    val appsPath = File(localappdata, "Microsoft//WindowsApps")
+    if (appsPath.exists()) appsPath else null
+  }

python/src/com/jetbrains/python/sdk/flavors/WinPythonSdkFlavor.java

@@ -13,14 +13,18 @@ import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.openapi.vfs.LocalFileSystem;
 import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.openapi.vfs.newvfs.NewVirtualFile;
+import com.intellij.util.containers.ContainerUtil;
 import com.jetbrains.python.PythonHelpersLocator;
+import kotlin.text.Regex;
+import org.apache.commons.lang.StringUtils;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
 import java.io.File;
 import java.util.*;
 
-import static com.jetbrains.python.sdk.flavors.WinAppxToolsKt.*;
+import static com.jetbrains.python.sdk.flavors.WinAppxToolsKt.getAppxFiles;
+import static com.jetbrains.python.sdk.flavors.WinAppxToolsKt.getAppxProduct;
 
 /**
  * This class knows how to find python in Windows Registry according to
@@ -30,8 +34,15 @@ import static com.jetbrains.python.sdk.flavors.WinAppxToolsKt.*;
  */
 public class WinPythonSdkFlavor extends CPythonSdkFlavor {
   @NotNull
-  private static final String NOTHING = "";
   private static final String[] REG_ROOTS = {"HKEY_LOCAL_MACHINE", "HKEY_CURRENT_USER"};
+  /**
+   * There may be a lot of python files in APPX folder. We do not need "w" files, but may need "python[version]?.exe"
+   */
+  private static final Regex PYTHON_EXE = new Regex("^python[0-9.]*?.exe$");
+  /**
+   * All Pythons from WinStore have "Python[something]" as their product name
+   */
+  private static final String APPX_PRODUCT = "Python";
   private static final Map<String, String> REGISTRY_MAP =
     ImmutableMap.of("Python", "python.exe",
                     "IronPython", "ipy.exe");
@@ -39,6 +50,9 @@ public class WinPythonSdkFlavor extends CPythonSdkFlavor {
   @NotNull
   private final ClearableLazyValue<Set<String>> myRegistryCache =
     ClearableLazyValue.createAtomic(() -> findInRegistry(getWinRegistryService()));
+  @NotNull
+  private final ClearableLazyValue<Set<String>> myAppxCache =
+    ClearableLazyValue.createAtomic(() -> getPythonsFromStore());
 
   public static WinPythonSdkFlavor getInstance() {
     return PythonSdkFlavor.EP_NAME.findExtension(WinPythonSdkFlavor.class);
@@ -65,10 +79,7 @@ public class WinPythonSdkFlavor extends CPythonSdkFlavor {
     }
 
     findInRegistry(candidates);
-
-    getAppXAppsInstalled((dir, name) -> name.equals("python.exe")).stream()
-      .findFirst()
-      .ifPresent(python -> candidates.add(python.getAbsolutePath()));
+    candidates.addAll(myAppxCache.getValue());
   }
 
   @Override
@@ -77,13 +88,18 @@ public class WinPythonSdkFlavor extends CPythonSdkFlavor {
       return true;
     }
 
+    if (myAppxCache.getValue().contains(path)) {
+      return true;
+    }
+
     final File file = new File(path);
-    return mayBeAppXReparsePoint(file) && isValidSdkPath(file);
+    return StringUtils.contains(getAppxProduct(file), APPX_PRODUCT) && isValidSdkPath(file);
   }
 
   @Override
   public void dropCaches() {
     myRegistryCache.drop();
+    myAppxCache.drop();
   }
 
 
@@ -117,6 +133,11 @@ public class WinPythonSdkFlavor extends CPythonSdkFlavor {
     }
   }
 
+  @NotNull
+  private static Set<String> getPythonsFromStore() {
+    return ContainerUtil.map2Set(getAppxFiles(APPX_PRODUCT, PYTHON_EXE), file -> file.getAbsolutePath());
+  }
+
   @NotNull
   private static Set<String> findInRegistry(@NotNull WinRegistryService registryService) {
     final Set<String> result = new HashSet<>();