Don't remove pre-generated skeletons on clean-up (PY-26751)

python/helpers/pycharm_generator_utils/module_redeclarator.py

@@ -1,8 +1,10 @@
 import keyword
+import os
 
-from pycharm_generator_utils.util_methods import *
 from pycharm_generator_utils.constants import *
+from pycharm_generator_utils.util_methods import *
 
+is_pregenerated = os.getenv("IS_PREGENERATED_SKELETONS", None)
 
 class emptylistdict(dict):
     """defaultdict not available before 2.5; simplest reimplementation using [] as default"""
@@ -781,7 +783,9 @@ class ModuleRedeclarator(object):
         out(0, "# module ", p_name, mod_name) # line 2
 
         BUILT_IN_HEADER = "(built-in)"
-        if self.mod_filename:
+        if is_pregenerated is not None:
+            filename = '(pre-generated)'
+        elif self.mod_filename:
             filename = self.mod_filename
         elif p_name in sys.builtin_module_names:
             filename = BUILT_IN_HEADER

python/src/com/jetbrains/python/sdk/skeletons/PySkeletonGenerator.java

@@ -61,6 +61,8 @@ public class PySkeletonGenerator {
   private final String mySkeletonsPath;
   @NotNull protected final Map<String, String> myEnv;
 
+  private boolean myPrebuilt = false;
+
   public void finishSkeletonsGeneration() {
   }
 
@@ -68,6 +70,10 @@ public class PySkeletonGenerator {
     return new File(name).exists();
   }
 
+  public void setPrebuilt(boolean prebuilt) {
+    myPrebuilt = prebuilt;
+  }
+
   public static class ListBinariesResult {
     public final int generatorVersion;
     public final Map<String, PySkeletonRefresher.PyBinaryItem> modules;
@@ -167,6 +173,10 @@ public class PySkeletonGenerator {
     final Map<String, String> extraEnv = PythonSdkType.getVirtualEnvExtraEnv(binaryPath);
     final Map<String, String> env = extraEnv != null ? PySdkUtil.mergeEnvVariables(myEnv, extraEnv) : myEnv;
 
+    if (myPrebuilt) {
+      env.put("IS_PREGENERATED_SKELETONS", "1");
+    }
+
     return getProcessOutput(parent_dir, ArrayUtil.toStringArray(commandLine), env, MINUTE * 10);
   }
 

python/src/com/jetbrains/python/sdk/skeletons/PySkeletonRefresher.java

@@ -270,7 +270,7 @@ public class PySkeletonRefresher {
     return mySkeletonsPath;
   }
 
-  public List<String> regenerateSkeletons(@Nullable SkeletonVersionChecker cachedChecker) throws InvalidSdkException {
+  public List<String> regenerateSkeletons(@Nullable SkeletonVersionChecker checker) throws InvalidSdkException {
     final List<String> errorList = new SmartList<>();
     final String homePath = mySdk.getHomePath();
     final String skeletonsPath = getSkeletonsPath();
@@ -281,6 +281,10 @@ public class PySkeletonRefresher {
     }
     final String readablePath = FileUtil.getLocationRelativeToUserHome(homePath);
 
+    if (checker != null && checker.isPregenerated()) {
+      mySkeletonsGenerator.setPrebuilt(true);
+    }
+
     mySkeletonsGenerator.prepare();
     myBlacklist = loadBlacklist();
 
@@ -304,8 +308,9 @@ public class PySkeletonRefresher {
     myPregeneratedSkeletons = PyPregeneratedSkeletonsProvider.findPregeneratedSkeletonsForSdk(mySdk, myGeneratorVersion);
 
     indicate(PyBundle.message("sdk.gen.reading.versions.file"));
-    if (cachedChecker != null) {
-      myVersionChecker = cachedChecker.withDefaultVersionIfUnknown(myGeneratorVersion);
+
+    if (checker != null) {
+      myVersionChecker = checker.withDefaultVersionIfUnknown(myGeneratorVersion);
     }
     else {
       myVersionChecker = new SkeletonVersionChecker(myGeneratorVersion);
@@ -585,7 +590,9 @@ public class PySkeletonRefresher {
         boolean canLive = header != null;
         if (canLive) {
           final String binaryFile = header.getBinaryFile();
-          canLive = SkeletonVersionChecker.BUILTIN_NAME.equals(binaryFile) || mySkeletonsGenerator.exists(binaryFile);
+          canLive = SkeletonVersionChecker.PREGENERATED.equals(binaryFile) ||
+                    SkeletonVersionChecker.BUILTIN_NAME.equals(binaryFile) ||
+                    mySkeletonsGenerator.exists(binaryFile);
         }
         if (!canLive) {
           mySkeletonsGenerator.deleteOrLog(item);

python/src/com/jetbrains/python/sdk/skeletons/SkeletonVersionChecker.java

@@ -16,12 +16,11 @@
 package com.jetbrains.python.sdk.skeletons;
 
 import com.intellij.openapi.diagnostic.Logger;
-import com.jetbrains.python.PythonHelpersLocator;
 import com.intellij.psi.util.QualifiedName;
+import com.jetbrains.python.PythonHelpersLocator;
 import org.jetbrains.annotations.NonNls;
 
 import java.io.*;
-import java.util.Comparator;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.TreeMap;
@@ -41,9 +40,12 @@ public class SkeletonVersionChecker {
 
   final static Pattern ONE_LINE = Pattern.compile("^(?:(\\w+(?:\\.\\w+)*|\\(built-in\\)|\\(default\\))\\s+(\\d+\\.\\d+))?\\s*(?:#.*)?$");
 
+  public static final int PREGENERATED_VERSION = -1;
+
   @NonNls static final String REQUIRED_VERSION_FNAME = "required_gen_version";
   @NonNls static final String DEFAULT_NAME = "(default)"; // version required if a package is not explicitly mentioned
   @NonNls public static final String BUILTIN_NAME = "(built-in)"; // version required for built-ins
+  @NonNls public static final String PREGENERATED = "(pre-generated)"; // pre-generated skeleton
   private TreeMap<QualifiedName, Integer> myExplicitVersion; // versions of regularly named packages
   private Integer myDefaultVersion; // version of (default)
   private Integer myBuiltinsVersion; // version of (built-it)
@@ -57,6 +59,10 @@ public class SkeletonVersionChecker {
     load();
   }
 
+  public boolean isPregenerated() {
+    return myDefaultVersion == PREGENERATED_VERSION;
+  }
+
   private static TreeMap<QualifiedName, Integer> createTreeMap() {
     return new TreeMap<>((left, right) -> {
       Iterator<String> lefts = left.getComponents().iterator();

python/tools/src/com/jetbrains/python/tools/BuildZippedSkeletons.kt

@@ -56,7 +56,7 @@ fun main(args: Array<String>) {
       val refresher = PySkeletonRefresher(null, null, sdk, skeletonsDir.absolutePath, null, null)
 
 
-      refresher.regenerateSkeletons(SkeletonVersionChecker(0))
+      refresher.regenerateSkeletons(SkeletonVersionChecker(SkeletonVersionChecker.PREGENERATED_VERSION))
 
 
       val dirPacked = File(skeletonsDir.parent, DefaultPregeneratedSkeletonsProvider.getPregeneratedSkeletonsName(sdk, refresher.generatorVersion, true, true))