From 9d0117859fb441afae0f5a6f270ee2da9ecb40f8 Mon Sep 17 00:00:00 2001 From: Dmitry Trofimov Date: Fri, 3 Nov 2017 17:33:49 +0100 Subject: [PATCH] Don't remove pre-generated skeletons on clean-up (PY-26751) --- .../module_redeclarator.py | 8 ++++++-- .../python/sdk/skeletons/PySkeletonGenerator.java | 10 ++++++++++ .../python/sdk/skeletons/PySkeletonRefresher.java | 15 +++++++++++---- .../sdk/skeletons/SkeletonVersionChecker.java | 10 ++++++++-- .../python/tools/BuildZippedSkeletons.kt | 2 +- 5 files changed, 36 insertions(+), 9 deletions(-) diff --git a/python/helpers/pycharm_generator_utils/module_redeclarator.py b/python/helpers/pycharm_generator_utils/module_redeclarator.py index 5bde7733589e..321aa27f21a5 100644 --- a/python/helpers/pycharm_generator_utils/module_redeclarator.py +++ b/python/helpers/pycharm_generator_utils/module_redeclarator.py @@ -1,8 +1,10 @@ import keyword +import os -from pycharm_generator_utils.util_methods import * from pycharm_generator_utils.constants import * +from pycharm_generator_utils.util_methods import * +is_pregenerated = os.getenv("IS_PREGENERATED_SKELETONS", None) class emptylistdict(dict): """defaultdict not available before 2.5; simplest reimplementation using [] as default""" @@ -781,7 +783,9 @@ class ModuleRedeclarator(object): out(0, "# module ", p_name, mod_name) # line 2 BUILT_IN_HEADER = "(built-in)" - if self.mod_filename: + if is_pregenerated is not None: + filename = '(pre-generated)' + elif self.mod_filename: filename = self.mod_filename elif p_name in sys.builtin_module_names: filename = BUILT_IN_HEADER diff --git a/python/src/com/jetbrains/python/sdk/skeletons/PySkeletonGenerator.java b/python/src/com/jetbrains/python/sdk/skeletons/PySkeletonGenerator.java index 0a709a31de0b..7f793822a9e8 100644 --- a/python/src/com/jetbrains/python/sdk/skeletons/PySkeletonGenerator.java +++ b/python/src/com/jetbrains/python/sdk/skeletons/PySkeletonGenerator.java @@ -61,6 +61,8 @@ public class PySkeletonGenerator { private final String mySkeletonsPath; @NotNull protected final Map myEnv; + private boolean myPrebuilt = false; + public void finishSkeletonsGeneration() { } @@ -68,6 +70,10 @@ public class PySkeletonGenerator { return new File(name).exists(); } + public void setPrebuilt(boolean prebuilt) { + myPrebuilt = prebuilt; + } + public static class ListBinariesResult { public final int generatorVersion; public final Map modules; @@ -167,6 +173,10 @@ public class PySkeletonGenerator { final Map extraEnv = PythonSdkType.getVirtualEnvExtraEnv(binaryPath); final Map env = extraEnv != null ? PySdkUtil.mergeEnvVariables(myEnv, extraEnv) : myEnv; + if (myPrebuilt) { + env.put("IS_PREGENERATED_SKELETONS", "1"); + } + return getProcessOutput(parent_dir, ArrayUtil.toStringArray(commandLine), env, MINUTE * 10); } diff --git a/python/src/com/jetbrains/python/sdk/skeletons/PySkeletonRefresher.java b/python/src/com/jetbrains/python/sdk/skeletons/PySkeletonRefresher.java index b2c0fa352630..86611e0a2c19 100644 --- a/python/src/com/jetbrains/python/sdk/skeletons/PySkeletonRefresher.java +++ b/python/src/com/jetbrains/python/sdk/skeletons/PySkeletonRefresher.java @@ -270,7 +270,7 @@ public class PySkeletonRefresher { return mySkeletonsPath; } - public List regenerateSkeletons(@Nullable SkeletonVersionChecker cachedChecker) throws InvalidSdkException { + public List regenerateSkeletons(@Nullable SkeletonVersionChecker checker) throws InvalidSdkException { final List errorList = new SmartList<>(); final String homePath = mySdk.getHomePath(); final String skeletonsPath = getSkeletonsPath(); @@ -281,6 +281,10 @@ public class PySkeletonRefresher { } final String readablePath = FileUtil.getLocationRelativeToUserHome(homePath); + if (checker != null && checker.isPregenerated()) { + mySkeletonsGenerator.setPrebuilt(true); + } + mySkeletonsGenerator.prepare(); myBlacklist = loadBlacklist(); @@ -304,8 +308,9 @@ public class PySkeletonRefresher { myPregeneratedSkeletons = PyPregeneratedSkeletonsProvider.findPregeneratedSkeletonsForSdk(mySdk, myGeneratorVersion); indicate(PyBundle.message("sdk.gen.reading.versions.file")); - if (cachedChecker != null) { - myVersionChecker = cachedChecker.withDefaultVersionIfUnknown(myGeneratorVersion); + + if (checker != null) { + myVersionChecker = checker.withDefaultVersionIfUnknown(myGeneratorVersion); } else { myVersionChecker = new SkeletonVersionChecker(myGeneratorVersion); @@ -585,7 +590,9 @@ public class PySkeletonRefresher { boolean canLive = header != null; if (canLive) { final String binaryFile = header.getBinaryFile(); - canLive = SkeletonVersionChecker.BUILTIN_NAME.equals(binaryFile) || mySkeletonsGenerator.exists(binaryFile); + canLive = SkeletonVersionChecker.PREGENERATED.equals(binaryFile) || + SkeletonVersionChecker.BUILTIN_NAME.equals(binaryFile) || + mySkeletonsGenerator.exists(binaryFile); } if (!canLive) { mySkeletonsGenerator.deleteOrLog(item); diff --git a/python/src/com/jetbrains/python/sdk/skeletons/SkeletonVersionChecker.java b/python/src/com/jetbrains/python/sdk/skeletons/SkeletonVersionChecker.java index 4a9ce3e4f0ef..5d0ae49d7bc4 100644 --- a/python/src/com/jetbrains/python/sdk/skeletons/SkeletonVersionChecker.java +++ b/python/src/com/jetbrains/python/sdk/skeletons/SkeletonVersionChecker.java @@ -16,12 +16,11 @@ package com.jetbrains.python.sdk.skeletons; import com.intellij.openapi.diagnostic.Logger; -import com.jetbrains.python.PythonHelpersLocator; import com.intellij.psi.util.QualifiedName; +import com.jetbrains.python.PythonHelpersLocator; import org.jetbrains.annotations.NonNls; import java.io.*; -import java.util.Comparator; import java.util.Iterator; import java.util.Map; import java.util.TreeMap; @@ -41,9 +40,12 @@ public class SkeletonVersionChecker { final static Pattern ONE_LINE = Pattern.compile("^(?:(\\w+(?:\\.\\w+)*|\\(built-in\\)|\\(default\\))\\s+(\\d+\\.\\d+))?\\s*(?:#.*)?$"); + public static final int PREGENERATED_VERSION = -1; + @NonNls static final String REQUIRED_VERSION_FNAME = "required_gen_version"; @NonNls static final String DEFAULT_NAME = "(default)"; // version required if a package is not explicitly mentioned @NonNls public static final String BUILTIN_NAME = "(built-in)"; // version required for built-ins + @NonNls public static final String PREGENERATED = "(pre-generated)"; // pre-generated skeleton private TreeMap myExplicitVersion; // versions of regularly named packages private Integer myDefaultVersion; // version of (default) private Integer myBuiltinsVersion; // version of (built-it) @@ -57,6 +59,10 @@ public class SkeletonVersionChecker { load(); } + public boolean isPregenerated() { + return myDefaultVersion == PREGENERATED_VERSION; + } + private static TreeMap createTreeMap() { return new TreeMap<>((left, right) -> { Iterator lefts = left.getComponents().iterator(); diff --git a/python/tools/src/com/jetbrains/python/tools/BuildZippedSkeletons.kt b/python/tools/src/com/jetbrains/python/tools/BuildZippedSkeletons.kt index 0eb5bc9e6ceb..f83e561b60c3 100644 --- a/python/tools/src/com/jetbrains/python/tools/BuildZippedSkeletons.kt +++ b/python/tools/src/com/jetbrains/python/tools/BuildZippedSkeletons.kt @@ -56,7 +56,7 @@ fun main(args: Array) { val refresher = PySkeletonRefresher(null, null, sdk, skeletonsDir.absolutePath, null, null) - refresher.regenerateSkeletons(SkeletonVersionChecker(0)) + refresher.regenerateSkeletons(SkeletonVersionChecker(SkeletonVersionChecker.PREGENERATED_VERSION)) val dirPacked = File(skeletonsDir.parent, DefaultPregeneratedSkeletonsProvider.getPregeneratedSkeletonsName(sdk, refresher.generatorVersion, true, true))