From 66f20794ed5d08024667e4167226d22dc4369322 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Mon, 3 Sep 2012 21:21:07 +0400 Subject: [PATCH 1/2] Refactored skeleton header parsing --- .../python/sdk/PySkeletonRefresher.java | 80 +++++++++++-------- 1 file changed, 47 insertions(+), 33 deletions(-) diff --git a/python/src/com/jetbrains/python/sdk/PySkeletonRefresher.java b/python/src/com/jetbrains/python/sdk/PySkeletonRefresher.java index ee85b2c2a5ad..35558ffebbe7 100644 --- a/python/src/com/jetbrains/python/sdk/PySkeletonRefresher.java +++ b/python/src/com/jetbrains/python/sdk/PySkeletonRefresher.java @@ -55,8 +55,8 @@ public class PySkeletonRefresher { private final static Pattern BLACKLIST_LINE = Pattern.compile("^([^=]+) = (\\d+\\.\\d+) (\\d+)\\s*$"); // we use the equals sign after filename so that we can freely include space in the filename - private final static Pattern OUR_VERSION_LINE = Pattern.compile("# from (\\S+) by generator (\\S+)\\s*"); - + // XXX: Path (the first component) may contain spaces, this header spec is deprecated + private static final Pattern VERSION_LINE_V1 = Pattern.compile("# from (\\S+) by generator (\\S+)\\s*"); private String myExtraSyspath; private VirtualFile myPregeneratedSkeletons; @@ -148,15 +148,6 @@ public class PySkeletonRefresher { return mySkeletonsPath; } - @Nullable - private static Integer getSkeletonVersion(File file) { - final Matcher headerMatcher = getParseHeader(file); - if (headerMatcher != null && headerMatcher.matches()) { - return fromVersionString(headerMatcher.group(2)); - } - return null; - } - List regenerateSkeletons(@Nullable SkeletonVersionChecker cachedChecker, @Nullable Ref migrationFlag) throws InvalidSdkException { final List errorList = new SmartList(); @@ -190,7 +181,8 @@ public class PySkeletonRefresher { final String builtinsFileName = PythonSdkType.getBuiltinsFileName(mySdk); final File builtinsFile = new File(skeletonsPath, builtinsFileName); - final boolean oldOrNonExisting = getSkeletonVersion(builtinsFile) == null; + final SkeletonHeader oldHeader = readSkeletonHeader(builtinsFile); + final boolean oldOrNonExisting = oldHeader == null || oldHeader.getVersion() == 0; if (migrationFlag != null && !migrationFlag.get() && oldOrNonExisting) { migrationFlag.set(true); @@ -246,8 +238,8 @@ public class PySkeletonRefresher { } } - final Integer builtinVersion = getSkeletonVersion(builtinsFile); - if (myPregeneratedSkeletons == null && (builtinVersion == null || builtinVersion < myVersionChecker.getBuiltinVersion())) { + final SkeletonHeader newHeader = readSkeletonHeader(builtinsFile); + if (myPregeneratedSkeletons == null && (newHeader == null || newHeader.getVersion() < myVersionChecker.getBuiltinVersion())) { indicate(PyBundle.message("sdk.gen.updating.builtins.$0", readablePath)); mySkeletonsGenerator.generateBuiltinSkeletons(mySdk); } @@ -284,27 +276,52 @@ public class PySkeletonRefresher { } @Nullable - private static Matcher getParseHeader(File infile) { + private static SkeletonHeader readSkeletonHeader(@NotNull File file) { try { - Reader input = new FileReader(infile); - LineNumberReader lines = new LineNumberReader(input); + final LineNumberReader reader = new LineNumberReader(new FileReader(file)); try { String line = null; - for (int i = 0; i < 3; i += 1) { // read three lines, skip first two - line = lines.readLine(); - if (line == null) return null; + // Read 3 lines, skip first 2: encoding, module name + for (int i = 0; i < 3; i++) { + line = reader.readLine(); + if (line == null) { + return null; + } + } + // Try the old whitespace-unsafe header format v1 first + final Matcher matcher = VERSION_LINE_V1.matcher(line); + if (matcher.matches()) { + return new SkeletonHeader(matcher.group(1), fromVersionString(matcher.group(2))); } - return OUR_VERSION_LINE.matcher(line); } finally { - lines.close(); + reader.close(); } } - catch (IOException ignore) { + catch (IOException e) { } return null; } + static class SkeletonHeader { + @NotNull private final String myFile; + private final int myVersion; + + public SkeletonHeader(@NotNull String binaryFile, int version) { + myFile = binaryFile; + myVersion = version; + } + + @NotNull + public String getBinaryFile() { + return myFile; + } + + public int getVersion() { + return myVersion; + } + } + private Map> loadBlacklist() { Map> ret = new HashMap>(); File blacklistFile = new File(mySkeletonsPath, BLACKLIST_FILE_NAME); @@ -420,12 +437,11 @@ public class PySkeletonRefresher { final String itemName = item.getName(); if (PyNames.INIT_DOT_PY.equals(itemName) && item.length() == 0) continue; // these are versionless if (BLACKLIST_FILE_NAME.equals(itemName)) continue; // don't touch the blacklist - Matcher headerMatcher = getParseHeader(item); - boolean canLive = headerMatcher != null && headerMatcher.matches(); + final SkeletonHeader header = readSkeletonHeader(item); + boolean canLive = header != null; if (canLive) { - String sourceName = headerMatcher.group(1); - canLive = - sourceName != null && (SkeletonVersionChecker.BUILTIN_NAME.equals(sourceName) || mySkeletonsGenerator.exists(sourceName)); + final String binaryFile = header.getBinaryFile(); + canLive = SkeletonVersionChecker.BUILTIN_NAME.equals(binaryFile) || mySkeletonsGenerator.exists(binaryFile); } if (!canLive) { mySkeletonsGenerator.deleteOrLog(item); @@ -524,13 +540,11 @@ public class PySkeletonRefresher { final String moduleName = binaryItem.getModule(); final File skeleton = getSkeleton(moduleName, getSkeletonsPath()); - - Matcher matcher = getParseHeader(skeleton); + final SkeletonHeader header = readSkeletonHeader(skeleton); boolean mustRebuild = true; // guilty unless proven fresh enough - if (matcher != null && matcher.matches()) { - int fileVersion = fromVersionString(matcher.group(2)); + if (header != null) { int requiredVersion = myVersionChecker.getRequiredVersion(moduleName); - mustRebuild = fileVersion < requiredVersion; + mustRebuild = header.getVersion() < requiredVersion; } if (!mustRebuild) { // ...but what if the lib was updated? mustRebuild = (skeleton.exists() && binaryItem.lastModified() > skeleton.lastModified()); From c581d9dc988809fb8e8c9e346f86af8a4c8903d8 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Mon, 3 Sep 2012 21:43:12 +0400 Subject: [PATCH 2/2] Fixed deleting up-to-date skeletons when their path contains whitespaces (PY-7421) --- python/helpers/generator3.py | 10 ++++---- .../python/sdk/PySkeletonRefresher.java | 24 +++++++++++++++---- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/python/helpers/generator3.py b/python/helpers/generator3.py index f2ab969f76cf..1f78bcb8942f 100644 --- a/python/helpers/generator3.py +++ b/python/helpers/generator3.py @@ -27,8 +27,6 @@ but seemingly no one uses them in C extensions yet anyway. VERSION = "1.116" # Must be a number-dot-number string, updated with each change that affects generated skeletons # Note: DON'T FORGET TO UPDATE! -VERSION_CONTROL_HEADER_FORMAT = '# from %s by generator %s' - import sys import os import string @@ -1836,9 +1834,8 @@ class ModuleRedeclarator(object): else: mod_name = " does not know its name" out(0, "# module ", p_name, mod_name) # line 2 - out(0, VERSION_CONTROL_HEADER_FORMAT % ( - self.mod_filename or getattr(self.module, "__file__", "(built-in)"), VERSION) - ) # line 3 + out(0, "# from %s" % self.mod_filename or getattr(self.module, "__file__", "(built-in)")) # line 3 + out(0, "# by generator %s" % VERSION) # line 4 if p_name == BUILTIN_MOD_NAME and version[0] == 2 and version[1] >= 6: out(0, "from __future__ import print_function") self.outDocAttr(out, self.module, 0) @@ -2485,7 +2482,8 @@ def processOne(name, mod_file_name, doing_builtins): if outfile is not None and not outfile.closed: outfile.write("# encoding: %s\n" % OUT_ENCODING) outfile.write("# module %s\n" % name) - outfile.write(VERSION_CONTROL_HEADER_FORMAT % (mod_file_name, VERSION)) + outfile.write("# from %s\n" % mod_file_name) + outfile.write("# by generator %s\n" % VERSION) outfile.write("\n\n") outfile.write("# Skeleton generation error:\n#\n# " + (msg % args) + "\n") if debug_mode: diff --git a/python/src/com/jetbrains/python/sdk/PySkeletonRefresher.java b/python/src/com/jetbrains/python/sdk/PySkeletonRefresher.java index 35558ffebbe7..2f22d67c224f 100644 --- a/python/src/com/jetbrains/python/sdk/PySkeletonRefresher.java +++ b/python/src/com/jetbrains/python/sdk/PySkeletonRefresher.java @@ -55,9 +55,13 @@ public class PySkeletonRefresher { private final static Pattern BLACKLIST_LINE = Pattern.compile("^([^=]+) = (\\d+\\.\\d+) (\\d+)\\s*$"); // we use the equals sign after filename so that we can freely include space in the filename - // XXX: Path (the first component) may contain spaces, this header spec is deprecated + // Path (the first component) may contain spaces, this header spec is deprecated private static final Pattern VERSION_LINE_V1 = Pattern.compile("# from (\\S+) by generator (\\S+)\\s*"); + // Skeleton header spec v2 + private static final Pattern FROM_LINE_V2 = Pattern.compile("# from (.*)$"); + private static final Pattern BY_LINE_V2 = Pattern.compile("# by generator (.*)$"); + private String myExtraSyspath; private VirtualFile myPregeneratedSkeletons; private int myGeneratorVersion; @@ -289,9 +293,21 @@ public class PySkeletonRefresher { } } // Try the old whitespace-unsafe header format v1 first - final Matcher matcher = VERSION_LINE_V1.matcher(line); - if (matcher.matches()) { - return new SkeletonHeader(matcher.group(1), fromVersionString(matcher.group(2))); + final Matcher v1Matcher = VERSION_LINE_V1.matcher(line); + if (v1Matcher.matches()) { + return new SkeletonHeader(v1Matcher.group(1), fromVersionString(v1Matcher.group(2))); + } + final Matcher fromMatcher = FROM_LINE_V2.matcher(line); + if (fromMatcher.matches()) { + final String binaryFile = fromMatcher.group(1); + line = reader.readLine(); + if (line != null) { + final Matcher byMatcher = BY_LINE_V2.matcher(line); + if (byMatcher.matches()) { + final int version = fromVersionString(byMatcher.group(1)); + return new SkeletonHeader(binaryFile, version); + } + } } } finally {