diff --git a/python/helpers/generator3.py b/python/helpers/generator3.py index 55190a83a5b7..b737a3def8ff 100644 --- a/python/helpers/generator3.py +++ b/python/helpers/generator3.py @@ -2432,8 +2432,11 @@ if __name__ == "__main__": names.append(BUILTIN_MOD_NAME) if '__main__' in names: names.remove('__main__') # we don't want ourselves processed + ok = True for name in names: - processOne(name, None, True) + ok = ok and processOne(name, None, True) + if not ok: + sys.exit(1) else: if len(args) > 2: diff --git a/python/helpers/pycharm/django_test_manage.py b/python/helpers/pycharm/django_test_manage.py index 113b1b95d799..9723dd24d630 100644 --- a/python/helpers/pycharm/django_test_manage.py +++ b/python/helpers/pycharm/django_test_manage.py @@ -13,6 +13,8 @@ settings_file = os.getenv('PYCHARM_DJANGO_SETTINGS_MODULE') or os.getenv('DJANGO if not settings_file: settings_file = 'settings' +os.environ.setdefault('DJANGO_SETTINGS_MODULE') + from django.core import management from django.core.management.commands.test import Command from django.conf import settings diff --git a/python/helpers/pydev/pydev_ipython_console.py b/python/helpers/pydev/pydev_ipython_console.py index ec318d00d7a7..7bb0049ad3de 100644 --- a/python/helpers/pydev/pydev_ipython_console.py +++ b/python/helpers/pydev/pydev_ipython_console.py @@ -119,7 +119,7 @@ class InterpreterInterface(BaseInterpreterInterface): server = self.get_server() if server is not None: - server.NotifyAboutMagic(magic_commands, self.interpreter.ipython.automagic) + server.NotifyAboutMagic(magic_commands, self.interpreter.is_automagic()) diff --git a/python/helpers/pydev/pydev_ipython_console_010.py b/python/helpers/pydev/pydev_ipython_console_010.py index 1c2a11a3dfb4..37a45abcba33 100644 --- a/python/helpers/pydev/pydev_ipython_console_010.py +++ b/python/helpers/pydev/pydev_ipython_console_010.py @@ -85,3 +85,13 @@ class PyDevFrontEnd(PrefilterFrontEnd): return True #needs more return False #execute complete (no more) + + def update(self, globals, locals): + locals['_oh'] = self.shell.user_ns['_oh'] + locals['_ip'] = self.shell.user_ns['_ip'] + self.shell.user_global_ns = globals + self.shell.user_ns = locals + + def is_automagic(self): + return True if self.ipython0.rc.automagic else False + diff --git a/python/helpers/pydev/pydev_ipython_console_011.py b/python/helpers/pydev/pydev_ipython_console_011.py index 8561a6a124ba..7c235842a609 100644 --- a/python/helpers/pydev/pydev_ipython_console_011.py +++ b/python/helpers/pydev/pydev_ipython_console_011.py @@ -131,3 +131,6 @@ class PyDevFrontEnd: # print(rep) return False #execute complete (no more) + def is_automagic(self): + return self.ipython.automagic + diff --git a/python/helpers/pydev/pydevconsole.py b/python/helpers/pydev/pydevconsole.py index 42e5dcf73477..07492bee5184 100644 --- a/python/helpers/pydev/pydevconsole.py +++ b/python/helpers/pydev/pydevconsole.py @@ -253,7 +253,10 @@ def start_server(host, port, interpreter): server.register_function(handshake) if IPYTHON: - interpreter.interpreter.ipython.hooks.editor = ipython_editor(interpreter) + try: + interpreter.interpreter.ipython.hooks.editor = ipython_editor(interpreter) + except : + pass server.serve_forever() diff --git a/python/helpers/pydev/pydevd_comm.py b/python/helpers/pydev/pydevd_comm.py index 3ebebfad84ea..c1311a6dc094 100644 --- a/python/helpers/pydev/pydevd_comm.py +++ b/python/helpers/pydev/pydevd_comm.py @@ -310,7 +310,7 @@ class WriterThread(PyDBDaemonThread): def addCommand(self, cmd): """ cmd is NetCommand """ self.cmdQueue.put(cmd) - + def OnRun(self): """ just loop and write responses """ diff --git a/python/pluginSrc/META-INF/plugin.xml b/python/pluginSrc/META-INF/plugin.xml index 0152e4cf5629..dcd0137f452e 100644 --- a/python/pluginSrc/META-INF/plugin.xml +++ b/python/pluginSrc/META-INF/plugin.xml @@ -4,7 +4,7 @@ Pythonid Python - + Smart editing for Python scripts 2.8.0 com.intellij.modules.ultimate diff --git a/python/skeletons/skeletons-mac-355-10.6-python-2.6.1.zip b/python/skeletons/skeletons-mac-355-10.6-python-2.6.1.zip new file mode 100644 index 000000000000..43659878e83a Binary files /dev/null and b/python/skeletons/skeletons-mac-355-10.6-python-2.6.1.zip differ diff --git a/python/src/com/jetbrains/python/actions/GenerateBinaryStubsFix.java b/python/src/com/jetbrains/python/actions/GenerateBinaryStubsFix.java index 499b3766e3c9..5259af0e0af8 100644 --- a/python/src/com/jetbrains/python/actions/GenerateBinaryStubsFix.java +++ b/python/src/com/jetbrains/python/actions/GenerateBinaryStubsFix.java @@ -63,7 +63,7 @@ public class GenerateBinaryStubsFix implements LocalQuickFix { final PySkeletonRefresher refresher = new PySkeletonRefresher(mySdk, null, null); refresher.generateSkeleton(myQualifiedName, "", assemblyRefs); final VirtualFile skeletonDir; - skeletonDir = LocalFileSystem.getInstance().findFileByPath(refresher.getSkeletonPath()); + skeletonDir = LocalFileSystem.getInstance().findFileByPath(refresher.getSkeletonsPath()); if (skeletonDir != null) { skeletonDir.refresh(true, true); } diff --git a/python/src/com/jetbrains/python/codeInsight/stdlib/StdlibTypes2.properties b/python/src/com/jetbrains/python/codeInsight/stdlib/StdlibTypes2.properties index 6c0fd3cda3e7..f44c59b3c64b 100644 --- a/python/src/com/jetbrains/python/codeInsight/stdlib/StdlibTypes2.properties +++ b/python/src/com/jetbrains/python/codeInsight/stdlib/StdlibTypes2.properties @@ -31,7 +31,7 @@ __builtin__.dir = \ __builtin__.divmod = \ :type x: int or long or float or complex \n\ :type y: int or long or float or complex \n\ - :rtype: (int or long or float or complex, int or long or float or complex) \n\ + :rtype: (int or long or float, int or long or float) \n\ __builtin__.enumerate.__init__ = \ :type iterable: collections.Iterable \n\ @@ -89,7 +89,7 @@ __builtin__.ord = \ __builtin__.pow = \ :type x: int or long or float or complex \n\ :type y: int or long or float or complex \n\ - :rtype: int or long or float or complex \n\ + :rtype: int or long or float \n\ __builtin__.range = \ :type start: int or None \n\ diff --git a/python/src/com/jetbrains/python/inspections/PyStringFormatInspection.java b/python/src/com/jetbrains/python/inspections/PyStringFormatInspection.java index ed00297945b6..031a11c491c6 100644 --- a/python/src/com/jetbrains/python/inspections/PyStringFormatInspection.java +++ b/python/src/com/jetbrains/python/inspections/PyStringFormatInspection.java @@ -44,12 +44,12 @@ public class PyStringFormatInspection extends PyInspection { public static class Visitor extends PyInspectionVisitor { private static class Inspection { private static final ImmutableMap FORMAT_CONVERSIONS = ImmutableMap.builder() - .put('d', "int or long") - .put('i', "int or long") - .put('o', "int or long") - .put('u', "int or long") - .put('x', "int or long") - .put('X', "int or long") + .put('d', "int or long or float") + .put('i', "int or long or float") + .put('o', "int or long or float") + .put('u', "int or long or float") + .put('x', "int or long or float") + .put('X', "int or long or float") .put('e', "float") .put('E', "float") .put('f', "float") diff --git a/python/src/com/jetbrains/python/psi/impl/PyReferenceImpl.java b/python/src/com/jetbrains/python/psi/impl/PyReferenceImpl.java index 019eca8a489a..6952073a52da 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyReferenceImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyReferenceImpl.java @@ -358,8 +358,8 @@ public class PyReferenceImpl implements PsiReferenceEx, PsiPolyVariantReference // TODO: Cython-dependent code without CythonLanguageDialect.isInsideCythonFile() check if (element instanceof PyParameter || element instanceof PyTargetExpression || element instanceof CythonVariable) { // Check if the reference is in the same or inner scope of the element scope, not shadowed by an intermediate declaration - PsiElement ourContainer = PsiTreeUtil.getParentOfType(getElement(), PsiNamedElement.class, PyLambdaExpression.class, PyComprehensionElement.class); - PsiElement theirContainer = PsiTreeUtil.getParentOfType(element, PsiNamedElement.class, PyLambdaExpression.class, PyComprehensionElement.class); + PsiElement ourContainer = PsiTreeUtil.getParentOfType(getElement(), ScopeOwner.class, PyComprehensionElement.class); + PsiElement theirContainer = PsiTreeUtil.getParentOfType(element, ScopeOwner.class, PyComprehensionElement.class); if (ourContainer != null) { if (ourContainer == theirContainer) { return true; diff --git a/python/src/com/jetbrains/python/sdk/PySkeletonRefresher.java b/python/src/com/jetbrains/python/sdk/PySkeletonRefresher.java index 2197f056457f..5f9e0e7ad9b9 100644 --- a/python/src/com/jetbrains/python/sdk/PySkeletonRefresher.java +++ b/python/src/com/jetbrains/python/sdk/PySkeletonRefresher.java @@ -61,6 +61,16 @@ public class PySkeletonRefresher { private Map> myBlacklist; private SkeletonVersionChecker myVersionChecker; + private static class ListBinariesResult { + public final int generatorVersion; + public final Map modules; + + ListBinariesResult(int generatorVersion, Map modules) { + this.generatorVersion = generatorVersion; + this.modules = modules; + } + } + /** * Creates a new object that refreshes skeletons of given SDK. * @param sdk a Python SDK @@ -72,7 +82,7 @@ public class PySkeletonRefresher { mySdk = sdk; mySkeletonsPath = skeletonsPath; } - + private void indicate(String msg) { if (myIndicator != null) { myIndicator.checkCanceled(); @@ -93,23 +103,24 @@ public class PySkeletonRefresher { } } - private String getExtraSyspath() { - if (myExtraSyspath == null) { - VirtualFile[] class_dirs = mySdk.getRootProvider().getFiles(OrderRootType.CLASSES); - StringBuilder arg_builder = new StringBuilder("\""); - int i = 0; - while (i < class_dirs.length) { - if (i > 0) arg_builder.append(File.pathSeparator); - if (class_dirs[i].isInLocalFileSystem()) { - final String pathname = class_dirs[i].getPath(); - if (!mySkeletonsPath.equals(pathname)) arg_builder.append(pathname); - } - i += 1; + private static String getExtraSysPath(Sdk sdk, String skeletonsPath) { + final VirtualFile[] classDirs = sdk.getRootProvider().getFiles(OrderRootType.CLASSES); + final StringBuilder builder = new StringBuilder("\""); + int i = 0; + while (i < classDirs.length) { + if (i > 0) { + builder.append(File.pathSeparator); } - arg_builder.append("\""); - myExtraSyspath = arg_builder.toString(); + if (classDirs[i].isInLocalFileSystem()) { + final String pathname = classDirs[i].getPath(); + if (!skeletonsPath.equals(pathname)) { + builder.append(pathname); + } + } + i += 1; } - return myExtraSyspath; + builder.append("\""); + return builder.toString(); } /** @@ -117,7 +128,7 @@ public class PySkeletonRefresher { * @return path name of skeleton dir for the SDK, guaranteed to be already created. */ @NotNull - public String getSkeletonPath() { + public String getSkeletonsPath() { if (mySkeletonsPath == null) { mySkeletonsPath = PythonSdkType.getSkeletonsPath(mySdk.getHomePath()); final File skeletonsDir = new File(mySkeletonsPath); @@ -132,14 +143,22 @@ public class PySkeletonRefresher { return regenerateSkeletons(cached_checker, null); } - List regenerateSkeletons( - @Nullable SkeletonVersionChecker cached_checker, - @Nullable Ref migration_flag - ) { - List error_list = new SmartList(); - String home_path = mySdk.getHomePath(); - final File skel_dir = new File(mySkeletonsPath); - if (!skel_dir.exists()) skel_dir.mkdirs(); + @Nullable + private static Integer getSkeletonVersion(File file) { + final Matcher headerMatcher = getParseHeader(file); + if (headerMatcher != null && headerMatcher.matches()) { + return fromVersionString(headerMatcher.group(2)); + } + return null; + } + + List regenerateSkeletons(@Nullable SkeletonVersionChecker cached_checker, + @Nullable Ref migration_flag) { + final List error_list = new SmartList(); + final String home_path = mySdk.getHomePath(); + final String skeletonsPath = getSkeletonsPath(); + final File skeletonsDir = new File(skeletonsPath); + if (!skeletonsDir.exists()) skeletonsDir.mkdirs(); final String readable_path = PythonSdkType.shortenDirName(home_path); myBlacklist = loadBlacklist(); @@ -147,11 +166,8 @@ public class PySkeletonRefresher { indicate(PyBundle.message("sdk.gen.querying.$0", readable_path)); // get generator version and binary libs list in one go - final List binaries_output = listBinaries(); - if (binaries_output.size() < 1) { - throw new InvalidSdkException("Empty output from " + GENERATOR3 + " for " + home_path); - } - myGeneratorVersion = fromVersionString(binaries_output.get(0).trim()); + final ListBinariesResult binaries = listBinaries(mySdk, getExtraSysPath(mySdk, getSkeletonsPath())); + myGeneratorVersion = binaries.generatorVersion; myPregeneratedSkeletons = findPregeneratedSkeletons(); indicate(PyBundle.message("sdk.gen.reading.versions.file")); @@ -159,12 +175,11 @@ public class PySkeletonRefresher { else myVersionChecker = new SkeletonVersionChecker(myGeneratorVersion); // check builtins - String builtinsFileName = PythonSdkType.getBuiltinsFileName(mySdk); - File builtins_file = new File(skel_dir, builtinsFileName); + final String builtinsFileName = PythonSdkType.getBuiltinsFileName(mySdk); + final File builtinsFile = new File(skeletonsPath, builtinsFileName); + + final boolean oldOrNonExisting = getSkeletonVersion(builtinsFile) == null; - Matcher header_matcher = getParseHeader(builtins_file); - final boolean oldOrNonExisting = header_matcher == null || // no file - !header_matcher.matches(); // no version line if (migration_flag != null && !migration_flag.get() && oldOrNonExisting) { migration_flag.set(true); Notifications.Bus.notify( @@ -177,28 +192,60 @@ public class PySkeletonRefresher { } if (myPregeneratedSkeletons != null && oldOrNonExisting) { - LOG.info("Unpacking pregenerated skeletons..."); + indicate("Unpacking pregenerated skeletons..."); try { - ZipUtil.extract(new File(JarFileSystem.getInstance().getVirtualFileForJar(myPregeneratedSkeletons).getPath()), - new File(mySkeletonsPath), null); + final VirtualFile jar = JarFileSystem.getInstance().getVirtualFileForJar(myPregeneratedSkeletons); + if (jar != null) { + ZipUtil.extract(new File(jar.getPath()), + new File(getSkeletonsPath()), null); + } } catch (IOException e) { LOG.info("Error unpacking pregenerated skeletons", e); } } - - if (myPregeneratedSkeletons == null && (oldOrNonExisting || fromVersionString(header_matcher.group(2)) < myVersionChecker.getBuiltinVersion())) { + + if (oldOrNonExisting) { + final Sdk base = PythonSdkType.getInstance().getVirtualEnvBaseSdk(mySdk); + if (base != null) { + indicate("Copying base SDK skeletons for virtualenv..."); + final String baseSkeletonsPath = PythonSdkType.getSkeletonsPath(base.getHomePath()); + final ListBinariesResult baseBinaries = listBinaries(base, getExtraSysPath(base, baseSkeletonsPath)); + for (Map.Entry entry : binaries.modules.entrySet()) { + final String module = entry.getKey(); + final File binary = entry.getValue(); + final File baseBinary = baseBinaries.modules.get(module); + final File fromFile = getSkeleton(module, baseSkeletonsPath); + if (baseBinaries.modules.containsKey(module) && + fromFile.exists() && + binary.length() == baseBinary.length()) { // Weak binary modules equality check + final File toFile = fromFile.isDirectory() ? + getPackageSkeleton(module, skeletonsPath) : + getModuleSkeleton(module, skeletonsPath); + try { + FileUtil.copy(fromFile, toFile); + } + catch (IOException e) { + LOG.info("Error copying base virtualenv SDK skeleton for " + module, e); + } + } + } + } + } + + final Integer builtinVersion = getSkeletonVersion(builtinsFile); + if (myPregeneratedSkeletons == null && (builtinVersion == null || builtinVersion < myVersionChecker.getBuiltinVersion())) { indicate(PyBundle.message("sdk.gen.updating.builtins.$0", readable_path)); generateBuiltinSkeletons(); } if (!oldOrNonExisting) { indicate(PyBundle.message("sdk.gen.cleaning.$0", readable_path)); - cleanUpSkeletons(skel_dir); + cleanUpSkeletons(skeletonsDir); } indicate(PyBundle.message("sdk.gen.updating.$0", readable_path)); - List skel_errors = updateOrCreateSkeletons(binaries_output); + List skel_errors = updateOrCreateSkeletons(binaries.modules); if (skel_errors.size() > 0) { indicateMinor(BLACKLIST_FILE_NAME); @@ -206,9 +253,9 @@ public class PySkeletonRefresher { if (error.isFresh()) error_list.add(error.getName()); myBlacklist.put(error.getPath(), new Pair(myGeneratorVersion, error.getTimestamp())); } - storeBlacklist(skel_dir, myBlacklist); + storeBlacklist(skeletonsDir, myBlacklist); } - else removeBlacklist(skel_dir); + else removeBlacklist(skeletonsDir); indicate(PyBundle.message("sdk.gen.reloading")); VirtualFile skeletonsVFile = LocalFileSystem.getInstance().refreshAndFindFileByPath(mySkeletonsPath); @@ -217,29 +264,53 @@ public class PySkeletonRefresher { return error_list; } - private List listBinaries() { - String homePath = mySdk.getHomePath(); - final String parent_dir = new File(homePath).getParent(); - long start_time = System.currentTimeMillis(); - final ProcessOutput run_result = SdkUtil.getProcessOutput( - parent_dir, - new String[]{homePath, PythonHelpersLocator.getHelperPath(GENERATOR3), "-v", "-L", "-s", getExtraSyspath()}, - PythonSdkType.getVirtualEnvAdditionalEnv(homePath), - MINUTE * 4 // see PY-3898 - ); - LOG.info("Retrieving binary module list took " + (System.currentTimeMillis() - start_time) + " ms"); - if (run_result.getExitCode() != 0) { - StringBuilder sb = new StringBuilder("failed to run ").append(GENERATOR3).append(" for ").append(homePath); - if (run_result.isTimeout()) sb.append(": timed out."); + @NotNull + private static ListBinariesResult listBinaries(Sdk sdk, String extraSysPath) { + final String homePath = sdk.getHomePath(); + final String parentDir = new File(homePath).getParent(); + final long startTime = System.currentTimeMillis(); + final String[] cmd = new String[] {homePath, PythonHelpersLocator.getHelperPath(GENERATOR3), "-v", "-L", "-s", extraSysPath}; + final ProcessOutput process = SdkUtil.getProcessOutput(parentDir, + cmd, + PythonSdkType.getVirtualEnvAdditionalEnv(homePath), + MINUTE * 4); // see PY-3898 + LOG.info("Retrieving binary module list took " + (System.currentTimeMillis() - startTime) + " ms"); + if (process.getExitCode() != 0) { + final StringBuilder sb = new StringBuilder("failed to run ").append(GENERATOR3).append(" for ").append(homePath); + if (process.isTimeout()) { + sb.append(": timed out."); + } else { - sb.append(", exit code ").append(run_result.getExitCode()).append(", stderr: \n-----\n"); - for (String err_line : run_result.getStderrLines()) sb.append(err_line).append("\n"); + sb.append(", exit code ") + .append(process.getExitCode()) + .append(", stderr: \n-----\n"); + for (String line : process.getStderrLines()) { + sb.append(line).append("\n"); + } sb.append("-----"); } throw new InvalidSdkException(sb.toString()); } - // stdout contains version in the first line and then the list of binaries - return run_result.getStdoutLines(); + final List lines = process.getStdoutLines(); + if (lines.size() < 1) { + throw new InvalidSdkException("Empty output from " + GENERATOR3 + " for " + homePath); + } + final Iterator iter = lines.iterator(); + final int generatorVersion = fromVersionString(iter.next().trim()); + final Map binaries = new HashMap(); + while (iter.hasNext()) { + final String line = iter.next(); + int cutpos = line.indexOf(' '); + if (cutpos >= 0) { + String moduleName = line.substring(0, cutpos); + String path = line.substring(cutpos + 1); + binaries.put(moduleName, new File(path)); + } + else { + LOG.error("Bad binaries line: '" + line + "', SDK " + homePath); // but don't die yet + } + } + return new ListBinariesResult(generatorVersion, binaries); } static final Pattern ourVersionLinePat = Pattern.compile("# from (\\S+) by generator (\\S+)\\s*"); @@ -442,51 +513,56 @@ public class PySkeletonRefresher { * (Re-)generates skeletons for all binary python modules. Up-to-date skeletons are not regenerated. * Does one module at a time: slower, but avoids certain conflicts. * - * @param binaries output of generator3 -L, list of prospective binary modules + * @param modules output of generator3 -L * @return blacklist data; whatever was not generated successfully is put here. */ - private List updateOrCreateSkeletons(List binaries) { - final List error_list = new SmartList(); - final Iterator bin_iter = binaries.iterator(); - bin_iter.next(); // skip version number. if it weren't here, we'd already die up in regenerateSkeletons() - final int count = Math.max(0, binaries.size() - 1); - int i = 0; - while (bin_iter.hasNext()) { + private List updateOrCreateSkeletons(Map modules) { + final List names = new ArrayList(modules.keySet()); + Collections.sort(names); + final List results = new ArrayList(); + final int count = names.size(); + for (int i = 0; i < count; i++) { checkCanceled(); if (myIndicator != null) { myIndicator.setFraction((double)i / count); } - final String line = bin_iter.next(); // line = "mod_name path" - int cutpos = line.indexOf(' '); - if (cutpos < 0) LOG.error("Bad binaries line: '" + line + "', SDK " + mySdk.getHomePath()); // but don't die yet - else { - String moduleName = line.substring(0, cutpos); - String module_lib_name = line.substring(cutpos+1); - if (updateOrCreateSkeleton(moduleName, module_lib_name, error_list)) continue; + final String name = names.get(i); + final File module = modules.get(name); + if (module != null) { + updateOrCreateSkeleton(name, module.getPath(), results); } - i++; } - return error_list; + return results; + } + + private static File getSkeleton(String moduleName, String skeletonsPath) { + final File module = getModuleSkeleton(moduleName, skeletonsPath); + return module.exists() ? module : getPackageSkeleton(moduleName, skeletonsPath); + } + + private static File getModuleSkeleton(String module, String skeletonsPath) { + final String modulePath = module.replace('.', '/'); + return new File(skeletonsPath, modulePath + ".py"); + } + + private static File getPackageSkeleton(String pkg, String skeletonsPath) { + final String packagePath = pkg.replace('.', '/'); + return new File(new File(skeletonsPath, packagePath), PyNames.INIT_DOT_PY); } private boolean updateOrCreateSkeleton(String moduleName, String moduleLibName, List error_list) { - final String modulePath = moduleName.replace('.', '/'); - String skeletonPath = getSkeletonPath(); // will create dirs as needed - File skeleton_file = new File(skeletonPath, modulePath + ".py"); - if (!skeleton_file.exists()) { - skeleton_file = new File(new File(skeletonPath, modulePath), PyNames.INIT_DOT_PY); - } - File lib_file = new File(moduleLibName); - Matcher matcher = getParseHeader(skeleton_file); + final File skeleton = getSkeleton(moduleName, getSkeletonsPath()); + final File binary = new File(moduleLibName); + Matcher matcher = getParseHeader(skeleton); boolean must_rebuild = true; // guilty unless proven fresh enough if (matcher != null && matcher.matches()) { int file_version = SkeletonVersionChecker.fromVersionString(matcher.group(2)); int required_version = myVersionChecker.getRequiredVersion(moduleName); must_rebuild = file_version < required_version; } - final long lib_file_timestamp = lib_file.lastModified(); + final long lib_file_timestamp = binary.lastModified(); if (!must_rebuild) { // ...but what if the lib was updated? - must_rebuild = (lib_file.exists() && skeleton_file.exists() && lib_file_timestamp > skeleton_file.lastModified()); + must_rebuild = (binary.exists() && skeleton.exists() && lib_file_timestamp > skeleton.lastModified()); // really we can omit both exists() calls but I keep these to make the logic clear } if (myBlacklist != null) { @@ -516,7 +592,7 @@ public class PySkeletonRefresher { private boolean copyPregeneratedSkeleton(String moduleName) { File targetDir; final String modulePath = moduleName.replace('.', '/'); - File skeletonsDir = new File(getSkeletonPath()); + File skeletonsDir = new File(getSkeletonsPath()); VirtualFile pregenerated = myPregeneratedSkeletons.findFileByRelativePath(modulePath + ".py"); if (pregenerated == null) { pregenerated = myPregeneratedSkeletons.findFileByRelativePath(modulePath + "/" + PyNames.INIT_DOT_PY); @@ -560,7 +636,11 @@ public class PySkeletonRefresher { return null; } LOG.info("Pregenerated skeletons root is " + root); - String version = mySdk.getVersionString().toLowerCase().replace(" ", "-"); + final String versionString = mySdk.getVersionString(); + if (versionString == null) { + return null; + } + String version = versionString.toLowerCase().replace(" ", "-"); File f; if (SystemInfo.isMac) { String osVersion = SystemInfo.OS_VERSION; @@ -569,7 +649,7 @@ public class PySkeletonRefresher { int secondDot = osVersion.indexOf('.', dot+1); if (secondDot >= 0) { osVersion = osVersion.substring(0, secondDot); - } + } } f = new File(root, "skeletons-mac-" + myGeneratorVersion + "-" + osVersion + "-" + version + ".zip"); } @@ -611,17 +691,18 @@ public class PySkeletonRefresher { * @param assemblyRefs refs that generator wants to know in .net environment, if applicable * @return true if generation completed successfully */ - public boolean generateSkeleton( - @NotNull String modname, @Nullable String modfilename, @Nullable List assemblyRefs - ) { + public boolean generateSkeleton(@NotNull String modname, @Nullable String modfilename, @Nullable List assemblyRefs) { boolean ret = true; String binaryPath = mySdk.getHomePath(); + if (myExtraSyspath == null) { + myExtraSyspath = getExtraSysPath(mySdk, mySkeletonsPath); + } final String parent_dir = new File(binaryPath).getParent(); List commandLine = new ArrayList(); commandLine.add(binaryPath); commandLine.add(PythonHelpersLocator.getHelperPath(GENERATOR3)); commandLine.add("-d"); - commandLine.add(getSkeletonPath()); + commandLine.add(getSkeletonsPath()); if (assemblyRefs != null && !assemblyRefs.isEmpty()) { commandLine.add("-c"); commandLine.add(StringUtil.join(assemblyRefs, ";")); @@ -630,7 +711,7 @@ public class PySkeletonRefresher { commandLine.add("-x"); } commandLine.add("-s"); - commandLine.add(getExtraSyspath()); + commandLine.add(myExtraSyspath); commandLine.add(modname); if (modfilename != null) commandLine.add(modfilename); diff --git a/python/src/com/jetbrains/python/sdk/PythonSdkType.java b/python/src/com/jetbrains/python/sdk/PythonSdkType.java index 7485e419846d..a0a73d07237b 100644 --- a/python/src/com/jetbrains/python/sdk/PythonSdkType.java +++ b/python/src/com/jetbrains/python/sdk/PythonSdkType.java @@ -103,7 +103,7 @@ public class PythonSdkType extends SdkType { @NotNull @NonNls public static String getBuiltinsFileName(Sdk sdk) { - final String version = sdk.getVersionString(); + final String version = sdk.getVersionString(); if (version != null && version.startsWith("Python 3")) { return PyBuiltinCache.BUILTIN_FILE_3K; } @@ -193,6 +193,28 @@ public class PythonSdkType extends SdkType { return result; } + public static boolean isVirtualEnv(Sdk sdk) { + final String path = sdk.getHomePath(); + return path != null && getVirtualEnvRoot(path) != null; + } + + @Nullable + public Sdk getVirtualEnvBaseSdk(Sdk sdk) { + if (isVirtualEnv(sdk)) { + final PythonSdkFlavor flavor = PythonSdkFlavor.getFlavor(sdk.getHomePath()); + final String version = getVersionString(sdk); + if (flavor != null && version != null) { + for (Sdk baseSdk : getAllSdks()) { + final PythonSdkFlavor baseFlavor = PythonSdkFlavor.getFlavor(baseSdk.getHomePath()); + if (!isVirtualEnv(baseSdk) && flavor.equals(baseFlavor) && version.equals(getVersionString(baseSdk))) { + return baseSdk; + } + } + } + } + return null; + } + /** * @param binaryPath must point to a Python interpreter * @return if the surroundings look like a virtualenv installation, its root is returned (normally the grandparent of binaryPath). @@ -425,24 +447,11 @@ public class PythonSdkType extends SdkType { final Task.Modal setupTask = new Task.Modal(project, "Setting up library files for " + sdk.getName(), false) { // TODO: make this a backgroundable task. see #setupSdkPaths(final Sdk sdk) and its modificator handling public void run(@NotNull final ProgressIndicator indicator) { - try { - sdkModificator.removeAllRoots(); - updateSdkRootsFromSysPath(sdkModificator, indicator); - if (!ApplicationManager.getApplication().isUnitTestMode()) { - new PySkeletonRefresher(sdk, getSkeletonsPath(sdk.getHomePath()), indicator).regenerateSkeletons(null); - PythonSdkUpdater.getInstance().markAlreadyUpdated(sdk.getHomePath()); - } - //sdkModificator.commitChanges() must happen outside, in dispatch thread. - } - catch (InvalidSdkException e) { - success.set(false); - LOG.warn(e); - Notifications.Bus.notify( - new Notification( - SKELETONS_TOPIC, "Refresh failed", "Skeleton re-generation failed. Please see PyCharm log for traceback. Send it to developers.", - NotificationType.WARNING - ) - ); + sdkModificator.removeAllRoots(); + updateSdkRootsFromSysPath(sdkModificator, indicator); + if (!ApplicationManager.getApplication().isUnitTestMode()) { + refreshSkeletonsOfSDK(sdk, getSkeletonsPath(sdk.getHomePath()), null); + PythonSdkUpdater.getInstance().markAlreadyUpdated(sdk.getHomePath()); } } }; @@ -648,13 +657,15 @@ public class PythonSdkType extends SdkType { } static void refreshSkeletonsOfSDK(Sdk sdk) { + refreshSkeletonsOfSDK(sdk, findSkeletonsPath(sdk), new Ref(false)); + } + + static void refreshSkeletonsOfSDK(Sdk sdk, String skeletonsPath, @Nullable Ref migrationFlag) { final Map> errors = new TreeMap>(); final List failed_sdks = new SmartList(); final ProgressIndicator indicator = ProgressManager.getInstance().getProgressIndicator(); List sdk_errors; - Ref migration_flag = new Ref(false); final String homePath = sdk.getHomePath(); - final String skeletonsPath = findSkeletonsPath(sdk); if (skeletonsPath == null) { LOG.info("Could not find skeletons path for SDK path " + homePath); } @@ -662,7 +673,7 @@ public class PythonSdkType extends SdkType { LOG.info("Refreshing skeletons for " + homePath); try { SkeletonVersionChecker checker = new SkeletonVersionChecker(0); // this default version won't be used - sdk_errors = new PySkeletonRefresher(sdk, skeletonsPath, indicator).regenerateSkeletons(checker, migration_flag); + sdk_errors = new PySkeletonRefresher(sdk, skeletonsPath, indicator).regenerateSkeletons(checker, migrationFlag); if (sdk_errors.size() > 0) { String sdk_name = sdk.getName(); List known_errors = errors.get(sdk_name); diff --git a/python/testData/completion/importKeyword.after.py b/python/testData/completion/importKeyword.after.py new file mode 100644 index 000000000000..d7b90079b958 --- /dev/null +++ b/python/testData/completion/importKeyword.after.py @@ -0,0 +1 @@ +from foo import #import \ No newline at end of file diff --git a/python/testData/completion/importKeyword.py b/python/testData/completion/importKeyword.py new file mode 100644 index 000000000000..9fb29a423d4f --- /dev/null +++ b/python/testData/completion/importKeyword.py @@ -0,0 +1 @@ +from foo #import \ No newline at end of file diff --git a/python/testData/refactoring/rename/renameTarget.py b/python/testData/refactoring/rename/renameTarget.py new file mode 100644 index 000000000000..3a780e0a5956 --- /dev/null +++ b/python/testData/refactoring/rename/renameTarget.py @@ -0,0 +1,15 @@ +if True: + foo = {} +else: + foo = object() + +if "a" == "b": + foo.x = 1 + foo.save() + +if "c" == "d": + foo.y = 1 + foo.save() + + + diff --git a/python/testData/refactoring/rename/renameTarget_after.py b/python/testData/refactoring/rename/renameTarget_after.py new file mode 100644 index 000000000000..70b49e0074fa --- /dev/null +++ b/python/testData/refactoring/rename/renameTarget_after.py @@ -0,0 +1,15 @@ +if True: + bar = {} +else: + bar = object() + +if "a" == "b": + bar.x = 1 + bar.save() + +if "c" == "d": + bar.y = 1 + bar.save() + + + diff --git a/python/testSrc/com/jetbrains/python/PythonCompletionTest.java b/python/testSrc/com/jetbrains/python/PythonCompletionTest.java index d228010b3317..19595928f85b 100644 --- a/python/testSrc/com/jetbrains/python/PythonCompletionTest.java +++ b/python/testSrc/com/jetbrains/python/PythonCompletionTest.java @@ -93,12 +93,12 @@ public class PythonCompletionTest extends PyTestCase { public void testKwParamsInCodeUsage() { //PY-1002 doTest(); } - - public void testKwParamsInCodeGetUsage() { //PY-1002 + + public void testKwParamsInCodeGetUsage() { //PY-1002 doTest(); } - public void testSuperInitKwParamsNotOnlySelfAndKwArgs() { //PY-1050 + public void testSuperInitKwParamsNotOnlySelfAndKwArgs() { //PY-1050 doTest(); } @@ -107,7 +107,7 @@ public class PythonCompletionTest extends PyTestCase { } public void testIsInstance() { - doTest(); + doTest(); } public void testIsInstanceAssert() { @@ -247,7 +247,7 @@ public class PythonCompletionTest extends PyTestCase { public void testNonlocal() { // PY-2289 doTest3K(); } - + public void testYield() { doTest(); } @@ -265,15 +265,15 @@ public class PythonCompletionTest extends PyTestCase { public void testSuperMethod() { // PY-170 doTest(); } - + public void testElse() { doTest(); } - + public void testElseNotIndented() { doTest(); } - + public void testElseInTryNotIndented() { doTest(); } @@ -281,7 +281,7 @@ public class PythonCompletionTest extends PyTestCase { public void testElif() { doTest(); } - + public void testElifNotIndented() { doTest(); } @@ -297,11 +297,11 @@ public class PythonCompletionTest extends PyTestCase { public void testFinallyInExcept() { doTest(); } - + public void testContinue() { doTest(); } - + public void testNoContinueInFinally() { final String testName = "completion/" + getTestName(true); myFixture.configureByFile(testName + ".py"); @@ -332,7 +332,7 @@ public class PythonCompletionTest extends PyTestCase { public void testNoIdentifiersInImport() { doTest(); } - + public void testSuperClassAttributes() { doTest(); } @@ -452,7 +452,7 @@ public class PythonCompletionTest extends PyTestCase { public void testMro() { // PY-3989 doTest(); } - + public void testNamedTuple() { // final String testName = "completion/" + getTestName(true); myFixture.configureByFile(testName + ".py"); @@ -461,15 +461,15 @@ public class PythonCompletionTest extends PyTestCase { assertTrue(strings.contains("lat")); assertTrue(strings.contains("long")); } - + public void testPrivateMemberType() { // PY-4589 doTest(); } - + public void testCompleteBeforeSyntaxError() { // PY-3792 doTest(); } - + // PY-4279 public void testFieldReassignment() { doTest(); @@ -478,4 +478,9 @@ public class PythonCompletionTest extends PyTestCase { public void testSuperInit() { // PY-5066 doTest(); } + + // PY-5144 + public void testImportKeyword() { + doTest(); + } } diff --git a/python/testSrc/com/jetbrains/python/refactoring/PyRenameTest.java b/python/testSrc/com/jetbrains/python/refactoring/PyRenameTest.java index 46d2fe3838a3..1517e2e668a3 100644 --- a/python/testSrc/com/jetbrains/python/refactoring/PyRenameTest.java +++ b/python/testSrc/com/jetbrains/python/refactoring/PyRenameTest.java @@ -81,6 +81,10 @@ public class PyRenameTest extends PyTestCase { public void testRenameKeywordArgument() { // PY-3890 doTest("baz"); } + + public void testRenameTarget() { // PY-5146 + doTest("bar"); + } public void testClassNameConflict() { // PY-2390 doRenameConflictTest("Foo", "A class named 'Foo' is already defined in classNameConflict.py");