diff --git a/python/build/pycharm_community_build.gant b/python/build/pycharm_community_build.gant index 29fd37469040..6ccd5d6d1a2a 100644 --- a/python/build/pycharm_community_build.gant +++ b/python/build/pycharm_community_build.gant @@ -1,6 +1,7 @@ import org.jetbrains.jps.LayoutInfo import static org.jetbrains.jps.idea.IdeaProjectLoader.guessHome +setProperty("home", guessHome(this as Script)) includeTargets << new File("${guessHome(this as Script)}/community/build/scripts/utils.gant") includeTargets << new File("${guessHome(this)}/build/scripts/libLicenses.gant") diff --git a/python/helpers/extra_syspath.py b/python/helpers/extra_syspath.py new file mode 100644 index 000000000000..6b47da20690c --- /dev/null +++ b/python/helpers/extra_syspath.py @@ -0,0 +1,14 @@ +import sys, os +qualified_name = sys.argv[-1] +path = qualified_name.split(".") + +try: + module = __import__(qualified_name, globals(), locals(), [path[-1]]) + try: + p = module.__path__ + sys.stdout.write(os.sep.join(p.split(os.sep)[:-1])) + sys.stdout.flush() + except AttributeError: + pass +except ImportError: + pass diff --git a/python/helpers/generator3.py b/python/helpers/generator3.py index 1b32313354af..8438e1809569 100644 --- a/python/helpers/generator3.py +++ b/python/helpers/generator3.py @@ -24,7 +24,7 @@ but seemingly no one uses them in C extensions yet anyway. # * re.search-bound, ~30% time, in likes of builtins and _gtk with complex docstrings. # None of this can seemingly be easily helped. Maybe there's a simpler and faster parser library? -VERSION = "1.128" # Must be a number-dot-number string, updated with each change that affects generated skeletons +VERSION = "1.129" # Must be a number-dot-number string, updated with each change that affects generated skeletons # Note: DON'T FORGET TO UPDATE! import sys @@ -1785,7 +1785,7 @@ class ModuleRedeclarator(object): out(0, "# " + msg % args) out(0, "") - def redoClass(self, out, p_class, p_name, indent, p_modname=None, seen=None): + def redoClass(self, out, p_class, p_name, indent, p_modname=None, seen=None, inspect_dir=False): """ Restores a class definition. @param out output function of a relevant buf @@ -1841,7 +1841,7 @@ class ModuleRedeclarator(object): others = {} we_are_the_base_class = p_modname == BUILTIN_MOD_NAME and p_name == "object" try: - if hasattr(p_class, "__dict__"): + if hasattr(p_class, "__dict__") and not inspect_dir: field_source = p_class.__dict__ field_keys = field_source.keys() # Jython 2.5.1 _codecs fail here else: @@ -1994,7 +1994,7 @@ class ModuleRedeclarator(object): self.imports_buf.out(0, "") self.imports_buf.out(0, "# imports") - def redo(self, p_name, imported_module_names): + def redo(self, p_name, inspect_dir): """ Restores module declarations. Intended for built-in modules and thus does not handle import statements. @@ -2022,7 +2022,10 @@ class ModuleRedeclarator(object): vars_complex = {} funcs = {} classes = {} - for item_name in self.module.__dict__: + module_dict = self.module.__dict__ + if inspect_dir: + module_dict = dir(self.module) + for item_name in module_dict: note("looking at %s", item_name) if item_name in ( "__dict__", "__doc__", "__module__", "__file__", "__name__", "__builtins__", "__package__"): @@ -2030,8 +2033,13 @@ class ModuleRedeclarator(object): try: item = getattr(self.module, item_name) # let getters do the magic except AttributeError: + if not item_name in self.module.__dict__: continue item = self.module.__dict__[item_name] # have it raw # check if it has percolated from an imported module + except NotImplementedError: + if not item_name in self.module.__dict__: continue + item = self.module.__dict__[item_name] # have it raw + # unless we're adamantly positive that the name was imported, we assume it is defined here mod_name = None # module from which p_name might have been imported # IronPython has non-trivial reexports in System module, but not in others: @@ -2164,7 +2172,7 @@ class ModuleRedeclarator(object): out(0, "# definition of ", item_name, " omitted") continue item = classes[item_name] - self.redoClass(out, item, item_name, 0, p_modname=p_name, seen=seen_classes) + self.redoClass(out, item, item_name, 0, p_modname=p_name, seen=seen_classes, inspect_dir=inspect_dir) self._defined[item_name] = True out(0, "") # empty line after each item @@ -2375,17 +2383,17 @@ def buildOutputName(subdir, name): return fname - +MODULES_INSPECT_DIR = ['gi.repository'] def redoModule(name, outfile, mod_file_name, doing_builtins, imported_module_names): # gobject does 'del _gobject' in its __init__.py, so the chained attribute lookup code # fails to find 'gobject._gobject'. thus we need to pull the module directly out of # sys.modules mod = sys.modules.get(name) + path = name.split('.') if not mod and sys.platform == 'cli': # "import System.Collections" in IronPython 2.7 doesn't actually put System.Collections in sys.modules # instead, sys.modules['System'] get set to a Microsoft.Scripting.Actions.NamespaceTracker and Collections can be # accessed as its attribute - path = name.split('.') mod = sys.modules[path[0]] for component in path[1:]: try: @@ -2397,17 +2405,17 @@ def redoModule(name, outfile, mod_file_name, doing_builtins, imported_module_nam if mod: action("restoring") r = ModuleRedeclarator(mod, outfile, mod_file_name, doing_builtins=doing_builtins) - r.redo(name, imported_module_names) + r.redo(name, ".".join(path[:-1]) in MODULES_INSPECT_DIR) action("flushing") r.flush() else: - report("Failed to find imported module in sys.modules") + report("Failed to find imported module in sys.modules " + name) # find_binaries functionality BIN_MODULE_FNAME_PAT = re.compile('([a-zA-Z_]+[0-9a-zA-Z]*)\\.(?:pyc|pyo|(?:[a-zA-Z_]+-\\d\\d[a-zA-Z]*\\.|.+-linux-gnu\\.)?(?:so|pyd))') # possible binary module filename: letter, alphanum architecture per PEP-3149 - +TYPELIB_MODULE_FNAME_PAT = re.compile("([a-zA-Z_]+[0-9a-zA-Z]*)[0-9a-zA-Z-.]*\\.typelib") def cut_binary_lib_suffix(path, f): """ @param path where f lives @@ -2415,7 +2423,7 @@ def cut_binary_lib_suffix(path, f): @return f without a binary suffix (that is, an importable name) if path+f is indeed a binary lib, or None. Note: if for .pyc or .pyo file a .py is found, None is returned. """ - if not f.endswith(".pyc") and not f.endswith(".pyo") and not f.endswith(".so") and not f.endswith(".pyd"): + if not f.endswith(".pyc") and not f.endswith(".typelib") and not f.endswith(".pyo") and not f.endswith(".so") and not f.endswith(".pyd"): return None ret = None m = BIN_MODULE_FNAME_PAT.match(f) @@ -2429,6 +2437,9 @@ def cut_binary_lib_suffix(path, f): fullname = os.path.join(path, f[:-1]) # check for __pycache__ is made outside if os.path.exists(fullname): ret = None + pat_match = TYPELIB_MODULE_FNAME_PAT.match(f) + if pat_match: + ret = "gi.repository." + pat_match.group(1) return ret @@ -2664,7 +2675,9 @@ def processOne(name, mod_file_name, doing_builtins): redoModule(name, outfile, mod_file_name, doing_builtins, imported_module_names) # The C library may have called Py_InitModule() multiple times to define several modules (gtk._gtk and gtk.gdk); # restore all of them - if imported_module_names: + path = name.split(".") + redo_imports = not ".".join(path[:-1]) in MODULES_INSPECT_DIR + if imported_module_names and redo_imports: for m in sys.modules.keys(): action("looking at possible submodule %r", m) # if module has __file__ defined, it has Python source code and doesn't need a skeleton diff --git a/python/pydevSrc/com/jetbrains/python/console/pydev/ConsoleCommunication.java b/python/pydevSrc/com/jetbrains/python/console/pydev/ConsoleCommunication.java index ed60737e5bf8..cc06c6f4cb99 100644 --- a/python/pydevSrc/com/jetbrains/python/console/pydev/ConsoleCommunication.java +++ b/python/pydevSrc/com/jetbrains/python/console/pydev/ConsoleCommunication.java @@ -1,5 +1,6 @@ package com.jetbrains.python.console.pydev; +import com.intellij.util.Function; import org.jetbrains.annotations.NotNull; import java.util.List; @@ -17,7 +18,7 @@ public interface ConsoleCommunication { boolean isExecuting(); - void execInterpreter(String s, ICallback callback); + void execInterpreter(String s, Function callback); void interrupt(); diff --git a/python/pydevSrc/com/jetbrains/python/console/pydev/ICallback.java b/python/pydevSrc/com/jetbrains/python/console/pydev/ICallback.java deleted file mode 100644 index accf52cfdae5..000000000000 --- a/python/pydevSrc/com/jetbrains/python/console/pydev/ICallback.java +++ /dev/null @@ -1,9 +0,0 @@ -/* - * Created on 13/10/2005 - */ -package com.jetbrains.python.console.pydev; - -public interface ICallback { - - Ret call(Arg arg); -} \ No newline at end of file diff --git a/python/pydevSrc/com/jetbrains/python/console/pydev/IScriptConsoleCommunication.java b/python/pydevSrc/com/jetbrains/python/console/pydev/IScriptConsoleCommunication.java index 4827b19edaad..3c998c8e1e32 100644 --- a/python/pydevSrc/com/jetbrains/python/console/pydev/IScriptConsoleCommunication.java +++ b/python/pydevSrc/com/jetbrains/python/console/pydev/IScriptConsoleCommunication.java @@ -9,6 +9,8 @@ *******************************************************************************/ package com.jetbrains.python.console.pydev; +import com.intellij.util.Function; + /** * Interface for the console communication. * @@ -23,7 +25,7 @@ public interface IScriptConsoleCommunication { * @return the response from the interpreter (contains the stdout, stderr, etc). * @throws Exception */ - void execInterpreter(String command, ICallback onResponseReceived); + void execInterpreter(String command, Function onResponseReceived); /** * Creates the completions to be applied in the interpreter. diff --git a/python/src/com/jetbrains/python/console/PydevConsoleCommunication.java b/python/src/com/jetbrains/python/console/PydevConsoleCommunication.java index 6c89cb03c049..0bea2b9267a3 100644 --- a/python/src/com/jetbrains/python/console/PydevConsoleCommunication.java +++ b/python/src/com/jetbrains/python/console/PydevConsoleCommunication.java @@ -12,6 +12,7 @@ import com.intellij.openapi.util.Pair; import com.intellij.openapi.util.text.StringUtil; import com.intellij.openapi.vfs.LocalFileSystem; import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.util.Function; import com.intellij.xdebugger.frame.XValueChildrenList; import com.jetbrains.python.console.parsing.PythonConsoleData; import com.jetbrains.python.console.pydev.*; @@ -295,7 +296,7 @@ public class PydevConsoleCommunication extends AbstractConsoleCommunication impl * * @param command the command to be executed in the client */ - public void execInterpreter(final String command, final ICallback onResponseReceived) { + public void execInterpreter(final String command, final Function onResponseReceived) { nextResponse = null; if (waitingForInput) { inputReceived = command; @@ -397,7 +398,7 @@ public class PydevConsoleCommunication extends AbstractConsoleCommunication impl } } } - onResponseReceived.call(nextResponse); + onResponseReceived.fun(nextResponse); } }, "Waiting for REPL response", true, myProject); } diff --git a/python/src/com/jetbrains/python/console/PydevConsoleExecuteActionHandler.java b/python/src/com/jetbrains/python/console/PydevConsoleExecuteActionHandler.java index f7aacea43f5f..45d6c8d9f921 100644 --- a/python/src/com/jetbrains/python/console/PydevConsoleExecuteActionHandler.java +++ b/python/src/com/jetbrains/python/console/PydevConsoleExecuteActionHandler.java @@ -17,11 +17,11 @@ import com.intellij.openapi.util.text.StringUtil; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.psi.codeStyle.CodeStyleSettingsManager; import com.intellij.psi.impl.source.codeStyle.IndentHelperImpl; +import com.intellij.util.Function; import com.jetbrains.python.PythonFileType; import com.jetbrains.python.PythonLanguage; import com.jetbrains.python.console.pydev.ConsoleCommunication; import com.jetbrains.python.console.pydev.ConsoleCommunicationListener; -import com.jetbrains.python.console.pydev.ICallback; import com.jetbrains.python.console.pydev.InterpreterResponse; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -169,8 +169,8 @@ public class PydevConsoleExecuteActionHandler extends ConsoleExecuteActionHandle else { executingPrompt(console); } - myConsoleCommunication.execInterpreter(command, new ICallback() { - public Object call(final InterpreterResponse interpreterResponse) { + myConsoleCommunication.execInterpreter(command, new Function() { + public Object fun(final InterpreterResponse interpreterResponse) { // clear myInputBuffer = null; // Handle prompt @@ -217,7 +217,7 @@ public class PydevConsoleExecuteActionHandler extends ConsoleExecuteActionHandle console.setPrompt(PyConsoleUtil.EXECUTING_PROMPT); } - private void more(LanguageConsoleImpl console, Editor currentEditor) { + private static void more(LanguageConsoleImpl console, Editor currentEditor) { if (!PyConsoleUtil.INDENT_PROMPT.equals(console.getPrompt())) { console.setPrompt(PyConsoleUtil.INDENT_PROMPT); PyConsoleUtil.scrollDown(currentEditor); diff --git a/python/src/com/jetbrains/python/inspections/PyShadowingNamesInspection.java b/python/src/com/jetbrains/python/inspections/PyShadowingNamesInspection.java index c36de1ddea9f..c19945da06df 100644 --- a/python/src/com/jetbrains/python/inspections/PyShadowingNamesInspection.java +++ b/python/src/com/jetbrains/python/inspections/PyShadowingNamesInspection.java @@ -91,6 +91,11 @@ public class PyShadowingNamesInspection extends PyInspection { } private void processElement(@NotNull PsiNameIdentifierOwner element) { + final ScopeOwner owner = ScopeUtil.getScopeOwner(element); + // Class-level names are rarely accessed inside at the class level, usually they are accessed as attributes + if (owner instanceof PyClass) { + return; + } final String name = element.getName(); if (name != null && !myIgnoredNames.contains(name)) { final PsiElement identifier = element.getNameIdentifier(); diff --git a/python/src/com/jetbrains/python/inspections/quickfix/ConvertDocstringQuickFix.java b/python/src/com/jetbrains/python/inspections/quickfix/ConvertDocstringQuickFix.java index 174f03332621..30a639c81bf1 100644 --- a/python/src/com/jetbrains/python/inspections/quickfix/ConvertDocstringQuickFix.java +++ b/python/src/com/jetbrains/python/inspections/quickfix/ConvertDocstringQuickFix.java @@ -32,7 +32,7 @@ public class ConvertDocstringQuickFix implements LocalQuickFix { public void applyFix(@NotNull Project project, @NotNull ProblemDescriptor descriptor) { PsiElement expression = descriptor.getPsiElement(); - if (expression instanceof PyStringLiteralExpression) { + if (expression instanceof PyStringLiteralExpression && expression.isWritable()) { PyElementGenerator elementGenerator = PyElementGenerator.getInstance(project); String stringText = expression.getText(); diff --git a/python/src/com/jetbrains/python/inspections/quickfix/GenerateBinaryStubsFix.java b/python/src/com/jetbrains/python/inspections/quickfix/GenerateBinaryStubsFix.java index 6ea36885f235..254ac4824286 100644 --- a/python/src/com/jetbrains/python/inspections/quickfix/GenerateBinaryStubsFix.java +++ b/python/src/com/jetbrains/python/inspections/quickfix/GenerateBinaryStubsFix.java @@ -1,11 +1,16 @@ package com.jetbrains.python.inspections.quickfix; +import com.google.common.collect.Lists; import com.intellij.codeInspection.LocalQuickFix; import com.intellij.codeInspection.ProblemDescriptor; +import com.intellij.execution.process.ProcessOutput; import com.intellij.notification.Notification; import com.intellij.openapi.module.Module; -import com.intellij.openapi.module.ModuleUtil; +import com.intellij.openapi.module.ModuleUtilCore; +import com.intellij.openapi.progress.ProgressIndicator; import com.intellij.openapi.progress.ProgressManager; +import com.intellij.openapi.progress.Task; +import com.intellij.openapi.progress.impl.BackgroundableProcessIndicator; import com.intellij.openapi.project.Project; import com.intellij.openapi.projectRoots.Sdk; import com.intellij.openapi.vfs.LocalFileSystem; @@ -13,16 +18,23 @@ import com.intellij.openapi.vfs.VirtualFile; import com.intellij.psi.PsiElement; import com.intellij.psi.PsiFile; import com.intellij.psi.PsiReference; +import com.intellij.psi.util.PsiTreeUtil; import com.intellij.util.Consumer; +import com.jetbrains.python.PythonHelpersLocator; import com.jetbrains.python.psi.*; +import com.jetbrains.python.psi.impl.PyQualifiedName; import com.jetbrains.python.psi.impl.references.PyImportReference; -import com.jetbrains.python.sdk.*; +import com.jetbrains.python.sdk.InvalidSdkException; +import com.jetbrains.python.sdk.PySdkUtil; +import com.jetbrains.python.sdk.PythonSdkType; import com.jetbrains.python.sdk.flavors.IronPythonSdkFlavor; import com.jetbrains.python.sdk.flavors.PythonSdkFlavor; +import com.jetbrains.python.sdk.skeletons.PySkeletonGenerator; import com.jetbrains.python.sdk.skeletons.PySkeletonRefresher; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import java.io.File; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -44,7 +56,17 @@ public class GenerateBinaryStubsFix implements LocalQuickFix { while (element.getParent() instanceof PyReferenceExpression) { element = element.getParent(); } - return element.getText(); + final String elementText = element.getText(); + + final PyFromImportStatement importStatementBase = PsiTreeUtil.getParentOfType(element, PyFromImportStatement.class); + if (importStatementBase != null) { + final PyQualifiedName qName = importStatementBase.getImportSourceQName(); + if (qName != null) { + return qName.append(elementText).toString(); + } + } + + return elementText; } @NotNull @@ -58,14 +80,23 @@ public class GenerateBinaryStubsFix implements LocalQuickFix { } public void applyFix(@NotNull final Project project, @NotNull final ProblemDescriptor descriptor) { - ProgressManager.getInstance().runProcessWithProgressSynchronously(new Runnable() { - public void run() { - ProgressManager.getInstance().getProgressIndicator().setIndeterminate(true); + final Task.Backgroundable backgroundable = new Task.Backgroundable(project, "Generating skeletons for binary module", false) { + + @Override + public void run(@NotNull ProgressIndicator indicator) { + indicator.setIndeterminate(true); List assemblyRefs = collectAssemblyReferences(descriptor.getPsiElement().getContainingFile()); try { final PySkeletonRefresher refresher = new PySkeletonRefresher(project, null, mySdk, null, null); - refresher.generateSkeleton(myQualifiedName, "", assemblyRefs, Consumer.EMPTY_CONSUMER); + + if (needBinaryList(myQualifiedName)) { + if (!generateSkeletonsForList(refresher, indicator)) return; + } + else { + //noinspection unchecked + refresher.generateSkeleton(myQualifiedName, "", assemblyRefs, Consumer.EMPTY_CONSUMER); + } final VirtualFile skeletonDir; skeletonDir = LocalFileSystem.getInstance().findFileByPath(refresher.getSkeletonsPath()); if (skeletonDir != null) { @@ -77,7 +108,43 @@ public class GenerateBinaryStubsFix implements LocalQuickFix { notification.notify(project); } } - }, "Generating skeletons for binary module", false, project); + }; + ProgressManager.getInstance().runProcessWithProgressAsynchronously(backgroundable, new BackgroundableProcessIndicator(backgroundable)); + } + + private boolean generateSkeletonsForList(@NotNull final PySkeletonRefresher refresher, ProgressIndicator indicator) throws InvalidSdkException { + final PySkeletonGenerator generator = new PySkeletonGenerator(refresher.getSkeletonsPath()); + indicator.setIndeterminate(false); + final String homePath = mySdk.getHomePath(); + if (homePath == null) return false; + final ProcessOutput runResult = PySdkUtil.getProcessOutput( + new File(homePath).getParent(), + new String[]{ + homePath, + PythonHelpersLocator.getHelperPath("extra_syspath.py"), myQualifiedName}, + PythonSdkType.getVirtualEnvAdditionalEnv(homePath), 5000 + ); + if (runResult.getExitCode() == 0 && !runResult.isTimeout()) { + final String extraPath = runResult.getStdout(); + final PySkeletonGenerator.ListBinariesResult binaries = generator.listBinaries(mySdk, extraPath); + final List names = Lists.newArrayList(binaries.modules.keySet()); + Collections.sort(names); + final int size = names.size(); + for (int i = 0; i != size; ++i) { + final String name = names.get(i); + indicator.setFraction((double)i / size); + if (needBinaryList(name)) { + indicator.setText2(name); + //noinspection unchecked + refresher.generateSkeleton(name, "", new ArrayList(), Consumer.EMPTY_CONSUMER); + } + } + } + return true; + } + + private static boolean needBinaryList(@NotNull final String qualifiedName) { + return qualifiedName.startsWith("gi.repository"); } private List collectAssemblyReferences(PsiFile file) { @@ -100,7 +167,7 @@ public class GenerateBinaryStubsFix implements LocalQuickFix { return result; } - public static boolean isApplicable(PsiReference ref) { + public static boolean isApplicable(@NotNull final PsiReference ref) { if (!(ref instanceof PyImportReference)) { return false; } @@ -112,12 +179,22 @@ public class GenerateBinaryStubsFix implements LocalQuickFix { if (flavor instanceof IronPythonSdkFlavor) { return getReferenceText(ref).matches("[A-Z][A-Za-z0-9]+(\\.[A-Z][A-Za-z0-9]+)*"); } + return isGtk(ref); + } + + private static boolean isGtk(@NotNull final PsiReference ref) { + final PyFromImportStatement importStatementBase = PsiTreeUtil.getParentOfType(ref.getElement(), PyFromImportStatement.class); + if (importStatementBase != null) { + final PyQualifiedName qName = importStatementBase.getImportSourceQName(); + if (qName != null && qName.matches("gi", "repository")) + return true; + } return false; } @Nullable - private static Sdk getPythonSdk(PsiReference ref) { - final Module module = ModuleUtil.findModuleForPsiElement(ref.getElement()); + private static Sdk getPythonSdk(@NotNull final PsiReference ref) { + final Module module = ModuleUtilCore.findModuleForPsiElement(ref.getElement()); return module == null ? null : PythonSdkType.findPythonSdk(module); } } diff --git a/python/src/com/jetbrains/python/inspections/quickfix/PyMakeFunctionFromMethodQuickFix.java b/python/src/com/jetbrains/python/inspections/quickfix/PyMakeFunctionFromMethodQuickFix.java index 4fcaa9fe0463..cf0f7300f856 100644 --- a/python/src/com/jetbrains/python/inspections/quickfix/PyMakeFunctionFromMethodQuickFix.java +++ b/python/src/com/jetbrains/python/inspections/quickfix/PyMakeFunctionFromMethodQuickFix.java @@ -56,7 +56,10 @@ public class PyMakeFunctionFromMethodQuickFix implements LocalQuickFix { classStatementList.add(PyElementGenerator.getInstance(project).createPassStatement()); } final PsiFile file = containingClass.getContainingFile(); - copy = file.addAfter(copy, containingClass); + PyClass aClass = PsiTreeUtil.getTopmostParentOfType(containingClass, PyClass.class); + if (aClass == null) + aClass = containingClass; + copy = file.addAfter(copy, aClass); for (UsageInfo usage : usages) { final PsiElement usageElement = usage.getElement(); diff --git a/python/src/com/jetbrains/python/psi/impl/PyBlockEvaluator.java b/python/src/com/jetbrains/python/psi/impl/PyBlockEvaluator.java index 19a9277438ed..f3356c2dce9b 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyBlockEvaluator.java +++ b/python/src/com/jetbrains/python/psi/impl/PyBlockEvaluator.java @@ -124,10 +124,12 @@ public class PyBlockEvaluator { PsiElement source = PyUtil.turnDirIntoInit(node.resolveImportSource()); if (source instanceof PyFile) { PyBlockEvaluator importEvaluator = new PyBlockEvaluator(myVisitedFiles); + importEvaluator.myDeclarationsToTrack.addAll(myDeclarationsToTrack); importEvaluator.evaluate((PyFile)source); if (node.isStarImport()) { // TODO honor __all__ here myNamespace.putAll(importEvaluator.myNamespace); + myDeclarations.putAll(importEvaluator.myDeclarations); } else { for (PyImportElement element : node.getImportElements()) { @@ -137,11 +139,26 @@ public class PyBlockEvaluator { name = element.getName(); } myNamespace.put(name, value); + List declarations = importEvaluator.getDeclarations(name); + if (myDeclarations.containsKey(name)) { + myDeclarations.get(name).addAll(declarations); + } + else { + myDeclarations.put(name, declarations); + } } } } } + @Override + public void visitPyIfStatement(PyIfStatement node) { + PyStatementList list = node.getIfPart().getStatementList(); + if (list != null) { + list.acceptChildren(this); + } + } + @Override public void visitPyReturnStatement(PyReturnStatement node) { myReturnValue = prepareEvaluator().evaluate(node.getExpression()); diff --git a/python/src/com/jetbrains/python/psi/impl/PyClassImpl.java b/python/src/com/jetbrains/python/psi/impl/PyClassImpl.java index ad1ed197eb7d..a08b7c0cf7b4 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyClassImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyClassImpl.java @@ -177,7 +177,9 @@ public class PyClassImpl extends PyPresentableElementImpl implement expression = unfoldClass(expression); if (expression instanceof PyReferenceExpression) { final PsiPolyVariantReference ref = ((PyReferenceExpression)expression).getReference(PyResolveContext.noProperties()); - return ref.resolve(); + if (ref != null) { + return ref.resolve(); + } } return null; } diff --git a/python/src/com/jetbrains/python/psi/impl/PyNamedParameterImpl.java b/python/src/com/jetbrains/python/psi/impl/PyNamedParameterImpl.java index 05cd3ccc91db..f3b2043003ea 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyNamedParameterImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyNamedParameterImpl.java @@ -308,7 +308,8 @@ public class PyNamedParameterImpl extends PyPresentableElementImpl 0 && parameters[0] == this) { if (PyNames.NEW.equals(function.getName())) { return true; } diff --git a/python/src/com/jetbrains/python/refactoring/extractmethod/PyExtractMethodUtil.java b/python/src/com/jetbrains/python/refactoring/extractmethod/PyExtractMethodUtil.java index 326e99676b5f..323d8808e10d 100644 --- a/python/src/com/jetbrains/python/refactoring/extractmethod/PyExtractMethodUtil.java +++ b/python/src/com/jetbrains/python/refactoring/extractmethod/PyExtractMethodUtil.java @@ -140,7 +140,8 @@ public class PyExtractMethodUtil { // replace statements with call callElement = replaceElements(elementsRange, callElement); callElement = CodeInsightUtilCore.forcePsiPostprocessAndRestoreElement(callElement); - processDuplicates(callElement, generatedMethod, finder, editor); + if (callElement != null) + processDuplicates(callElement, generatedMethod, finder, editor); // Set editor setSelectionAndCaret(editor, callElement); diff --git a/python/src/com/jetbrains/python/sdk/PySdkUtil.java b/python/src/com/jetbrains/python/sdk/PySdkUtil.java index 587cae1b104f..cb1b7d6c9b12 100644 --- a/python/src/com/jetbrains/python/sdk/PySdkUtil.java +++ b/python/src/com/jetbrains/python/sdk/PySdkUtil.java @@ -115,42 +115,8 @@ public class PySdkUtil { commands.add("/c"); } Collections.addAll(commands, command); - String[] new_env = null; - if (addEnv != null) { - Map env_map = new HashMap(System.getenv()); - // turn additional ent into map - Map add_map = new HashMap(); - for (String env_item : addEnv) { - int pos = env_item.indexOf('='); - if (pos > 0) { - String key = env_item.substring(0, pos); - String value = env_item.substring(pos + 1, env_item.length()); - add_map.put(key, value); - } - else { - LOG.warn(String.format("Invalid env value: '%s'", env_item)); - } - } - // fuse old and new - for (Map.Entry entry : add_map.entrySet()) { - final String key = entry.getKey(); - final String value = entry.getValue(); - final String old_value = env_map.get(key); - if (old_value != null) { - env_map.put(key, value + old_value); - } - else { - env_map.put(key, value); - } - } - new_env = new String[env_map.size()]; - int i = 0; - for (Map.Entry entry : env_map.entrySet()) { - new_env[i] = entry.getKey() + "=" + entry.getValue(); - i += 1; - } - } - Process process = Runtime.getRuntime().exec(ArrayUtil.toStringArray(commands), new_env, new File(homePath)); + String[] newEnv = buildAdditionalEnv(addEnv); + Process process = Runtime.getRuntime().exec(ArrayUtil.toStringArray(commands), newEnv, new File(homePath)); CapturingProcessHandler processHandler = new CapturingProcessHandler(process); if (stdin != null) { final OutputStream processInput = processHandler.getProcessInput(); @@ -184,6 +150,50 @@ public class PySdkUtil { } } + private static String[] buildAdditionalEnv(String[] addEnv) { + String[] newEnv = null; + if (addEnv != null) { + Map envMap = buildEnvMap(addEnv); + newEnv = new String[envMap.size()]; + int i = 0; + for (Map.Entry entry : envMap.entrySet()) { + newEnv[i] = entry.getKey() + "=" + entry.getValue(); + i += 1; + } + } + return newEnv; + } + + public static Map buildEnvMap(String[] addEnv) { + Map envMap = new HashMap(System.getenv()); + // turn additional ent into map + Map addMap = new HashMap(); + for (String envItem : addEnv) { + int pos = envItem.indexOf('='); + if (pos > 0) { + String key = envItem.substring(0, pos); + String value = envItem.substring(pos + 1, envItem.length()); + addMap.put(key, value); + } + else { + LOG.warn(String.format("Invalid env value: '%s'", envItem)); + } + } + // fuse old and new + for (Map.Entry entry : addMap.entrySet()) { + final String key = entry.getKey(); + final String value = entry.getValue(); + final String oldValue = envMap.get(key); + if (oldValue != null) { + envMap.put(key, value + oldValue); + } + else { + envMap.put(key, value); + } + } + return envMap; + } + public static boolean isRemote(@Nullable Sdk sdk) { return sdk != null && sdk.getSdkAdditionalData() instanceof RemoteCredentials; } diff --git a/python/src/com/jetbrains/python/sdk/skeletons/PySkeletonGenerator.java b/python/src/com/jetbrains/python/sdk/skeletons/PySkeletonGenerator.java index 0fd9c63f5ef3..bc2298387434 100644 --- a/python/src/com/jetbrains/python/sdk/skeletons/PySkeletonGenerator.java +++ b/python/src/com/jetbrains/python/sdk/skeletons/PySkeletonGenerator.java @@ -1,6 +1,9 @@ package com.jetbrains.python.sdk.skeletons; import com.google.common.collect.Maps; +import com.intellij.execution.ExecutionException; +import com.intellij.execution.configurations.GeneralCommandLine; +import com.intellij.execution.process.CapturingProcessHandler; import com.intellij.execution.process.ProcessOutput; import com.intellij.openapi.application.ex.ApplicationManagerEx; import com.intellij.openapi.diagnostic.Logger; @@ -43,7 +46,7 @@ public class PySkeletonGenerator { return new File(name).exists(); } - protected static class ListBinariesResult { + public static class ListBinariesResult { public final int generatorVersion; public final Map modules; @@ -162,54 +165,66 @@ public class PySkeletonGenerator { @NotNull public ListBinariesResult listBinaries(@NotNull Sdk sdk, @NotNull String extraSysPath) throws InvalidSdkException { final String homePath = sdk.getHomePath(); - final String parentDir = new File(homePath).getParent(); final long startTime = System.currentTimeMillis(); - final String[] cmd = new String[]{homePath, PythonHelpersLocator.getHelperPath(GENERATOR3), "-v", "-L", "-s", extraSysPath}; - final ProcessOutput process = getProcessOutput(parentDir, - cmd, - PythonSdkType.getVirtualEnvAdditionalEnv(homePath), - MINUTE * 4); // see PY-3898 - LOG.info("Retrieving binary module list took " + (System.currentTimeMillis() - startTime) + " ms"); - if (process.getExitCode() != 0) { - final StringBuilder sb = new StringBuilder("failed to run ").append(GENERATOR3).append(" for ").append(homePath); - if (process.isTimeout()) { - sb.append(": timed out."); - } - else { - sb.append(", exit code ") - .append(process.getExitCode()) - .append(", stderr: \n-----\n"); - for (String line : process.getStderrLines()) { - sb.append(line).append("\n"); + final String[] cmd = new String[]{homePath, PythonHelpersLocator.getHelperPath(GENERATOR3), "-v", "-L"}; + final GeneralCommandLine commandLine = new GeneralCommandLine(cmd); + commandLine.addParameter("-s"); + commandLine.addParameter(extraSysPath); + final String[] additionalEnv = PythonSdkType.getVirtualEnvAdditionalEnv(homePath); + if (additionalEnv != null) { + final Map map = PySdkUtil.buildEnvMap(additionalEnv); + commandLine.getEnvironment().putAll(map); + } + try { + CapturingProcessHandler handler = new CapturingProcessHandler(commandLine); + final ProcessOutput process = handler.runProcess(MINUTE * 4); + LOG.info("Retrieving binary module list took " + (System.currentTimeMillis() - startTime) + " ms"); + if (process.getExitCode() != 0) { + final StringBuilder sb = new StringBuilder("failed to run ").append(GENERATOR3).append(" for ").append(homePath); + if (process.isTimeout()) { + sb.append(": timed out."); } - sb.append("-----"); + else { + sb.append(", exit code ") + .append(process.getExitCode()) + .append(", stderr: \n-----\n"); + for (String line : process.getStderrLines()) { + sb.append(line).append("\n"); + } + sb.append("-----"); + } + throw new InvalidSdkException(sb.toString()); } + final List lines = process.getStdoutLines(); + if (lines.size() < 1) { + throw new InvalidSdkException("Empty output from " + GENERATOR3 + " for " + homePath); + } + final Iterator iter = lines.iterator(); + final int generatorVersion = fromVersionString(iter.next().trim()); + final Map binaries = Maps.newHashMap(); + while (iter.hasNext()) { + final String line = iter.next(); + int cutpos = line.indexOf('\t'); + if (cutpos >= 0) { + String[] strs = line.split("\t"); + String moduleName = strs[0]; + String path = strs[1]; + int length = Integer.parseInt(strs[2]); + int lastModified = Integer.parseInt(strs[3]); + + binaries.put(moduleName, new PySkeletonRefresher.PyBinaryItem(moduleName, path, length, lastModified)); + } + else { + LOG.error("Bad binaries line: '" + line + "', SDK " + homePath); // but don't die yet + } + } + return new ListBinariesResult(generatorVersion, binaries); + } + catch (ExecutionException e) { + final StringBuilder sb = new StringBuilder("failed to run ").append(GENERATOR3).append(" for ").append(homePath); + sb.append(e.getCause()); throw new InvalidSdkException(sb.toString()); } - final List lines = process.getStdoutLines(); - if (lines.size() < 1) { - throw new InvalidSdkException("Empty output from " + GENERATOR3 + " for " + homePath); - } - final Iterator iter = lines.iterator(); - final int generatorVersion = fromVersionString(iter.next().trim()); - final Map binaries = Maps.newHashMap(); - while (iter.hasNext()) { - final String line = iter.next(); - int cutpos = line.indexOf('\t'); - if (cutpos >= 0) { - String[] strs = line.split("\t"); - String moduleName = strs[0]; - String path = strs[1]; - int length = Integer.parseInt(strs[2]); - int lastModified = Integer.parseInt(strs[3]); - - binaries.put(moduleName, new PySkeletonRefresher.PyBinaryItem(moduleName, path, length, lastModified)); - } - else { - LOG.error("Bad binaries line: '" + line + "', SDK " + homePath); // but don't die yet - } - } - return new ListBinariesResult(generatorVersion, binaries); } public boolean deleteOrLog(@NotNull File item) { diff --git a/python/src/com/jetbrains/python/sdk/skeletons/PySkeletonRefresher.java b/python/src/com/jetbrains/python/sdk/skeletons/PySkeletonRefresher.java index 8ad4d1b82138..b01d920bdb74 100644 --- a/python/src/com/jetbrains/python/sdk/skeletons/PySkeletonRefresher.java +++ b/python/src/com/jetbrains/python/sdk/skeletons/PySkeletonRefresher.java @@ -190,21 +190,22 @@ public class PySkeletonRefresher { private static String calculateExtraSysPath(@NotNull Sdk sdk, @Nullable String skeletonsPath) { final VirtualFile[] classDirs = sdk.getRootProvider().getFiles(OrderRootType.CLASSES); - final StringBuilder builder = new StringBuilder("\""); - int i = 0; - while (i < classDirs.length) { - if (i > 0) { + final StringBuilder builder = new StringBuilder(""); + int countAddedPaths = 0; + for (VirtualFile file : classDirs) { + if (countAddedPaths > 0) { builder.append(File.pathSeparator); } - if (classDirs[i].isInLocalFileSystem()) { - final String pathname = classDirs[i].getPath(); + if (file.isInLocalFileSystem()) { + final String pathname = file.getPath(); if (pathname != null && !pathname.equals(skeletonsPath)) { builder.append(pathname); + countAddedPaths += 1; } } - i += 1; } - builder.append("\""); + + builder.append(""); return builder.toString(); } diff --git a/python/src/com/jetbrains/python/testing/nosetest/PythonNoseTestConfigurationProducer.java b/python/src/com/jetbrains/python/testing/nosetest/PythonNoseTestConfigurationProducer.java index e5b25fdcfb42..d39aaed1066e 100644 --- a/python/src/com/jetbrains/python/testing/nosetest/PythonNoseTestConfigurationProducer.java +++ b/python/src/com/jetbrains/python/testing/nosetest/PythonNoseTestConfigurationProducer.java @@ -20,7 +20,11 @@ public class PythonNoseTestConfigurationProducer extends protected boolean isAvailable(@NotNull final Location location) { final PsiElement element = location.getPsiElement(); Module module = location.getModule(); - if (module == null) module = ModuleManager.getInstance(element.getProject()).getModules()[0]; + if (module == null) { + final Module[] modules = ModuleManager.getInstance(element.getProject()).getModules(); + if (modules.length == 0) return false; + module = modules[0]; + } final Sdk sdk = PythonSdkType.findPythonSdk(module); return (TestRunnerService.getInstance(module).getProjectConfiguration().equals( PythonTestConfigurationsModel.PYTHONS_NOSETEST_NAME) && sdk != null); diff --git a/python/testData/inspections/PyShadowingNamesInspection/test.py b/python/testData/inspections/PyShadowingNamesInspection/test.py index d87dba054ebc..98eec4846722 100644 --- a/python/testData/inspections/PyShadowingNamesInspection/test.py +++ b/python/testData/inspections/PyShadowingNamesInspection/test.py @@ -34,7 +34,7 @@ def test_builtin_class_name(): def test_builtin_method_name(): class C: - def list(self): + def list(self): pass @@ -50,7 +50,7 @@ def test_builtin_qualified_name(): # PY-10164 def test_builtin_class_attribute(): class C: - id = 1 + id = 1 def test_outer_function(): @@ -73,17 +73,17 @@ def test_outer_class(): def foo(): def bar(): class C: - def baz(self): + def baz(self): pass def bar(): pass baz = 2 - def spam(self): + def spam(self): pass - quux = 1 + quux = 1 def test_outer_global():