From 992b5a3fcd7a44a6122cdd4d0ea4c71e119ef7e1 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Wed, 16 Feb 2011 16:18:28 +0100 Subject: [PATCH 1/8] @NotNull, fix calculating SDK for out of project files during completion (PY-1975) --- python/src/com/jetbrains/python/psi/impl/PyBuiltinCache.java | 2 +- .../src/com/jetbrains/python/psi/resolve/ResolveImportUtil.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/src/com/jetbrains/python/psi/impl/PyBuiltinCache.java b/python/src/com/jetbrains/python/psi/impl/PyBuiltinCache.java index 3b39bfd5cd0a..8d487ffea4ea 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyBuiltinCache.java +++ b/python/src/com/jetbrains/python/psi/impl/PyBuiltinCache.java @@ -75,7 +75,7 @@ public class PyBuiltinCache { sdk = PythonSdkType.findPythonSdk(module); } else { - final VirtualFile vfile = psifile.getVirtualFile(); + final VirtualFile vfile = psifile instanceof PsiFile ? ((PsiFile) psifile).getOriginalFile().getVirtualFile() : psifile.getVirtualFile(); if (vfile != null) { // reality final ProjectRootManager projectRootManager = ProjectRootManager.getInstance(project); sdk = projectRootManager.getProjectSdk(); diff --git a/python/src/com/jetbrains/python/psi/resolve/ResolveImportUtil.java b/python/src/com/jetbrains/python/psi/resolve/ResolveImportUtil.java index 0113fd88def7..2ac7917f8e9f 100644 --- a/python/src/com/jetbrains/python/psi/resolve/ResolveImportUtil.java +++ b/python/src/com/jetbrains/python/psi/resolve/ResolveImportUtil.java @@ -437,7 +437,7 @@ public class ResolveImportUtil { // no module, another way to look in SDK roots final PsiFile elt_psifile = elt.getContainingFile(); if (elt_psifile != null) { // formality - final VirtualFile elt_vfile = elt_psifile.getVirtualFile(); + final VirtualFile elt_vfile = elt_psifile.getOriginalFile().getVirtualFile(); if (elt_vfile != null) { // reality final ProjectFileIndex fileIndex = ProjectRootManager.getInstance(elt.getProject()).getFileIndex(); final List orderEntries = fileIndex.getOrderEntriesForFile(elt_vfile); From 5cbfad8b1c3fd227bb3ec219dcbad786d61919ed Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Wed, 16 Feb 2011 16:33:44 +0100 Subject: [PATCH 2/8] fix 'unhashable instance' exception in generator --- python/helpers/generator3.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/helpers/generator3.py b/python/helpers/generator3.py index 52b58bf7dffb..d5d171a9410f 100644 --- a/python/helpers/generator3.py +++ b/python/helpers/generator3.py @@ -1432,18 +1432,18 @@ class ModuleRedeclarator(object): @param indent indentation level @param p_class the class that contains this function as a method @param p_modname module name - @param seen {func: name} map of functions already seen in the same namespace - """ + @param seen {id(func): name} map of functions already seen in the same namespace; + id() because *some* functions are unhashable (eg _elementtree.Comment in py2.7) """ action("redoing func %r of class %r", p_name, p_class) if seen is not None: - other_func = seen.get(p_func, None) + other_func = seen.get(id(p_func), None) if other_func and getattr(other_func, "__doc__", None) is getattr(p_func, "__doc__", None): # _bisect.bisect == _bisect.bisect_right in py31, but docs differ - out(indent, p_name, " = ", seen[p_func]) + out(indent, p_name, " = ", seen[id(p_func)]) out(indent, "") return else: - seen[p_func] = p_name + seen[id(p_func)] = p_name # real work classname = p_class and p_class.__name__ or None if p_class and hasattr(p_class, '__mro__'): From 4cba4f70424110468584a1f1ebd4adcd9c8a5a08 Mon Sep 17 00:00:00 2001 From: Dmitry Trofimov Date: Wed, 16 Feb 2011 20:22:48 +0300 Subject: [PATCH 3/8] Fixed resolveModulesInRoots to make work resolve for django project name. Needed to fix PY-2239. --- .../python/psi/resolve/ResolveImportUtil.java | 92 ++++++++++--------- 1 file changed, 50 insertions(+), 42 deletions(-) diff --git a/python/src/com/jetbrains/python/psi/resolve/ResolveImportUtil.java b/python/src/com/jetbrains/python/psi/resolve/ResolveImportUtil.java index 0113fd88def7..937b98de14e5 100644 --- a/python/src/com/jetbrains/python/psi/resolve/ResolveImportUtil.java +++ b/python/src/com/jetbrains/python/psi/resolve/ResolveImportUtil.java @@ -2,7 +2,6 @@ package com.jetbrains.python.psi.resolve; import com.google.common.collect.Sets; import com.intellij.facet.FacetManager; -import com.intellij.facet.ProjectFacetManager; import com.intellij.openapi.extensions.Extensions; import com.intellij.openapi.module.Module; import com.intellij.openapi.module.ModuleUtil; @@ -229,17 +228,7 @@ public class ResolveImportUtil { } List found_in_roots = resolveModulesInRoots(qualifiedName, source_file); if (found_in_roots.size() > 0) return found_in_roots; - // resolve the name considering every source root as a package dir, as if it's a deployed package. django console does so. - boolean has_djando_facet = false; - final Module source_module = ModuleUtil.findModuleForPsiElement(source_file); - if (source_module != null) { - has_djando_facet = FacetManager.getInstance(source_module).getFacetByType(DjangoFacetType.ID) != null; - } - if (has_djando_facet) { - ResolveInRootVisitor visitor = new ResolveInRootAsTopPackageVisitor(qualifiedName, source_file.getManager(), source_file, true); - visitRoots(source_file, visitor); - return visitor.results; - } + return Collections.emptyList(); } } @@ -274,7 +263,7 @@ public class ResolveImportUtil { } @Nullable - public static PsiElement resolveModuleInRoots(@NotNull PyQualifiedName moduleQualifiedName, PsiElement foothold) { + public static PsiElement resolveModuleInRoots(@NotNull PyQualifiedName moduleQualifiedName, @Nullable PsiElement foothold) { final List candidates = resolveModulesInRoots(moduleQualifiedName, foothold); return candidates.isEmpty() ? null : candidates.get(0); } @@ -287,7 +276,7 @@ public class ResolveImportUtil { * @return the list of matching directories or files, or an empty list if nothing was found */ @NotNull - public static List resolveModulesInRoots(@NotNull PyQualifiedName moduleQualifiedName, PsiElement foothold) { + public static List resolveModulesInRoots(@NotNull PyQualifiedName moduleQualifiedName, @Nullable PsiElement foothold) { if (foothold == null || !foothold.isValid()) return Collections.emptyList(); PsiFile footholdFile = foothold.getContainingFile(); if (footholdFile == null || !footholdFile.isValid()) return Collections.emptyList(); @@ -299,11 +288,31 @@ public class ResolveImportUtil { return cachedResults; } } - ResolveInRootVisitor visitor = new ResolveInRootVisitor(moduleQualifiedName, foothold.getManager(), footholdFile, true); - visitRoots(foothold, visitor); + + List results = visitRoots(moduleQualifiedName, foothold); + if (cache != null) { - cache.put(moduleQualifiedName, visitor.results); + cache.put(moduleQualifiedName, results); } + return results; + } + + private static List visitRoots(@NotNull PyQualifiedName moduleQualifiedName, @NotNull PsiElement foothold) { + PsiFile footholdFile = foothold.getContainingFile(); + // resolve the name considering every source root as a package dir, as if it's a deployed package. django console does so. + boolean has_djando_facet = false; + final Module source_module = ModuleUtil.findModuleForPsiElement(foothold); + if (source_module != null) { + has_djando_facet = FacetManager.getInstance(source_module).getFacetByType(DjangoFacetType.ID) != null; + } + ResolveInRootVisitor visitor; + if (has_djando_facet) { + visitor = new ResolveInRootAsTopPackageVisitor(moduleQualifiedName, foothold.getManager(), footholdFile, true); + } + else { + visitor = new ResolveInRootVisitor(moduleQualifiedName, foothold.getManager(), footholdFile, true); + } + visitRoots(foothold, visitor); return visitor.results; } @@ -513,7 +522,7 @@ public class ResolveImportUtil { /** * Looks for a name among element's module's roots; if there's no module, then among project's roots. * - * @param context PSI element that defines the module and/or the project. + * @param context PSI element that defines the module and/or the project. * @param refName module name to be found among roots. * @return a PsiFile, a child of a root. */ @@ -559,7 +568,7 @@ public class ResolveImportUtil { static class ResolveInRootVisitor implements RootVisitor { final PsiFile myFootholdFile; - private final boolean myCheckForPackage; + final boolean myCheckForPackage; final @NotNull PyQualifiedName myQualifiedName; final @NotNull PsiManager myPsiManager; final List results = new ArrayList(); @@ -582,17 +591,17 @@ public class ResolveImportUtil { if (module != null) { results.add(module); } + return true; } @Nullable - private PsiElement resolveInRoot(VirtualFile root, - PyQualifiedName qualifiedName, - PsiManager psiManager, - @Nullable PsiFile foothold_file, - boolean checkForPackage) { + protected PsiElement resolveInRoot(VirtualFile root, + PyQualifiedName qualifiedName, + PsiManager psiManager, + @Nullable PsiFile foothold_file, + boolean checkForPackage) { PsiElement module = root.isDirectory() ? psiManager.findDirectory(root) : psiManager.findFile(root); - module = patchModule(module, qualifiedName); if (module == null) return null; for (String component : qualifiedName.getComponents()) { if (component == null) { @@ -603,18 +612,6 @@ public class ResolveImportUtil { } return module; } - - /** - * @param root_module file or dir representing a module's root. - * @param name what we try to resolve - * @return altered parent module; this implementation returns root_module unmodified. - */ - @Nullable - @SuppressWarnings({"MethodMayBeStatic"}) - protected PsiElement patchModule(PsiElement root_module, PyQualifiedName name) { - return root_module; - } - } /** @@ -629,13 +626,24 @@ public class ResolveImportUtil { super(qName, psiManager, foothold_file, checkForPackage); } - @Nullable @Override - protected PsiElement patchModule(PsiElement root_module, PyQualifiedName name) { - if (name.matchesPrefix(PyQualifiedName.fromDottedString(((PsiNamedElement)root_module).getName()))) { - return root_module.getParent(); + public boolean visitRoot(VirtualFile root) { + if (!root.isValid()) { + return true; } - return null; + PsiElement module = resolveInRoot(root, myQualifiedName, myPsiManager, myFootholdFile, myCheckForPackage); + if (module != null) { + results.add(module); + } + + if (myQualifiedName.matchesPrefix(PyQualifiedName.fromDottedString(root.getName()))) { + module = resolveInRoot(root.getParent(), myQualifiedName, myPsiManager, myFootholdFile, myCheckForPackage); + if (module != null) { + results.add(module); + } + } + + return true; } } From 795031fdfe081b898a3d008c862844d33b8e096d Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Wed, 16 Feb 2011 19:41:22 +0100 Subject: [PATCH 4/8] in debug mode, exit if failed to import name --- python/helpers/generator3.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/helpers/generator3.py b/python/helpers/generator3.py index cfe2c15d0bc5..20dd7ac6d7f1 100644 --- a/python/helpers/generator3.py +++ b/python/helpers/generator3.py @@ -2085,6 +2085,8 @@ if __name__ == "__main__": __import__(name) # sys.modules will fill up with what we want except ImportError: report("Name %r failed to import", name) + if debug_mode: + sys.exit(1) continue if my_finder: From 9fe76612007b634169ac7a67a8956a7415fdcc63 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Wed, 16 Feb 2011 20:21:01 +0100 Subject: [PATCH 5/8] diagnostics for ImportError --- python/helpers/generator3.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/helpers/generator3.py b/python/helpers/generator3.py index 20dd7ac6d7f1..4523a2a23477 100644 --- a/python/helpers/generator3.py +++ b/python/helpers/generator3.py @@ -2084,7 +2084,8 @@ if __name__ == "__main__": try: __import__(name) # sys.modules will fill up with what we want except ImportError: - report("Name %r failed to import", name) + exctype, value = sys.exc_info()[:2] + report("Name %r failed to import: %r", name, str(value)) if debug_mode: sys.exit(1) continue From 2cc29dd319b8cc4003ef70c37ac2cf5364eaaa2c Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Wed, 16 Feb 2011 20:30:12 +0100 Subject: [PATCH 6/8] fix another incompatibility of generator3 with PyQt4 --- python/helpers/generator3.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python/helpers/generator3.py b/python/helpers/generator3.py index 4523a2a23477..958c0bb39349 100644 --- a/python/helpers/generator3.py +++ b/python/helpers/generator3.py @@ -1703,11 +1703,15 @@ class ModuleRedeclarator(object): Intended for built-in modules and thus does not handle import statements. @param p_name name of module """ - action("redong module %r", p_name) + action("redoing module %r", p_name) self.redoSimpleHeader(p_name) # find whatever other self.imported_modules the module knows; effectively these are imports module_type = type(sys) - for item_name, item in self.module.__dict__.items(): + for item_name in self.module.__dict__.keys(): + try: + item = self.module.__dict__[item_name] + except: + continue if isinstance(item, module_type): self.imported_modules[item_name] = item self.addImportHeaderIfNeeded() From ccfda7332f0a6e3f13c97b1b6cf3354222cd1bc1 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Wed, 16 Feb 2011 20:56:31 +0100 Subject: [PATCH 7/8] even further crash-proof import restoration --- python/helpers/generator3.py | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/python/helpers/generator3.py b/python/helpers/generator3.py index 958c0bb39349..94ecd9a4a95f 100644 --- a/python/helpers/generator3.py +++ b/python/helpers/generator3.py @@ -1692,20 +1692,8 @@ class ModuleRedeclarator(object): out(0, "# from file " + self.module.__file__) self.outDocAttr(out, self.module, 0) - def addImportHeaderIfNeeded(self): - if self.imports_buf.isEmpty(): - self.imports_buf.out(0, "") - self.imports_buf.out(0, "# imports") - def redo(self, p_name, imported_module_names): - """ - Restores module declarations. - Intended for built-in modules and thus does not handle import statements. - @param p_name name of module - """ - action("redoing module %r", p_name) - self.redoSimpleHeader(p_name) - # find whatever other self.imported_modules the module knows; effectively these are imports + def redoImports(self): module_type = type(sys) for item_name in self.module.__dict__.keys(): try: @@ -1721,6 +1709,28 @@ class ModuleRedeclarator(object): else: self.imports_buf.out(0, item_name, " = None # ??? name unknown; ", ref_notice) + def addImportHeaderIfNeeded(self): + if self.imports_buf.isEmpty(): + self.imports_buf.out(0, "") + self.imports_buf.out(0, "# imports") + + def redo(self, p_name, imported_module_names): + """ + Restores module declarations. + Intended for built-in modules and thus does not handle import statements. + @param p_name name of module + """ + action("redoing module %r %r", p_name, str(self.module)) + self.redoSimpleHeader(p_name) + + # find whatever other self.imported_modules the module knows; effectively these are imports + try: + self.redoImports() + except: + pass + + module_type = type(sys) + # group what else we have into buckets vars_simple = {} vars_complex = {} From 8f1d502811004f4b5c322be601d2626fb28050cc Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Wed, 16 Feb 2011 21:02:54 +0100 Subject: [PATCH 8/8] fix testdata --- python/testData/completion/importItself.after.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/testData/completion/importItself.after.py b/python/testData/completion/importItself.after.py index 17090ac3c5fd..5fc2231a302c 100644 --- a/python/testData/completion/importItself.after.py +++ b/python/testData/completion/importItself.after.py @@ -1 +1 @@ -from package1.submodule2 \ No newline at end of file +from package1.submodule2 import \ No newline at end of file