From 5812115a5cf0c47ad51b530ab6ffa1fa23b2f04a Mon Sep 17 00:00:00 2001 From: Dmitry Cheryasov Date: Sat, 25 Jun 2011 18:53:03 +0300 Subject: [PATCH 1/7] Fixed worst Qt / pygame import issues (PY-3985) --- python/helpers/generator3.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/python/helpers/generator3.py b/python/helpers/generator3.py index 3d5aebf7a628..282313a627fc 100644 --- a/python/helpers/generator3.py +++ b/python/helpers/generator3.py @@ -1810,8 +1810,8 @@ class ModuleRedeclarator(object): vars_complex = {} funcs = {} classes = {} - our_package = packageOf(p_name) for item_name in self.module.__dict__: + note("looking at %s", item_name) if item_name in ("__dict__", "__doc__", "__module__", "__file__", "__name__", "__builtins__", "__package__"): continue # handled otherwise try: @@ -1831,20 +1831,23 @@ class ModuleRedeclarator(object): mod_name = getattr(item, '__module__', None) except: pass - import_from_top = our_package.startswith(packageOf(mod_name, True)) # e.g. p_name="pygame.rect" and mod_name="pygame" + # we assume that module foo.bar never imports foo; foo may import foo.bar. (see pygame and pygame.rect) + maybe_import_mod_name = mod_name or "" + import_is_from_top = len(p_name) > len(maybe_import_mod_name) and p_name.startswith(maybe_import_mod_name) + note("mod_name = %s, prospective = %s, from top = %s", mod_name, maybe_import_mod_name, import_is_from_top) want_to_import = False if (mod_name and mod_name != BUILTIN_MOD_NAME and mod_name != p_name and mod_name not in surely_not_imported_mods - and not import_from_top + and not import_is_from_top ): # import looks valid, but maybe it's a .py file? we're certain not to import from .py # e.g. this rules out _collections import collections and builtins import site. try: imported = __import__(mod_name) # ok to repeat, Python caches for us if imported: - qualifieds = name.split(".")[1:] + qualifieds = mod_name.split(".")[1:] for qual in qualifieds: imported = getattr(imported, qual, None) if not imported: From 24c3cbd4dc51f25929a6b877147377a754066ffe Mon Sep 17 00:00:00 2001 From: Dmitry Cheryasov Date: Sun, 26 Jun 2011 12:45:20 +0300 Subject: [PATCH 2/7] WIP: ignore sip fake bases --- python/helpers/generator3.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/python/helpers/generator3.py b/python/helpers/generator3.py index 282313a627fc..a25effdf3d8b 100644 --- a/python/helpers/generator3.py +++ b/python/helpers/generator3.py @@ -1056,6 +1056,18 @@ class ModuleRedeclarator(object): # 'xml.parsers.expat': ('ExpatError', 'error'), } + KNOWN_FAKE_BASES = [] # list of classes that pretend to be base classes but are mere wrappers. + try: + import sip as sip_module # Qt specifically likes it + if hasattr(sip_module, 'wrapper'): + KNOWN_FAKE_BASES.append(sip_module.wrapper) + if hasattr(sip_module, 'simplewrapper'): + KNOWN_FAKE_BASES.append(sip_module.simplewrapper) + del sip_module + except: + pass + + # Some builtin classes effectively change __init__ signature without overriding it. # This callable serves as a placeholder to be replaced via REDEFINED_BUILTIN_SIGS def fake_builtin_init(self): pass # just a callable, sig doesn't matter @@ -1625,11 +1637,15 @@ class ModuleRedeclarator(object): seen[p_class] = p_name bases = getBases(p_class) base_def = "" + skipped_bases = [] if bases: skip_qualifiers = [p_modname, BUILTIN_MOD_NAME, 'exceptions'] skip_qualifiers.extend(self.KNOWN_FAKE_REEXPORTERS.get(p_modname, ())) bases_list = [] # what we'll render in the class decl for base in bases: # somehow import every base class + if base in self.KNOWN_FAKE_BASES: + skipped_bases.append(str(base)) + continue base_name = base.__name__ qual_module_name = self.qualifierOf(base, skip_qualifiers) got_existing_import = False @@ -1646,20 +1662,19 @@ class ModuleRedeclarator(object): else: bases_list.append(base_name) base_def = "(" + ", ".join(bases_list) + ")" - out(indent, "class ", p_name, base_def, ":") + out(indent, "class ", p_name, base_def, ":", skipped_bases and "# skipped bases: " + ", ".join(skipped_bases) or "") self.outDocAttr(out, p_class, indent + 1) # inner parts methods = {} properties = {} others = {} we_are_the_base_class = p_modname == BUILTIN_MOD_NAME and p_name == "object" - has_dict = hasattr(p_class, "__dict__") - if has_dict: - field_source = p_class.__dict__ - else: - field_source = dir(p_class) # this includes unwanted inherited methods, but no dict + inheritance is rare try: - field_keys = field_source.keys() # Jython 2.5.1 _codecs fail here + if hasattr(p_class, "__dict__"): + field_source = p_class.__dict__ + field_keys = field_source.keys() # Jython 2.5.1 _codecs fail here + else: + field_keys = dir(p_class) # this includes unwanted inherited methods, but no dict + inheritance is rare except: field_keys = () for item_name in field_keys: From c88f8d8a0c31df62ecdb0aee5ac9222441ec3354 Mon Sep 17 00:00:00 2001 From: Dmitry Cheryasov Date: Mon, 27 Jun 2011 16:22:39 +0300 Subject: [PATCH 3/7] skip wrapper-only base classes, fix 'failed to find module', sqlite as fake reexporter. --- python/helpers/generator3.py | 55 ++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/python/helpers/generator3.py b/python/helpers/generator3.py index a25effdf3d8b..79a6b8cfc714 100644 --- a/python/helpers/generator3.py +++ b/python/helpers/generator3.py @@ -24,7 +24,7 @@ but seemingly no one uses them in C extensions yet anyway. # * re.search-bound, ~30% time, in likes of builtins and _gtk with complex docstrings. # None of this can seemingly be easily helped. Maybe there's a simpler and faster parser library? -VERSION = "1.93" # Must be a number-dot-number string, updated with each change that affects generated skeletons +VERSION = "1.94" # Must be a number-dot-number string, updated with each change that affects generated skeletons # Note: DON'T FORGET TO UPDATE! import sys @@ -1037,16 +1037,18 @@ class ModuleRedeclarator(object): # We list all such Ys keyed by X, all fully-qualified names: # {"real_definer_module": ("fake_reexporter_module",..)} KNOWN_FAKE_REEXPORTERS = { - "gtk._gtk": ('gtk',), - "gobject._gobject": ('gobject',), - "numpy.core.multiarray": ('numpy', 'numpy.core'), - "numpy.core._dotblas": ('numpy', 'numpy.core'), - "numpy.core.umath": ('numpy', 'numpy.core'), bin_collections_name: ('collections',), "_functools": ('functools',), "_socket": ('socket',), # .error, etc - "gnomecanvas": ("gnome.canvas",), "pyexpat": ('xml.parsers.expat',), + "_bsddb": ('bsddb.db',), + "pysqlite2._sqlite": ('pysqlite2.dbapi2',), # errors + "numpy.core.multiarray": ('numpy', 'numpy.core'), + "numpy.core._dotblas": ('numpy', 'numpy.core'), + "numpy.core.umath": ('numpy', 'numpy.core'), + "gtk._gtk": ('gtk', 'gtk.gdk',), + "gobject._gobject": ('gobject',), + "gnomecanvas": ("gnome.canvas",), } # names that look genuinely exported but aren't. @@ -1056,13 +1058,15 @@ class ModuleRedeclarator(object): # 'xml.parsers.expat': ('ExpatError', 'error'), } - KNOWN_FAKE_BASES = [] # list of classes that pretend to be base classes but are mere wrappers. + KNOWN_FAKE_BASES = [] + # list of classes that pretend to be base classes but are mere wrappers, and their defining modules + # [(class, module),...] -- real objects, not names try: import sip as sip_module # Qt specifically likes it if hasattr(sip_module, 'wrapper'): - KNOWN_FAKE_BASES.append(sip_module.wrapper) + KNOWN_FAKE_BASES.append((sip_module.wrapper, sip_module)) if hasattr(sip_module, 'simplewrapper'): - KNOWN_FAKE_BASES.append(sip_module.simplewrapper) + KNOWN_FAKE_BASES.append((sip_module.simplewrapper, sip_module)) del sip_module except: pass @@ -1642,10 +1646,12 @@ class ModuleRedeclarator(object): skip_qualifiers = [p_modname, BUILTIN_MOD_NAME, 'exceptions'] skip_qualifiers.extend(self.KNOWN_FAKE_REEXPORTERS.get(p_modname, ())) bases_list = [] # what we'll render in the class decl - for base in bases: # somehow import every base class - if base in self.KNOWN_FAKE_BASES: + for base in bases: + if [1 for (cls, mdl) in self.KNOWN_FAKE_BASES if cls == base and mdl != self.module]: + # our base is a wrapper and our module is not its defining module skipped_bases.append(str(base)) continue + # somehow import every base class base_name = base.__name__ qual_module_name = self.qualifierOf(base, skip_qualifiers) got_existing_import = False @@ -1662,7 +1668,7 @@ class ModuleRedeclarator(object): else: bases_list.append(base_name) base_def = "(" + ", ".join(bases_list) + ")" - out(indent, "class ", p_name, base_def, ":", skipped_bases and "# skipped bases: " + ", ".join(skipped_bases) or "") + out(indent, "class ", p_name, base_def, ":", skipped_bases and " # skipped bases: " + ", ".join(skipped_bases) or "") self.outDocAttr(out, p_class, indent + 1) # inner parts methods = {} @@ -2098,19 +2104,18 @@ def redoModule(name, out_name, mod_file_name, doing_builtins, imported_module_na # fails to find 'gobject._gobject'. thus we need to pull the module directly out of # sys.modules mod = sys.modules[name] - if not mod: + if mod: + action("opening %r", out_name) + outfile = fopen(out_name, "w") + action("restoring") + r = ModuleRedeclarator(mod, outfile, mod_file_name, doing_builtins=doing_builtins) + r.redo(name, imported_module_names) + action("flushing") + r.flush() + action("closing %r", out_name) + outfile.close() + else: report("Failed to find imported module in sys.modules") - #sys.exit(0) - - action("opening %r", out_name) - outfile = fopen(out_name, "w") - action("restoring") - r = ModuleRedeclarator(mod, outfile, mod_file_name, doing_builtins=doing_builtins) - r.redo(name, imported_module_names) - action("flushing") - r.flush() - action("closing %r", out_name) - outfile.close() # find_binaries functionality From 7d97b7aaf055173fbc076f107979f04493c29f91 Mon Sep 17 00:00:00 2001 From: Dmitry Cheryasov Date: Mon, 27 Jun 2011 17:53:36 +0300 Subject: [PATCH 4/7] Removed unused KNOWN_FAKE_EXPORTS data. --- python/helpers/generator3.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/python/helpers/generator3.py b/python/helpers/generator3.py index 79a6b8cfc714..ffba28af4f2c 100644 --- a/python/helpers/generator3.py +++ b/python/helpers/generator3.py @@ -1051,13 +1051,6 @@ class ModuleRedeclarator(object): "gnomecanvas": ("gnome.canvas",), } - # names that look genuinely exported but aren't. - # e.g. 'xml.parsers.expat.ExpatError' is actually defined in pyexpat, and xml.parsers.expat imports it from there. - # {'qualified_purported_module': ('name',..)} - KNOWN_FAKE_EXPORTS = { - # 'xml.parsers.expat': ('ExpatError', 'error'), - } - KNOWN_FAKE_BASES = [] # list of classes that pretend to be base classes but are mere wrappers, and their defining modules # [(class, module),...] -- real objects, not names From 39e5fead546fbf3b9b74185388dfe34ce749cacf Mon Sep 17 00:00:00 2001 From: Dmitry Cheryasov Date: Mon, 27 Jun 2011 18:30:49 +0300 Subject: [PATCH 5/7] Added modules strictly requiring generator 1.94 --- python/helpers/required_gen_version | 33 +++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/python/helpers/required_gen_version b/python/helpers/required_gen_version index 00fbe3ff2a91..4c935b6c6b70 100644 --- a/python/helpers/required_gen_version +++ b/python/helpers/required_gen_version @@ -9,13 +9,34 @@ (built-in) 1.93 # skeletons of all built-in modules are built together # Note: modules like itertools, etc are "(built-in)" and are ignored if given separately -# pygame._numericsndarray 1.81 +PyQt4.Qsci 1.94 +PyQt4.QtAssistant 1.94 +PyQt4.QtCore 1.94 +PyQt4.QtDesigner 1.94 +PyQt4.QtGui 1.94 +PyQt4.QtHelp 1.94 +PyQt4.QtNetwork 1.94 +PyQt4.QtScriptTools 1.94 +PyQt4.QtScript 1.94 +PyQt4.QtSvg 1.94 +PyQt4.QtTest 1.94 +PyQt4.Qt 1.94 +PyQt4.QtWebKit 1.94 +PyQt4.QtXmlPatterns 1.94 +PyQt4.QtXml 1.94 + +pygame.fastevent 1.94 +pygame.image 1.94 + +sip 1.94 + +pysqlite2._sqlite 1.94 +_bsddb 1.94 + +h5py.h5 1.94 +h5py.h5i 1.94 +h5py.h5g 1.94 #numpy 0.9 # older than default is ok -#numpy.core 1.2 -#numpy.core.multiarray 1.2 -#numpy.random 1.3 - -#pygame.rect 1.4 From 70767a4f164381895811c017b51cef47ecefc8ee Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Mon, 27 Jun 2011 14:07:32 +0200 Subject: [PATCH 6/7] EA-28293 - IAE: FileUtil.pathsEqual --- .../com/jetbrains/python/run/PyPluginCommonOptionsForm.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/pluginSrc/com/jetbrains/python/run/PyPluginCommonOptionsForm.java b/python/pluginSrc/com/jetbrains/python/run/PyPluginCommonOptionsForm.java index 11625f6d585a..d3c05ae99880 100644 --- a/python/pluginSrc/com/jetbrains/python/run/PyPluginCommonOptionsForm.java +++ b/python/pluginSrc/com/jetbrains/python/run/PyPluginCommonOptionsForm.java @@ -102,7 +102,8 @@ public class PyPluginCommonOptionsForm implements AbstractPyCommonOptionsForm { final List allSdks = PythonSdkType.getAllSdks(); Sdk selection = null; for (Sdk sdk : allSdks) { - if (FileUtil.pathsEqual(sdk.getHomePath(), sdkHome)) selection = sdk; + String homePath = sdk.getHomePath(); + if (homePath != null && FileUtil.pathsEqual(homePath, sdkHome)) selection = sdk; sdkList.add(sdk); } From 10ca0aa4e74e49cc7688f404aaa5bcc2ab3c9d76 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Mon, 27 Jun 2011 19:05:53 +0200 Subject: [PATCH 7/7] perform parameter rename without using BaseRefactoringProcessor, avoiding its invokeLater() call (PY-3914) --- .../refactoring/extractmethod/PyExtractMethodUtil.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/python/src/com/jetbrains/python/refactoring/extractmethod/PyExtractMethodUtil.java b/python/src/com/jetbrains/python/refactoring/extractmethod/PyExtractMethodUtil.java index 7f65c22b6d1c..7901c66ae416 100644 --- a/python/src/com/jetbrains/python/refactoring/extractmethod/PyExtractMethodUtil.java +++ b/python/src/com/jetbrains/python/refactoring/extractmethod/PyExtractMethodUtil.java @@ -17,12 +17,14 @@ import com.intellij.psi.codeStyle.CodeStyleManager; import com.intellij.psi.impl.source.codeStyle.CodeEditUtil; import com.intellij.psi.util.PsiTreeUtil; import com.intellij.refactoring.RefactoringBundle; -import com.intellij.refactoring.RefactoringFactory; import com.intellij.refactoring.extractMethod.AbstractExtractMethodDialog; import com.intellij.refactoring.extractMethod.AbstractVariableData; import com.intellij.refactoring.extractMethod.ExtractMethodDecorator; import com.intellij.refactoring.extractMethod.ExtractMethodValidator; +import com.intellij.refactoring.listeners.RefactoringElementListenerComposite; +import com.intellij.refactoring.rename.RenameUtil; import com.intellij.refactoring.util.CommonRefactoringUtil; +import com.intellij.usageView.UsageInfo; import com.intellij.util.Function; import com.intellij.util.containers.hash.HashMap; import com.jetbrains.python.PyBundle; @@ -281,7 +283,10 @@ public class PyExtractMethodUtil { final String name = parameter.getName(); final String newName = map.get(name); if (name != null && newName != null && !name.equals(newName)){ - RefactoringFactory.getInstance(project).createRename(parameter, newName).run(); + Map allRenames = new java.util.HashMap(); + allRenames.put(parameter, newName); + UsageInfo[] usages = RenameUtil.findUsages(parameter, newName, false, false, allRenames); + RenameUtil.doRename(parameter, newName, usages, project, new RefactoringElementListenerComposite()); } } // Change signature according to pass settings and