From 360f36f9a930ab4be1ef7bcbf2850c92189eb6cb Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Wed, 23 Dec 2009 18:15:08 +0300 Subject: [PATCH 01/32] advance stub version --- python/src/com/jetbrains/python/PythonLanguage.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/src/com/jetbrains/python/PythonLanguage.java b/python/src/com/jetbrains/python/PythonLanguage.java index 48728927da41..4ab403b57e35 100644 --- a/python/src/com/jetbrains/python/PythonLanguage.java +++ b/python/src/com/jetbrains/python/PythonLanguage.java @@ -21,7 +21,7 @@ public class PythonLanguage extends Language { private final IFileElementType ELTYPE_FILE = new IStubFileElementType(this) { @Override public int getStubVersion() { - return 2; + return 3; } }; private final Set> _annotators = new CopyOnWriteArraySet>(); From 258ec0e0e46071686dd0e141694516b13457a079 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Wed, 23 Dec 2009 18:31:28 +0300 Subject: [PATCH 02/32] nicer looking completion items: use type text instead of tail text --- .../jetbrains/python/psi/impl/PyReferenceExpressionImpl.java | 4 ++-- .../com/jetbrains/python/psi/resolve/VariantsProcessor.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/src/com/jetbrains/python/psi/impl/PyReferenceExpressionImpl.java b/python/src/com/jetbrains/python/psi/impl/PyReferenceExpressionImpl.java index 1ad575b1b1a8..9b62b502c25b 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyReferenceExpressionImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyReferenceExpressionImpl.java @@ -488,13 +488,13 @@ public class PyReferenceExpressionImpl extends PyElementImpl implements PyRefere if (from_import_stmt != null) { final PyReferenceExpression import_src = from_import_stmt.getImportSource(); if (import_src != null) { - processor.setNotice(" | " + import_src.getName()); + processor.setNotice(import_src.getName()); PyResolveUtil.treeCrawlUp(processor, true, import_src.resolve()); // names from that module } } } // include builtin names - processor.setNotice(" | __builtin__"); + processor.setNotice("__builtin__"); PyResolveUtil.treeCrawlUp(processor, true, PyBuiltinCache.getInstance(this).getBuiltinsFile()); // names from __builtin__ // if we're a normal module, add module's attrs diff --git a/python/src/com/jetbrains/python/psi/resolve/VariantsProcessor.java b/python/src/com/jetbrains/python/psi/resolve/VariantsProcessor.java index a0ce769dd651..26282a2d5ff0 100644 --- a/python/src/com/jetbrains/python/psi/resolve/VariantsProcessor.java +++ b/python/src/com/jetbrains/python/psi/resolve/VariantsProcessor.java @@ -41,7 +41,7 @@ public class VariantsProcessor implements PsiScopeProcessor { } protected static LookupElementBuilder setItemNotice(final LookupElementBuilder item, String notice) { - return item.setTailText(notice); + return item.setTypeText(notice); } public LookupElement[] getResult() { @@ -86,7 +86,7 @@ public class VariantsProcessor implements PsiScopeProcessor { final PyFromImportStatement from_import = (PyFromImportStatement)maybe_from_import; PyReferenceExpression src = from_import.getImportSource(); if (src != null) { - lookup_item = setItemNotice(lookup_item, " | " + src.getName()); + lookup_item = setItemNotice(lookup_item, src.getName()); } } } From 14e98ae6d15f02344581442835acb0295318d767 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Wed, 23 Dec 2009 19:46:39 +0300 Subject: [PATCH 03/32] more generic name for the class --- python/src/META-INF/python-plugin-common.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/src/META-INF/python-plugin-common.xml b/python/src/META-INF/python-plugin-common.xml index b89447092402..4b247cde2d91 100644 --- a/python/src/META-INF/python-plugin-common.xml +++ b/python/src/META-INF/python-plugin-common.xml @@ -117,7 +117,7 @@ - + From 33812f6c2e552edc21b6c17eb84f27ae8bbbcd4a Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Wed, 23 Dec 2009 21:05:08 +0300 Subject: [PATCH 04/32] hoping to have tests for Django some time soon --- .../com/jetbrains/python/fixtures/PyLightFixtureTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/testSrc/com/jetbrains/python/fixtures/PyLightFixtureTestCase.java b/python/testSrc/com/jetbrains/python/fixtures/PyLightFixtureTestCase.java index 81bd67425c77..75c55fe17e1c 100644 --- a/python/testSrc/com/jetbrains/python/fixtures/PyLightFixtureTestCase.java +++ b/python/testSrc/com/jetbrains/python/fixtures/PyLightFixtureTestCase.java @@ -53,7 +53,7 @@ public abstract class PyLightFixtureTestCase extends UsefulTestCase { return ourPyDescriptor; } - private static class PyLightProjectDescriptor implements LightProjectDescriptor { + protected static class PyLightProjectDescriptor implements LightProjectDescriptor { public ModuleType getModuleType() { return EmptyModuleType.getInstance(); } From 6d7ab288f67ad983e43d32e3558949f3715cacc0 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Thu, 24 Dec 2009 13:24:24 +0300 Subject: [PATCH 05/32] Python debugger --- python/helpers/pydev/coverage.py | 1167 +++++++++++++++++ python/helpers/pydev/pydevd.py | 954 ++++++++++++++ .../pydev/pydevd_additional_thread_info.py | 138 ++ python/helpers/pydev/pydevd_comm.py | 694 ++++++++++ python/helpers/pydev/pydevd_constants.py | 156 +++ python/helpers/pydev/pydevd_file_utils.py | 215 +++ python/helpers/pydev/pydevd_frame.py | 194 +++ python/helpers/pydev/pydevd_io.py | 31 + python/helpers/pydev/pydevd_psyco_stub.py | 36 + python/helpers/pydev/pydevd_reload.py | 200 +++ python/helpers/pydev/pydevd_resolver.py | 352 +++++ python/helpers/pydev/pydevd_tracing.py | 78 ++ python/helpers/pydev/pydevd_vars.py | 389 ++++++ python/helpers/pydev/pydevd_vm_type.py | 41 + python/helpers/pydev/runfiles.py | 530 ++++++++ python/helpers/pydev/unittest2.py | 63 + python/python.iml | 1 + python/src/META-INF/python-plugin-common.xml | 2 + .../python/PythonHelpersLocator.java | 8 +- .../python/debugger/PyDebugProcess.java | 226 ++++ .../python/debugger/PyDebugRunner.java | 81 ++ .../python/debugger/PyDebugValue.java | 60 + .../debugger/PyDebuggerEditorsProvider.java | 69 + .../python/debugger/PyDebuggerEvaluator.java | 58 + .../python/debugger/PyDebuggerException.java | 14 + .../python/debugger/PyExecutionStack.java | 53 + .../debugger/PyLineBreakpointHandler.java | 33 + .../python/debugger/PyLineBreakpointType.java | 49 + .../python/debugger/PyPositionConverter.java | 19 + .../python/debugger/PySourcePosition.java | 41 + .../python/debugger/PyStackFrame.java | 104 ++ .../python/debugger/PyStackFrameInfo.java | 34 + .../python/debugger/PySuspendContext.java | 41 + .../python/debugger/PyThreadInfo.java | 54 + .../python/psi/PyExpressionCodeFragment.java | 5 + .../impl/PyExpressionCodeFragmentImpl.java | 69 + .../python/run/CommandLinePatcher.java | 10 + .../python/run/PythonCommandLineState.java | 13 +- 38 files changed, 6279 insertions(+), 3 deletions(-) create mode 100644 python/helpers/pydev/coverage.py create mode 100644 python/helpers/pydev/pydevd.py create mode 100644 python/helpers/pydev/pydevd_additional_thread_info.py create mode 100644 python/helpers/pydev/pydevd_comm.py create mode 100644 python/helpers/pydev/pydevd_constants.py create mode 100644 python/helpers/pydev/pydevd_file_utils.py create mode 100644 python/helpers/pydev/pydevd_frame.py create mode 100644 python/helpers/pydev/pydevd_io.py create mode 100644 python/helpers/pydev/pydevd_psyco_stub.py create mode 100644 python/helpers/pydev/pydevd_reload.py create mode 100644 python/helpers/pydev/pydevd_resolver.py create mode 100644 python/helpers/pydev/pydevd_tracing.py create mode 100644 python/helpers/pydev/pydevd_vars.py create mode 100644 python/helpers/pydev/pydevd_vm_type.py create mode 100644 python/helpers/pydev/runfiles.py create mode 100644 python/helpers/pydev/unittest2.py create mode 100644 python/src/com/jetbrains/python/debugger/PyDebugProcess.java create mode 100644 python/src/com/jetbrains/python/debugger/PyDebugRunner.java create mode 100644 python/src/com/jetbrains/python/debugger/PyDebugValue.java create mode 100644 python/src/com/jetbrains/python/debugger/PyDebuggerEditorsProvider.java create mode 100644 python/src/com/jetbrains/python/debugger/PyDebuggerEvaluator.java create mode 100644 python/src/com/jetbrains/python/debugger/PyDebuggerException.java create mode 100644 python/src/com/jetbrains/python/debugger/PyExecutionStack.java create mode 100644 python/src/com/jetbrains/python/debugger/PyLineBreakpointHandler.java create mode 100644 python/src/com/jetbrains/python/debugger/PyLineBreakpointType.java create mode 100644 python/src/com/jetbrains/python/debugger/PyPositionConverter.java create mode 100644 python/src/com/jetbrains/python/debugger/PySourcePosition.java create mode 100644 python/src/com/jetbrains/python/debugger/PyStackFrame.java create mode 100644 python/src/com/jetbrains/python/debugger/PyStackFrameInfo.java create mode 100644 python/src/com/jetbrains/python/debugger/PySuspendContext.java create mode 100644 python/src/com/jetbrains/python/debugger/PyThreadInfo.java create mode 100644 python/src/com/jetbrains/python/psi/PyExpressionCodeFragment.java create mode 100644 python/src/com/jetbrains/python/psi/impl/PyExpressionCodeFragmentImpl.java create mode 100644 python/src/com/jetbrains/python/run/CommandLinePatcher.java diff --git a/python/helpers/pydev/coverage.py b/python/helpers/pydev/coverage.py new file mode 100644 index 000000000000..534f0c879553 --- /dev/null +++ b/python/helpers/pydev/coverage.py @@ -0,0 +1,1167 @@ +#!/usr/bin/python +# +# Perforce Defect Tracking Integration Project +# +# +# COVERAGE.PY -- COVERAGE TESTING +# +# Gareth Rees, Ravenbrook Limited, 2001-12-04 +# Ned Batchelder, 2004-12-12 +# http://nedbatchelder.com/code/modules/coverage.html +# +# +# 1. INTRODUCTION +# +# This module provides coverage testing for Python code. +# +# The intended readership is all Python developers. +# +# This document is not confidential. +# +# See [GDR 2001-12-04a] for the command-line interface, programmatic +# interface and limitations. See [GDR 2001-12-04b] for requirements and +# design. + +r"""Usage: + +coverage.py -x [-p] MODULE.py [ARG1 ARG2 ...] + Execute module, passing the given command-line arguments, collecting + coverage data. With the -p option, write to a temporary file containing + the machine name and process ID. + +coverage.py -e + Erase collected coverage data. + +coverage.py -waitfor + it's the same as -r -m, but... + goes to a raw_input() and waits for the files that should be executed... + +coverage.py -c + Collect data from multiple coverage files (as created by -p option above) + and store it into a single file representing the union of the coverage. + +coverage.py -r [-m] [-o dir1,dir2,...] FILE1 FILE2 ... + Report on the statement coverage for the given files. With the -m + option, show line numbers of the statements that weren't executed. + +coverage.py -a [-d dir] [-o dir1,dir2,...] FILE1 FILE2 ... + Make annotated copies of the given files, marking statements that + are executed with > and statements that are missed with !. With + the -d option, make the copies in that directory. Without the -d + option, make each copy in the same directory as the original. + +-o dir,dir2,... + Omit reporting or annotating files when their filename path starts with + a directory listed in the omit list. + e.g. python coverage.py -i -r -o c:\python23,lib\enthought\traits + +Coverage data is saved in the file .coverage by default. Set the +COVERAGE_FILE environment variable to save it somewhere else.""" + +__version__ = "2.78.20070930" # see detailed history at the end of this file. + +import compiler +import compiler.visitor +import glob +import os +import re +import string +import symbol +import sys +import threading +import token +import types +import email +from socket import gethostname + +# Python version compatibility +try: + strclass = basestring # new to 2.3 +except: + strclass = str + +# 2. IMPLEMENTATION +# +# This uses the "singleton" pattern. +# +# The word "morf" means a module object (from which the source file can +# be deduced by suitable manipulation of the __file__ attribute) or a +# filename. +# +# When we generate a coverage report we have to canonicalize every +# filename in the coverage dictionary just in case it refers to the +# module we are reporting on. It seems a shame to throw away this +# information so the data in the coverage dictionary is transferred to +# the 'cexecuted' dictionary under the canonical filenames. +# +# The coverage dictionary is called "c" and the trace function "t". The +# reason for these short names is that Python looks up variables by name +# at runtime and so execution time depends on the length of variables! +# In the bottleneck of this application it's appropriate to abbreviate +# names to increase speed. + +class StatementFindingAstVisitor(compiler.visitor.ASTVisitor): + """ A visitor for a parsed Abstract Syntax Tree which finds executable + statements. + """ + def __init__(self, statements, excluded, suite_spots): + compiler.visitor.ASTVisitor.__init__(self) + self.statements = statements + self.excluded = excluded + self.suite_spots = suite_spots + self.excluding_suite = 0 + + def doRecursive(self, node): + for n in node.getChildNodes(): + self.dispatch(n) + + visitStmt = visitModule = doRecursive + + def doCode(self, node): + if hasattr(node, 'decorators') and node.decorators: + self.dispatch(node.decorators) + self.recordAndDispatch(node.code) + else: + self.doSuite(node, node.code) + + visitFunction = visitClass = doCode + + def getFirstLine(self, node): + # Find the first line in the tree node. + lineno = node.lineno + for n in node.getChildNodes(): + f = self.getFirstLine(n) + if lineno and f: + lineno = min(lineno, f) + else: + lineno = lineno or f + return lineno + + def getLastLine(self, node): + # Find the first line in the tree node. + lineno = node.lineno + for n in node.getChildNodes(): + lineno = max(lineno, self.getLastLine(n)) + return lineno + + def doStatement(self, node): + self.recordLine(self.getFirstLine(node)) + + visitAssert = visitAssign = visitAssTuple = visitPrint = \ + visitPrintnl = visitRaise = visitSubscript = visitDecorators = \ + doStatement + + def visitPass(self, node): + # Pass statements have weird interactions with docstrings. If this + # pass statement is part of one of those pairs, claim that the statement + # is on the later of the two lines. + l = node.lineno + if l: + lines = self.suite_spots.get(l, [l, l]) + self.statements[lines[1]] = 1 + + def visitDiscard(self, node): + # Discard nodes are statements that execute an expression, but then + # discard the results. This includes function calls, so we can't + # ignore them all. But if the expression is a constant, the statement + # won't be "executed", so don't count it now. + if node.expr.__class__.__name__ != 'Const': + self.doStatement(node) + + def recordNodeLine(self, node): + # Stmt nodes often have None, but shouldn't claim the first line of + # their children (because the first child might be an ignorable line + # like "global a"). + if node.__class__.__name__ != 'Stmt': + return self.recordLine(self.getFirstLine(node)) + else: + return 0 + + def recordLine(self, lineno): + # Returns a bool, whether the line is included or excluded. + if lineno: + # Multi-line tests introducing suites have to get charged to their + # keyword. + if lineno in self.suite_spots: + lineno = self.suite_spots[lineno][0] + # If we're inside an excluded suite, record that this line was + # excluded. + if self.excluding_suite: + self.excluded[lineno] = 1 + return 0 + # If this line is excluded, or suite_spots maps this line to + # another line that is exlcuded, then we're excluded. + elif self.excluded.has_key(lineno) or \ + self.suite_spots.has_key(lineno) and \ + self.excluded.has_key(self.suite_spots[lineno][1]): + return 0 + # Otherwise, this is an executable line. + else: + self.statements[lineno] = 1 + return 1 + return 0 + + default = recordNodeLine + + def recordAndDispatch(self, node): + self.recordNodeLine(node) + self.dispatch(node) + + def doSuite(self, intro, body, exclude=0): + exsuite = self.excluding_suite + if exclude or (intro and not self.recordNodeLine(intro)): + self.excluding_suite = 1 + self.recordAndDispatch(body) + self.excluding_suite = exsuite + + def doPlainWordSuite(self, prevsuite, suite): + # Finding the exclude lines for else's is tricky, because they aren't + # present in the compiler parse tree. Look at the previous suite, + # and find its last line. If any line between there and the else's + # first line are excluded, then we exclude the else. + lastprev = self.getLastLine(prevsuite) + firstelse = self.getFirstLine(suite) + for l in range(lastprev + 1, firstelse): + if self.suite_spots.has_key(l): + self.doSuite(None, suite, exclude=self.excluded.has_key(l)) + break + else: + self.doSuite(None, suite) + + def doElse(self, prevsuite, node): + if node.else_: + self.doPlainWordSuite(prevsuite, node.else_) + + def visitFor(self, node): + self.doSuite(node, node.body) + self.doElse(node.body, node) + + visitWhile = visitFor + + def visitIf(self, node): + # The first test has to be handled separately from the rest. + # The first test is credited to the line with the "if", but the others + # are credited to the line with the test for the elif. + self.doSuite(node, node.tests[0][1]) + for t, n in node.tests[1:]: + self.doSuite(t, n) + self.doElse(node.tests[-1][1], node) + + def visitTryExcept(self, node): + self.doSuite(node, node.body) + for i in range(len(node.handlers)): + a, b, h = node.handlers[i] + if not a: + # It's a plain "except:". Find the previous suite. + if i > 0: + prev = node.handlers[i - 1][2] + else: + prev = node.body + self.doPlainWordSuite(prev, h) + else: + self.doSuite(a, h) + self.doElse(node.handlers[-1][2], node) + + def visitTryFinally(self, node): + self.doSuite(node, node.body) + self.doPlainWordSuite(node.body, node.final) + + def visitWith(self, node): + self.doSuite(node, node.body) + + def visitGlobal(self, node): + # "global" statements don't execute like others (they don't call the + # trace function), so don't record their line numbers. + pass + + +def getCoverageLoc(): + global cache_location + return cache_location + + +class CoverageException(Exception): pass + +class coverage: + # Name of the cache file (unless environment variable is set). + cache_default = ".coverage" + + # Environment variable naming the cache file. + cache_env = "COVERAGE_FILE" + + # A dictionary with an entry for (Python source file name, line number + # in that file) if that line has been executed. + c = {} + + # A map from canonical Python source file name to a dictionary in + # which there's an entry for each line number that has been + # executed. + cexecuted = {} + + # Cache of results of calling the analysis2() method, so that you can + # specify both -r and -a without doing double work. + analysis_cache = {} + + # Cache of results of calling the canonical_filename() method, to + # avoid duplicating work. + canonical_filename_cache = {} + + def __init__(self): + self.usecache = 1 + self.cache = None + self.parallel_mode = False + self.exclude_re = '' + self.nesting = 0 + self.cstack = [] + self.xstack = [] + self.relative_dir = os.path.normcase(os.path.abspath(os.curdir) + os.sep) + self.exclude('# *pragma[: ]*[nN][oO] *[cC][oO][vV][eE][rR]') + + # t(f, x, y). This method is passed to sys.settrace as a trace function. + # See [van Rossum 2001-07-20b, 9.2] for an explanation of sys.settrace and + # the arguments and return value of the trace function. + # See [van Rossum 2001-07-20a, 3.2] for a description of frame and code + # objects. + + def t(self, f, w, unused): #pragma: no cover + if w == 'line': + #print "Executing %s @ %d" % (f.f_code.co_filename, f.f_lineno) + self.c[(f.f_code.co_filename, f.f_lineno)] = 1 + for c in self.cstack: + c[(f.f_code.co_filename, f.f_lineno)] = 1 + return self.t + + def help(self, error=None): #pragma: no cover + if error: + print error + print + print __doc__ + sys.exit(1) + + def command_line(self, argv, help_fn=None): + import getopt + help_fn = help_fn or self.help + settings = {} + optmap = { + '-a': 'annotate', + '-c': 'collect', + '-d:': 'directory=', + '-e': 'erase', + '-h': 'help', + '-i': 'ignore-errors', + '-m': 'show-missing', + '-p': 'parallel-mode', + '-r': 'report', + '-x': 'execute', + '-o:': 'omit=', + } + short_opts = string.join(map(lambda o: o[1:], optmap.keys()), '') + long_opts = optmap.values() + options, args = getopt.getopt(argv, short_opts, long_opts) + for o, a in options: + if optmap.has_key(o): + settings[optmap[o]] = 1 + elif optmap.has_key(o + ':'): + settings[optmap[o + ':']] = a + elif o[2:] in long_opts: + settings[o[2:]] = 1 + elif o[2:] + '=' in long_opts: + settings[o[2:] + '='] = a + else: #pragma: no cover + pass # Can't get here, because getopt won't return anything unknown. + + if settings.get('help'): + help_fn() + + for i in ['erase', 'execute']: + for j in ['annotate', 'report', 'collect']: + if settings.get(i) and settings.get(j): + help_fn("You can't specify the '%s' and '%s' " + "options at the same time." % (i, j)) + + args_needed = (settings.get('execute') + or settings.get('annotate') + or settings.get('report')) + action = (settings.get('erase') + or settings.get('collect') + or args_needed) + if not action: + help_fn("You must specify at least one of -e, -x, -c, -r, or -a.") + if not args_needed and args: + help_fn("Unexpected arguments: %s" % " ".join(args)) + + self.parallel_mode = settings.get('parallel-mode') + self.get_ready() + + if settings.get('erase'): + self.erase() + if settings.get('execute'): + if not args: + help_fn("Nothing to do.") + sys.argv = args + self.start() + import __main__ + sys.path[0] = os.path.dirname(sys.argv[0]) + execfile(sys.argv[0], __main__.__dict__) + if settings.get('collect'): + self.collect() + if not args: + args = self.cexecuted.keys() + + ignore_errors = settings.get('ignore-errors') + show_missing = settings.get('show-missing') + directory = settings.get('directory=') + + omit = settings.get('omit=') + if omit is not None: + omit = omit.split(',') + else: + omit = [] + + if settings.get('report'): + self.report(args, show_missing, ignore_errors, omit_prefixes=omit) + if settings.get('annotate'): + self.annotate(args, directory, ignore_errors, omit_prefixes=omit) + + def use_cache(self, usecache, cache_file=None): + self.usecache = usecache + if cache_file and not self.cache: + self.cache_default = cache_file + + def get_ready(self, parallel_mode=False): + if self.usecache and not self.cache: + self.cache = getCoverageLoc() + if self.parallel_mode: + self.cache += "." + gethostname() + "." + str(os.getpid()) + self.restore() + self.analysis_cache = {} + + def start(self, parallel_mode=False): + self.get_ready() + if self.nesting == 0: #pragma: no cover + sys.settrace(self.t) + if hasattr(threading, 'settrace'): + threading.settrace(self.t) + self.nesting += 1 + + def stop(self): + self.nesting -= 1 + if self.nesting == 0: #pragma: no cover + sys.settrace(None) + if hasattr(threading, 'settrace'): + threading.settrace(None) + + def erase(self): + self.get_ready() + self.c = {} + self.analysis_cache = {} + self.cexecuted = {} + if self.cache and os.path.exists(self.cache): + os.remove(self.cache) + + def exclude(self, re): + if self.exclude_re: + self.exclude_re += "|" + self.exclude_re += "(" + re + ")" + + def begin_recursive(self): + self.cstack.append(self.c) + self.xstack.append(self.exclude_re) + + def end_recursive(self): + self.c = self.cstack.pop() + self.exclude_re = self.xstack.pop() + + # save(). Save coverage data to the coverage cache. + + def save(self): + if self.usecache and self.cache: + self.canonicalize_filenames() + cache = open(self.cache, 'wb') + import marshal + marshal.dump(self.cexecuted, cache) + cache.close() + + # restore(). Restore coverage data from the coverage cache (if it exists). + + def restore(self): + self.c = {} + self.cexecuted = {} + assert self.usecache + if os.path.exists(self.cache): + self.cexecuted = self.restore_file(self.cache) + + def restore_file(self, file_name): + try: + cache = open(file_name, 'rb') + import marshal + cexecuted = marshal.load(cache) + cache.close() + if isinstance(cexecuted, types.DictType): + return cexecuted + else: + return {} + except: + return {} + + # collect(). Collect data in multiple files produced by parallel mode + + def collect(self): + cache_dir, local = os.path.split(self.cache) + for f in os.listdir(cache_dir or '.'): + if not f.startswith(local): + continue + + full_path = os.path.join(cache_dir, f) + cexecuted = self.restore_file(full_path) + self.merge_data(cexecuted) + + def merge_data(self, new_data): + for file_name, file_data in new_data.items(): + if self.cexecuted.has_key(file_name): + self.merge_file_data(self.cexecuted[file_name], file_data) + else: + self.cexecuted[file_name] = file_data + + def merge_file_data(self, cache_data, new_data): + for line_number in new_data.keys(): + if not cache_data.has_key(line_number): + cache_data[line_number] = new_data[line_number] + + # canonical_filename(filename). Return a canonical filename for the + # file (that is, an absolute path with no redundant components and + # normalized case). See [GDR 2001-12-04b, 3.3]. + + def canonical_filename(self, filename): + if not self.canonical_filename_cache.has_key(filename): + f = filename + if os.path.isabs(f) and not os.path.exists(f): + f = os.path.basename(f) + if not os.path.isabs(f): + for path in [os.curdir] + sys.path: + g = os.path.join(path, f) + if os.path.exists(g): + f = g + break + cf = os.path.normcase(os.path.abspath(f)) + self.canonical_filename_cache[filename] = cf + return self.canonical_filename_cache[filename] + + # canonicalize_filenames(). Copy results from "c" to "cexecuted", + # canonicalizing filenames on the way. Clear the "c" map. + + def canonicalize_filenames(self): + for filename, lineno in self.c.keys(): + if filename == '': + # Can't do anything useful with exec'd strings, so skip them. + continue + f = self.canonical_filename(filename) + if not self.cexecuted.has_key(f): + self.cexecuted[f] = {} + self.cexecuted[f][lineno] = 1 + self.c = {} + + # morf_filename(morf). Return the filename for a module or file. + + def morf_filename(self, morf): + if isinstance(morf, types.ModuleType) or (hasattr(email, 'LazyImporter') and isinstance(morf, email.LazyImporter)): + if not hasattr(morf, '__file__'): + raise CoverageException("Module has no __file__ attribute.") + f = morf.__file__ + else: + f = morf + return self.canonical_filename(f) + + # analyze_morf(morf). Analyze the module or filename passed as + # the argument. If the source code can't be found, raise an error. + # Otherwise, return a tuple of (1) the canonical filename of the + # source code for the module, (2) a list of lines of statements + # in the source code, (3) a list of lines of excluded statements, + # and (4), a map of line numbers to multi-line line number ranges, for + # statements that cross lines. + + def analyze_morf(self, morf): + if self.analysis_cache.has_key(morf): + return self.analysis_cache[morf] + filename = self.morf_filename(morf) + ext = os.path.splitext(filename)[1] + if ext == '.pyc': + if not os.path.exists(filename[:-1]): + raise CoverageException( + "No source for compiled code '%s'." % filename + ) + filename = filename[:-1] + source = open(filename, 'r') + try: + lines, excluded_lines, line_map = self.find_executable_statements( + source.read(), exclude=self.exclude_re + ) + except SyntaxError, synerr: + raise CoverageException( + "Couldn't parse '%s' as Python source: '%s' at line %d" % + (filename, synerr.msg, synerr.lineno) + ) + source.close() + result = filename, lines, excluded_lines, line_map + self.analysis_cache[morf] = result + return result + + def first_line_of_tree(self, tree): + while True: + if len(tree) == 3 and type(tree[2]) == type(1): + return tree[2] + tree = tree[1] + + def last_line_of_tree(self, tree): + while True: + if len(tree) == 3 and type(tree[2]) == type(1): + return tree[2] + tree = tree[-1] + + def find_docstring_pass_pair(self, tree, spots): + for i in range(1, len(tree)): + if self.is_string_constant(tree[i]) and self.is_pass_stmt(tree[i + 1]): + first_line = self.first_line_of_tree(tree[i]) + last_line = self.last_line_of_tree(tree[i + 1]) + self.record_multiline(spots, first_line, last_line) + + def is_string_constant(self, tree): + try: + return tree[0] == symbol.stmt and tree[1][1][1][0] == symbol.expr_stmt + except: + return False + + def is_pass_stmt(self, tree): + try: + return tree[0] == symbol.stmt and tree[1][1][1][0] == symbol.pass_stmt + except: + return False + + def record_multiline(self, spots, i, j): + for l in range(i, j + 1): + spots[l] = (i, j) + + def get_suite_spots(self, tree, spots): + """ Analyze a parse tree to find suite introducers which span a number + of lines. + """ + for i in range(1, len(tree)): + if type(tree[i]) == type(()): + if tree[i][0] == symbol.suite: + # Found a suite, look back for the colon and keyword. + lineno_colon = lineno_word = None + for j in range(i - 1, 0, -1): + if tree[j][0] == token.COLON: + # Colons are never executed themselves: we want the + # line number of the last token before the colon. + lineno_colon = self.last_line_of_tree(tree[j - 1]) + elif tree[j][0] == token.NAME: + if tree[j][1] == 'elif': + # Find the line number of the first non-terminal + # after the keyword. + t = tree[j + 1] + while t and token.ISNONTERMINAL(t[0]): + t = t[1] + if t: + lineno_word = t[2] + else: + lineno_word = tree[j][2] + break + elif tree[j][0] == symbol.except_clause: + # "except" clauses look like: + # ('except_clause', ('NAME', 'except', lineno), ...) + if tree[j][1][0] == token.NAME: + lineno_word = tree[j][1][2] + break + if lineno_colon and lineno_word: + # Found colon and keyword, mark all the lines + # between the two with the two line numbers. + self.record_multiline(spots, lineno_word, lineno_colon) + + # "pass" statements are tricky: different versions of Python + # treat them differently, especially in the common case of a + # function with a doc string and a single pass statement. + self.find_docstring_pass_pair(tree[i], spots) + + elif tree[i][0] == symbol.simple_stmt: + first_line = self.first_line_of_tree(tree[i]) + last_line = self.last_line_of_tree(tree[i]) + if first_line != last_line: + self.record_multiline(spots, first_line, last_line) + self.get_suite_spots(tree[i], spots) + + def find_executable_statements(self, text, exclude=None): + # Find lines which match an exclusion pattern. + excluded = {} + suite_spots = {} + if exclude: + reExclude = re.compile(exclude) + lines = text.split('\n') + for i in range(len(lines)): + if reExclude.search(lines[i]): + excluded[i + 1] = 1 + + # Parse the code and analyze the parse tree to find out which statements + # are multiline, and where suites begin and end. + import parser + tree = parser.suite(text + '\n\n').totuple(1) + self.get_suite_spots(tree, suite_spots) + #print "Suite spots:", suite_spots + + # Use the compiler module to parse the text and find the executable + # statements. We add newlines to be impervious to final partial lines. + statements = {} + ast = compiler.parse(text + '\n\n') + visitor = StatementFindingAstVisitor(statements, excluded, suite_spots) + compiler.walk(ast, visitor, walker=visitor) + + lines = statements.keys() + lines.sort() + excluded_lines = excluded.keys() + excluded_lines.sort() + return lines, excluded_lines, suite_spots + + # format_lines(statements, lines). Format a list of line numbers + # for printing by coalescing groups of lines as long as the lines + # represent consecutive statements. This will coalesce even if + # there are gaps between statements, so if statements = + # [1,2,3,4,5,10,11,12,13,14] and lines = [1,2,5,10,11,13,14] then + # format_lines will return "1-2, 5-11, 13-14". + + def format_lines(self, statements, lines): + pairs = [] + i = 0 + j = 0 + start = None + pairs = [] + while i < len(statements) and j < len(lines): + if statements[i] == lines[j]: + if start == None: + start = lines[j] + end = lines[j] + j = j + 1 + elif start: + pairs.append((start, end)) + start = None + i = i + 1 + if start: + pairs.append((start, end)) + def stringify(pair): + start, end = pair + if start == end: + return "%d" % start + else: + return "%d-%d" % (start, end) + ret = string.join(map(stringify, pairs), ",") + return ret + + # Backward compatibility with version 1. + def analysis(self, morf): + f, s, _, m, mf = self.analysis2(morf) + return f, s, m, mf + + def analysis2(self, morf): + filename, statements, excluded, line_map = self.analyze_morf(morf) + self.canonicalize_filenames() + if not self.cexecuted.has_key(filename): + self.cexecuted[filename] = {} + missing = [] + for line in statements: + lines = line_map.get(line, [line, line]) + for l in range(lines[0], lines[1] + 1): + if self.cexecuted[filename].has_key(l): + break + else: + missing.append(line) + return (filename, statements, excluded, missing, + self.format_lines(statements, missing)) + + def relative_filename(self, filename): + """ Convert filename to relative filename from self.relative_dir. + """ + return filename.replace(self.relative_dir, "") + + def morf_name(self, morf): + """ Return the name of morf as used in report. + """ + if isinstance(morf, types.ModuleType) or (hasattr(email, 'LazyImporter') and isinstance(morf, email.LazyImporter)): + return morf.__name__ + else: + return self.relative_filename(os.path.splitext(morf)[0]) + + def filter_by_prefix(self, morfs, omit_prefixes): + """ Return list of morfs where the morf name does not begin + with any one of the omit_prefixes. + """ + filtered_morfs = [] + for morf in morfs: + for prefix in omit_prefixes: + if self.morf_name(morf).startswith(prefix): + break + else: + filtered_morfs.append(morf) + + return filtered_morfs + + def morf_name_compare(self, x, y): + return cmp(self.morf_name(x), self.morf_name(y)) + + def report(self, morfs, show_missing=1, ignore_errors=0, file=None, omit_prefixes=[]): + ''' + @param morfs: list of files that we want to get information from + + The report is created in the following format: + Name Stmts Exec Cover Missing + --------------------------------------------- + file_to_test @ 7 @ 6 @ 85% @ 8 + file_to_test2 @ 13 @ 9 @ 69% @ 12-14, 17 + --------------------------------------------- + TOTAL 20 15 75% + + @returns a list of tuples in the format ('file_to_test2', 13, 9, 69.230769230769226, '12-14, 17') + @note: 'file' param was 'out' + ''' + + + if not isinstance(morfs, types.ListType): + morfs = [morfs] + # On windows, the shell doesn't expand wildcards. Do it here. + globbed = [] + for morf in morfs: + if isinstance(morf, strclass): + globbed.extend(glob.glob(morf)) + else: + globbed.append(morf) + morfs = globbed + + morfs = self.filter_by_prefix(morfs, omit_prefixes) + morfs.sort(self.morf_name_compare) + + max_name = max([5, ] + map(len, map(self.morf_name, morfs))) + fmt_name = "%%- %ds " % max_name + fmt_err = fmt_name + "%s: %s" + header = fmt_name % "Name" + " Stmts Exec Cover" + fmt_coverage = fmt_name + "@% 6d @% 6d @% 5d%%" + if show_missing: + header = header + " Missing" + fmt_coverage = fmt_coverage + "@ %s" + if not file: + file = sys.stdout + print >> file, header + print >> file, "-" * len(header) + total_statements = 0 + total_executed = 0 + for morf in morfs: + name = self.morf_name(morf) + try: + _, statements, _, missing, readable = self.analysis2(morf) + n = len(statements) + m = n - len(missing) + if n > 0: + pc = 100.0 * m / n + else: + pc = 100.0 + args = (morf, n, m, pc) + if show_missing: + args = args + (readable,) + print >> file, fmt_coverage % args + total_statements = total_statements + n + total_executed = total_executed + m + except KeyboardInterrupt: #pragma: no cover + raise + except: + if not ignore_errors: + typ, msg = sys.exc_info()[:2] + print >> file, fmt_err % (morf, typ, msg) + if len(morfs) > 1: + print >> file, "-" * len(header) + if total_statements > 0: + pc = 100.0 * total_executed / total_statements + else: + pc = 100.0 + args = ("TOTAL", total_statements, total_executed, pc) + if show_missing: + args = args + ("",) + print >> file, fmt_coverage % args + + # annotate(morfs, ignore_errors). + + blank_re = re.compile(r"\s*(#|$)") + else_re = re.compile(r"\s*else\s*:\s*(#|$)") + + def annotate(self, morfs, directory=None, ignore_errors=0, omit_prefixes=[]): + morfs = self.filter_by_prefix(morfs, omit_prefixes) + for morf in morfs: + try: + filename, statements, excluded, missing, _ = self.analysis2(morf) + self.annotate_file(filename, statements, excluded, missing, directory) + except KeyboardInterrupt: + raise + except: + if not ignore_errors: + raise + + def annotate_file(self, filename, statements, excluded, missing, directory=None): + source = open(filename, 'r') + if directory: + dest_file = os.path.join(directory, + os.path.basename(filename) + + ',cover') + else: + dest_file = filename + ',cover' + dest = open(dest_file, 'w') + lineno = 0 + i = 0 + j = 0 + covered = 1 + while 1: + line = source.readline() + if line == '': + break + lineno = lineno + 1 + while i < len(statements) and statements[i] < lineno: + i = i + 1 + while j < len(missing) and missing[j] < lineno: + j = j + 1 + if i < len(statements) and statements[i] == lineno: + covered = j >= len(missing) or missing[j] > lineno + if self.blank_re.match(line): + dest.write(' ') + elif self.else_re.match(line): + # Special logic for lines containing only 'else:'. + # See [GDR 2001-12-04b, 3.2]. + if i >= len(statements) and j >= len(missing): + dest.write('! ') + elif i >= len(statements) or j >= len(missing): + dest.write('> ') + elif statements[i] == missing[j]: + dest.write('! ') + else: + dest.write('> ') + elif lineno in excluded: + dest.write('- ') + elif covered: + dest.write('> ') + else: + dest.write('! ') + dest.write(line) + source.close() + dest.close() + + +# Module functions call methods in the singleton object. +def use_cache(*args, **kw): + return the_coverage.use_cache(*args, **kw) + +def start(*args, **kw): + return the_coverage.start(*args, **kw) + +def stop(*args, **kw): + return the_coverage.stop(*args, **kw) + +def erase(*args, **kw): + return the_coverage.erase(*args, **kw) + +def begin_recursive(*args, **kw): + return the_coverage.begin_recursive(*args, **kw) + +def end_recursive(*args, **kw): + return the_coverage.end_recursive(*args, **kw) + +def exclude(*args, **kw): + return the_coverage.exclude(*args, **kw) + +def analysis(*args, **kw): + return the_coverage.analysis(*args, **kw) + +def analysis2(*args, **kw): + return the_coverage.analysis2(*args, **kw) + +def report(*args, **kw): + return the_coverage.report(*args, **kw) + +def annotate(*args, **kw): + return the_coverage.annotate(*args, **kw) + +def annotate_file(*args, **kw): + return the_coverage.annotate_file(*args, **kw) + + +# Command-line interface. +if __name__ == '__main__': +# it's the same as -r -m, but... +# goes to a raw_input() and waits for the files that should be executed... + + global cache_location #let's set the cache location now... + cache_location = sys.argv[1] #first parameter is the cache location. + sys.argv.remove(cache_location) + print cache_location + + global the_coverage + # Singleton object. + the_coverage = coverage() + + if len(sys.argv) == 2: + + if '-waitfor' == sys.argv[1]: + sys.argv.remove('-waitfor') + sys.argv.append('-r') + sys.argv.append('-m') + + #second gets the files to be executed + s = raw_input() + s = s.replace('\r', '') + s = s.replace('\n', '') + files = s.split('|') + files = [v for v in files if len(v) > 0] + sys.argv += files + + if '-x' in sys.argv: + # Save coverage data when Python exits. (The atexit module wasn't + # introduced until Python 2.0, so use sys.exitfunc when it's not + # available.) + try: + import atexit + atexit.register(the_coverage.save) + except ImportError: + sys.exitfunc = the_coverage.save + + the_coverage.command_line(sys.argv[1:]) + + +# A. REFERENCES +# +# [GDR 2001-12-04a] "Statement coverage for Python"; Gareth Rees; +# Ravenbrook Limited; 2001-12-04; +# . +# +# [GDR 2001-12-04b] "Statement coverage for Python: design and +# analysis"; Gareth Rees; Ravenbrook Limited; 2001-12-04; +# . +# +# [van Rossum 2001-07-20a] "Python Reference Manual (releae 2.1.1)"; +# Guide van Rossum; 2001-07-20; +# . +# +# [van Rossum 2001-07-20b] "Python Library Reference"; Guido van Rossum; +# 2001-07-20; . +# +# +# B. DOCUMENT HISTORY +# +# 2001-12-04 GDR Created. +# +# 2001-12-06 GDR Added command-line interface and source code +# annotation. +# +# 2001-12-09 GDR Moved design and interface to separate documents. +# +# 2001-12-10 GDR Open cache file as binary on Windows. Allow +# simultaneous -e and -x, or -a and -r. +# +# 2001-12-12 GDR Added command-line help. Cache analysis so that it +# only needs to be done once when you specify -a and -r. +# +# 2001-12-13 GDR Improved speed while recording. Portable between +# Python 1.5.2 and 2.1.1. +# +# 2002-01-03 GDR Module-level functions work correctly. +# +# 2002-01-07 GDR Update sys.path when running a file with the -x option, +# so that it matches the value the program would get if it were run on +# its own. +# +# 2004-12-12 NMB Significant code changes. +# - Finding executable statements has been rewritten so that docstrings and +# other quirks of Python execution aren't mistakenly identified as missing +# lines. +# - Lines can be excluded from consideration, even entire suites of lines. +# - The filesystem cache of covered lines can be disabled programmatically. +# - Modernized the code. +# +# 2004-12-14 NMB Minor tweaks. Return 'analysis' to its original behavior +# and add 'analysis2'. Add a global for 'annotate', and factor it, adding +# 'annotate_file'. +# +# 2004-12-31 NMB Allow for keyword arguments in the module global functions. +# Thanks, Allen. +# +# 2005-12-02 NMB Call threading.settrace so that all threads are measured. +# Thanks Martin Fuzzey. Add a file argument to report so that reports can be +# captured to a different destination. +# +# 2005-12-03 NMB coverage.py can now measure itself. +# +# 2005-12-04 NMB Adapted Greg Rogers' patch for using relative filenames, +# and sorting and omitting files to report on. +# +# 2006-07-23 NMB Applied Joseph Tate's patch for function decorators. +# +# 2006-08-21 NMB Applied Sigve Tjora and Mark van der Wal's fixes for argument +# handling. +# +# 2006-08-22 NMB Applied Geoff Bache's parallel mode patch. +# +# 2006-08-23 NMB Refactorings to improve testability. Fixes to command-line +# logic for parallel mode and collect. +# +# 2006-08-25 NMB "#pragma: nocover" is excluded by default. +# +# 2006-09-10 NMB Properly ignore docstrings and other constant expressions that +# appear in the middle of a function, a problem reported by Tim Leslie. +# Minor changes to avoid lint warnings. +# +# 2006-09-17 NMB coverage.erase() shouldn't clobber the exclude regex. +# Change how parallel mode is invoked, and fix erase() so that it erases the +# cache when called programmatically. +# +# 2007-07-21 NMB In reports, ignore code executed from strings, since we can't +# do anything useful with it anyway. +# Better file handling on Linux, thanks Guillaume Chazarain. +# Better shell support on Windows, thanks Noel O'Boyle. +# Python 2.2 support maintained, thanks Catherine Proulx. +# +# 2007-07-22 NMB Python 2.5 now fully supported. The method of dealing with +# multi-line statements is now less sensitive to the exact line that Python +# reports during execution. Pass statements are handled specially so that their +# disappearance during execution won't throw off the measurement. +# +# 2007-07-23 NMB Now Python 2.5 is *really* fully supported: the body of the +# new with statement is counted as executable. +# +# 2007-07-29 NMB Better packaging. +# +# 2007-09-30 NMB Don't try to predict whether a file is Python source based on +# the extension. Extensionless files are often Pythons scripts. Instead, simply +# parse the file and catch the syntax errors. Hat tip to Ben Finney. + +# C. COPYRIGHT AND LICENCE +# +# Copyright 2001 Gareth Rees. All rights reserved. +# Copyright 2004-2007 Ned Batchelder. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the +# distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# HOLDERS AND CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR +# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH +# DAMAGE. +# diff --git a/python/helpers/pydev/pydevd.py b/python/helpers/pydev/pydevd.py new file mode 100644 index 000000000000..bc15188f4061 --- /dev/null +++ b/python/helpers/pydev/pydevd.py @@ -0,0 +1,954 @@ +#IMPORTANT: pydevd_constants must be the 1st thing defined because it'll keep a reference to the original sys._getframe +from pydevd_constants import * #@UnusedWildImport + +from pydevd_comm import CMD_CHANGE_VARIABLE, \ + CMD_EVALUATE_EXPRESSION, \ + CMD_EXEC_EXPRESSION, \ + CMD_GET_FRAME, \ + CMD_GET_VARIABLE, \ + CMD_LIST_THREADS, \ + CMD_REMOVE_BREAK, \ + CMD_RUN, \ + CMD_SET_BREAK, \ + CMD_STEP_INTO, \ + CMD_STEP_OVER, \ + CMD_STEP_RETURN, \ + CMD_THREAD_CREATE, \ + CMD_THREAD_KILL, \ + CMD_THREAD_RUN, \ + CMD_THREAD_SUSPEND, \ + CMD_RUN_TO_LINE, \ + CMD_RELOAD_CODE, \ + CMD_VERSION, \ + GetGlobalDebugger, \ + InternalChangeVariable, \ + InternalEvaluateExpression, \ + InternalGetFrame, \ + InternalGetVariable, \ + InternalTerminateThread, \ + NetCommand, \ + NetCommandFactory, \ + PyDBDaemonThread, \ + PydevQueue, \ + ReaderThread, \ + SetGlobalDebugger, \ + WriterThread, \ + PydevdFindThreadById, \ + PydevdLog, \ + StartClient, \ + StartServer + +import pydevd_file_utils +import traceback +import pydevd_vm_type +import pydevd_tracing +import pydevd_io +import pydevd_additional_thread_info +import time + +DONT_TRACE = { + #commonly used things from the stdlib that we don't want to trace + 'threading.py':1, + 'Queue.py':1, + 'socket.py':1, + + #things from pydev that we don't want to trace + 'pydevd_additional_thread_info.py':1, + 'pydevd_comm.py':1, + 'pydevd_constants.py':1, + 'pydevd_file_utils.py':1, + 'pydevd_frame.py':1, + 'pydevd_io.py':1 , + 'pydevd_resolver.py':1 , + 'pydevd_tracing.py':1 , + 'pydevd_vars.py':1, + 'pydevd_vm_type.py':1, + 'pydevd.py':1 , + 'pydevd_psyco_stub.py':1 + } + +if IS_PY3K: + #if we try to trace io.py it seems it can get halted (see http://bugs.python.org/issue4716) + DONT_TRACE['io.py'] = 1 + + +connected = False +bufferStdOutToServer = False +bufferStdErrToServer = False + +PyDBUseLocks = True + + +#======================================================================================================================= +# PyDBCommandThread +#======================================================================================================================= +class PyDBCommandThread(PyDBDaemonThread): + + def __init__(self, pyDb): + PyDBDaemonThread.__init__(self) + self.pyDb = pyDb + self.setName('pydevd.CommandThread') + + def OnRun(self): + time.sleep(5) #this one will only start later on (because otherwise we may not have any non-daemon threads + + run_traced = True + + if pydevd_vm_type.GetVmType() == pydevd_vm_type.PydevdVmType.JYTHON and sys.hexversion <= 0x020201f0: + #don't run untraced threads if we're in jython 2.2.1 or lower + #jython bug: if we start a thread and another thread changes the tracing facility + #it affects other threads (it's not set only for the thread but globally) + #Bug: http://sourceforge.net/tracker/index.php?func=detail&aid=1870039&group_id=12867&atid=112867 + run_traced = False + + if run_traced: + pydevd_tracing.SetTrace(None) # no debugging on this thread + + try: + while not self.killReceived: + try: + self.pyDb.processInternalCommands() + except: + PydevdLog(0, 'Finishing debug communication...(2)') + time.sleep(0.5) + except: + pass + #only got this error in interpreter shutdown + #PydevdLog(0, 'Finishing debug communication...(3)') + + + + +#======================================================================================================================= +# PyDB +#======================================================================================================================= +class PyDB: + """ Main debugging class + Lots of stuff going on here: + + PyDB starts two threads on startup that connect to remote debugger (RDB) + The threads continuously read & write commands to RDB. + PyDB communicates with these threads through command queues. + Every RDB command is processed by calling processNetCommand. + Every PyDB net command is sent to the net by posting NetCommand to WriterThread queue + + Some commands need to be executed on the right thread (suspend/resume & friends) + These are placed on the internal command queue. + """ + + RUNNING_THREAD_IDS = {} #this is a dict of thread ids pointing to thread ids. Whenever a command + #is passed to the java end that acknowledges that a thread was created, + #the thread id should be passed here -- and if at some time we do not find + #that thread alive anymore, we must remove it from this list and make + #the java side know that the thread was killed. + + def __init__(self): + SetGlobalDebugger(self) + pydevd_tracing.ReplaceSysSetTraceFunc() + self.reader = None + self.writer = None + self.quitting = None + self.cmdFactory = NetCommandFactory() + self.cmdQueue = {} # the hash of Queues. Key is thread id, value is thread + self.breakpoints = {} + self.readyToRun = False + self.lock = threading.RLock() + self.internalQueueLock = threading.Lock() + self.finishDebuggingSession = False + + def acquire(self): + if PyDBUseLocks: + self.lock.acquire() + return True + + def release(self): + if PyDBUseLocks: + self.lock.release() + return True + + def initializeNetwork(self, sock): + try: + sock.settimeout(None) # infinite, no timeouts from now on - jython does not have it + except: + pass + self.writer = WriterThread(sock) + self.reader = ReaderThread(sock) + self.writer.start() + self.reader.start() + + time.sleep(0.1) # give threads time to start + + def connect(self, host, port): + if host: + s = StartClient(host, port) + else: + s = StartServer(port) + + self.initializeNetwork(s) + + + def getInternalQueue(self, thread_id): + """ returns intenal command queue for a given thread. + if new queue is created, notify the RDB about it """ + try: + return self.cmdQueue[thread_id] + except KeyError: + self.internalQueueLock.acquire() + try: + self.cmdQueue[thread_id] = PydevQueue.Queue() + all_threads = threading.enumerate() + cmd = None + for t in all_threads: + if GetThreadId(t) == thread_id: + if not hasattr(t, 'additionalInfo'): + #see http://sourceforge.net/tracker/index.php?func=detail&aid=1955428&group_id=85796&atid=577329 + #Let's create the additional info right away! + t.additionalInfo = pydevd_additional_thread_info.PyDBAdditionalThreadInfo() + + self.RUNNING_THREAD_IDS[thread_id] = t + cmd = self.cmdFactory.makeThreadCreatedMessage(t) + break + + if cmd: + PydevdLog(2, "found a new thread ", str(thread_id)) + self.writer.addCommand(cmd) + else: + PydevdLog(0, "could not find thread by id to register") + finally: + self.internalQueueLock.release() + + return self.cmdQueue[thread_id] + + + def postInternalCommand(self, int_cmd, thread_id): + """ if thread_id is *, post to all """ + if thread_id == "*": + for k in self.cmdQueue.keys(): + self.cmdQueue[k].put(int_cmd) + + else: + queue = self.getInternalQueue(thread_id) + queue.put(int_cmd) + + def checkOutput(self, out, outCtx): + '''Checks the output to see if we have to send some buffered output to the debug server + + @param out: sys.stdout or sys.stderr + @param outCtx: the context indicating: 1=stdout and 2=stderr (to know the colors to write it) + ''' + + try: + v = out.getvalue() + if v: + self.cmdFactory.makeIoMessage(v, outCtx, self) + except: + traceback.print_exc() + + + def processInternalCommands(self): + '''This function processes internal commands + ''' + + self.acquire() + try: + if bufferStdOutToServer: + self.checkOutput(sys.stdoutBuf, 1) #@UndefinedVariable + + if bufferStdErrToServer: + self.checkOutput(sys.stderrBuf, 2) #@UndefinedVariable + + currThreadId = GetThreadId(threading.currentThread()) + threads = threading.enumerate() + foundNonPyDBDaemonThread = False + foundThreads = {} + + for t in threads: + tId = GetThreadId(t) + if t.isAlive(): + foundThreads[tId] = tId + + if not isinstance(t, PyDBDaemonThread): + foundNonPyDBDaemonThread = True + queue = self.getInternalQueue(GetThreadId(t)) + cmdsToReadd = [] #some commands must be processed by the thread itself... if that's the case, + #we will re-add the commands to the queue after executing. + try: + while True: + int_cmd = queue.get(False) + if int_cmd.canBeExecutedBy(currThreadId): + PydevdLog(2, "processing internal command ", str(int_cmd)) + int_cmd.doIt(self) + else: + PydevdLog(2, "NOT processing internal command ", str(int_cmd)) + cmdsToReadd.append(int_cmd) + + except PydevQueue.Empty: + for int_cmd in cmdsToReadd: + queue.put(int_cmd) + # this is how we exit + + if not foundNonPyDBDaemonThread: + self.finishDebuggingSession = True + for t in threads: + if hasattr(t, 'doKill'): + t.doKill() + + for tId in self.RUNNING_THREAD_IDS.keys(): + try: + if not DictContains(foundThreads, tId): + self.processThreadNotAlive(tId) + except: + sys.stderr.write('Error iterating through %s (%s) - %s\n' % (foundThreads, foundThreads.__class__, dir(foundThreads))) + raise + + finally: + self.release() + + def processNetCommand(self, cmd_id, seq, text): + '''Processes a command received from the Java side + + @param cmd_id: the id of the command + @param seq: the sequence of the command + @param text: the text received in the command + + @note: this method is run as a big switch... after doing some tests, it's not clear whether changing it for + a dict id --> function call will have better performance result. A simple test with xrange(10000000) showed + that the gains from having a fast access to what should be executed are lost because of the function call in + a way that if we had 10 elements in the switch the if..elif are better -- but growing the number of choices + makes the solution with the dispatch look better -- so, if this gets more than 20-25 choices at some time, + it may be worth refactoring it (actually, reordering the ifs so that the ones used mostly come before + probably will give better performance). + ''' + + self.acquire() + try: + try: + cmd = None + if cmd_id == CMD_RUN: + self.readyToRun = True + + elif cmd_id == CMD_VERSION: + # response is version number + cmd = self.cmdFactory.makeVersionMessage(seq) + + elif cmd_id == CMD_LIST_THREADS: + # response is a list of threads + cmd = self.cmdFactory.makeListThreadsMessage(seq) + + elif cmd_id == CMD_THREAD_KILL: + int_cmd = InternalTerminateThread(text) + self.postInternalCommand(int_cmd, text) + + elif cmd_id == CMD_THREAD_SUSPEND: + t = PydevdFindThreadById(text) + if t: + additionalInfo = None + try: + additionalInfo = t.additionalInfo + except AttributeError: + pass #that's ok, no info currently set + + if additionalInfo is not None: + for frame in additionalInfo.IterFrames(): + frame.f_trace = self.trace_dispatch + SetTraceForParents(frame, self.trace_dispatch) + del frame + + self.setSuspend(t, CMD_THREAD_SUSPEND) + + elif cmd_id == CMD_THREAD_RUN: + t = PydevdFindThreadById(text) + if t: + t.additionalInfo.pydev_step_cmd = None + t.additionalInfo.pydev_step_stop = None + t.additionalInfo.pydev_state = STATE_RUN + + elif cmd_id == CMD_STEP_INTO or cmd_id == CMD_STEP_OVER or cmd_id == CMD_STEP_RETURN: + #we received some command to make a single step + t = PydevdFindThreadById(text) + if t: + t.additionalInfo.pydev_step_cmd = cmd_id + t.additionalInfo.pydev_state = STATE_RUN + + elif cmd_id == CMD_RUN_TO_LINE: + #we received some command to make a single step + thread_id, line, func_name = text.split('\t', 2) + t = PydevdFindThreadById(thread_id) + if t: + t.additionalInfo.pydev_step_cmd = cmd_id + t.additionalInfo.pydev_next_line = int(line) + t.additionalInfo.pydev_func_name = func_name + t.additionalInfo.pydev_state = STATE_RUN + + + elif cmd_id == CMD_RELOAD_CODE: + #we received some command to make a reload of a module + module_name = text.strip() + from pydevd_reload import xreload + if not DictContains(sys.modules, module_name): + if '.' in module_name: + new_module_name = module_name.split('.')[-1] + if DictContains(sys.modules, new_module_name): + module_name = new_module_name + + if not DictContains(sys.modules, module_name): + sys.stderr.write('pydev debugger: Unable to find module to reload: "'+module_name+'".\n') + sys.stderr.write('pydev debugger: This usually means you are trying to reload the __main__ module (which cannot be reloaded).\n') + + else: + sys.stderr.write('pydev debugger: Reloading: '+module_name+'\n') + xreload(sys.modules[module_name]) + + + elif cmd_id == CMD_CHANGE_VARIABLE: + #the text is: thread\tstackframe\tFRAME|GLOBAL\tattribute_to_change\tvalue_to_change + try: + thread_id, frame_id, scope, attr_and_value = text.split('\t', 3) + + tab_index = attr_and_value.rindex('\t') + attr = attr_and_value[0:tab_index].replace('\t', '.') + value = attr_and_value[tab_index + 1:] + int_cmd = InternalChangeVariable(seq, thread_id, frame_id, scope, attr, value) + self.postInternalCommand(int_cmd, thread_id) + + except: + traceback.print_exc() + + elif cmd_id == CMD_GET_VARIABLE: + #we received some command to get a variable + #the text is: thread_id\tframe_id\tFRAME|GLOBAL\tattributes* + try: + thread_id, frame_id, scopeattrs = text.split('\t', 2) + + if scopeattrs.find('\t') != -1: # there are attibutes beyond scope + scope, attrs = scopeattrs.split('\t', 1) + else: + scope, attrs = (scopeattrs, None) + + int_cmd = InternalGetVariable(seq, thread_id, frame_id, scope, attrs) + self.postInternalCommand(int_cmd, thread_id) + + except: + traceback.print_exc() + + elif cmd_id == CMD_GET_FRAME: + thread_id, frame_id, scope = text.split('\t', 2) + + int_cmd = InternalGetFrame(seq, thread_id, frame_id) + self.postInternalCommand(int_cmd, thread_id) + + elif cmd_id == CMD_SET_BREAK: + #func name: 'None': match anything. Empty: match global, specified: only method context. + + #command to add some breakpoint. + # text is file\tline. Add to breakpoints dictionary + file, line, condition = text.split('\t', 2) + if condition.startswith('**FUNC**'): + func_name, condition = condition.split('\t', 1) + + #We must restore new lines and tabs as done in + #AbstractDebugTarget.breakpointAdded + condition = condition.replace("@_@NEW_LINE_CHAR@_@", '\n').\ + replace("@_@TAB_CHAR@_@", '\t').strip() + + func_name = func_name[8:] + else: + func_name = 'None' #Match anything if not specified. + + + file = pydevd_file_utils.NormFileToServer(file) + + if not os.path.exists(file): + sys.stderr.write('pydev debugger: warning: trying to add breakpoint'\ + ' to file that does not exist: %s (will have no effect)\n' % (file,)) + + line = int(line) + + if DEBUG_TRACE_BREAKPOINTS > 0: + sys.stderr.write('Added breakpoint:%s - line:%s - func_name:%s\n' % (file, line, func_name)) + + if DictContains(self.breakpoints, file): + breakDict = self.breakpoints[file] + else: + breakDict = {} + + if len(condition) <= 0 or condition == None or condition == "None": + breakDict[line] = (True, None, func_name) + else: + breakDict[line] = (True, condition, func_name) + + + self.breakpoints[file] = breakDict + + #and enable the tracing for existing threads (because there may be frames being executed that + #are currently untraced). + threads = threading.enumerate() + for t in threads: + if not t.getName().startswith('pydevd.'): + #TODO: optimize so that we only actually add that tracing if it's in + #the new breakpoint context. + additionalInfo = None + try: + additionalInfo = t.additionalInfo + except AttributeError: + pass #that's ok, no info currently set + + if additionalInfo is not None: + for frame in additionalInfo.IterFrames(): + frame.f_trace = self.trace_dispatch + SetTraceForParents(frame, self.trace_dispatch) + del frame + + elif cmd_id == CMD_REMOVE_BREAK: + #command to remove some breakpoint + #text is file\tline. Remove from breakpoints dictionary + file, line = text.split('\t', 1) + file = pydevd_file_utils.NormFileToServer(file) + try: + line = int(line) + except ValueError: + pass + + else: + try: + del self.breakpoints[file][line] #remove the breakpoint in that line + if DEBUG_TRACE_BREAKPOINTS > 0: + sys.stderr.write('Removed breakpoint:%s\n' % (file,)) + except KeyError: + #ok, it's not there... + if DEBUG_TRACE_BREAKPOINTS > 0: + #Sometimes, when adding a breakpoint, it adds a remove command before (don't really know why) + sys.stderr.write("breakpoint not found: %s - %s\n" % (file, line)) + + elif cmd_id == CMD_EVALUATE_EXPRESSION or cmd_id == CMD_EXEC_EXPRESSION: + #command to evaluate the given expression + #text is: thread\tstackframe\tLOCAL\texpression + thread_id, frame_id, scope, expression = text.split('\t', 3) + int_cmd = InternalEvaluateExpression(seq, thread_id, frame_id, expression, + cmd_id == CMD_EXEC_EXPRESSION) + self.postInternalCommand(int_cmd, thread_id) + + + else: + #I have no idea what this is all about + cmd = self.cmdFactory.makeErrorMessage(seq, "unexpected command " + str(cmd_id)) + + if cmd is not None: + self.writer.addCommand(cmd) + del cmd + + except Exception: + traceback.print_exc() + cmd = self.cmdFactory.makeErrorMessage(seq, + "Unexpected exception in processNetCommand.\nInitial params: %s" % ((cmd_id, seq, text),)) + + self.writer.addCommand(cmd) + finally: + self.release() + + def processThreadNotAlive(self, threadId): + """ if thread is not alive, cancel trace_dispatch processing """ + thread = self.RUNNING_THREAD_IDS.get(threadId, None) + if thread is None: + return + + del self.RUNNING_THREAD_IDS[threadId] + wasNotified = thread.additionalInfo.pydev_notify_kill + + if not wasNotified: + cmd = self.cmdFactory.makeThreadKilledMessage(threadId) + self.writer.addCommand(cmd) + thread.additionalInfo.pydev_notify_kill = True + + def setSuspend(self, thread, stop_reason): + thread.additionalInfo.pydev_state = STATE_SUSPEND + thread.stop_reason = stop_reason + + def doWaitSuspend(self, thread, frame, event, arg): #@UnusedVariable + """ busy waits until the thread state changes to RUN + it expects thread's state as attributes of the thread. + Upon running, processes any outstanding Stepping commands. + """ + self.processInternalCommands() + cmd = self.cmdFactory.makeThreadSuspendMessage(GetThreadId(thread), frame, thread.stop_reason) + self.writer.addCommand(cmd) + + info = thread.additionalInfo + while info.pydev_state == STATE_SUSPEND and not self.finishDebuggingSession: + self.processInternalCommands() + time.sleep(0.2) + + #process any stepping instructions + if info.pydev_step_cmd == CMD_STEP_INTO: + info.pydev_step_stop = None + + elif info.pydev_step_cmd == CMD_STEP_OVER: + info.pydev_step_stop = frame + if frame.f_trace is None: + frame.f_trace = self.trace_dispatch + SetTraceForParents(frame, self.trace_dispatch) + + elif info.pydev_step_cmd == CMD_RUN_TO_LINE: + if frame.f_trace is None: + frame.f_trace = self.trace_dispatch + SetTraceForParents(frame, self.trace_dispatch) + + if event == 'line': + #If we're already in the correct context, we have to stop it now, because we can act only on + #line events -- if a return was the next statement it wouldn't work (so, we have this code + #repeated at pydevd_frame). + stop = False + curr_func_name = frame.f_code.co_name + + #global context is set with an empty name + if curr_func_name in ('?', ''): + curr_func_name = '' + + if curr_func_name == info.pydev_func_name: + line = info.pydev_next_line + if frame.f_lineno == line: + stop = True + else: + if frame.f_trace is None: + frame.f_trace = self.trace_dispatch + frame.f_lineno = line + frame.f_trace = None + stop = True + if stop: + info.pydev_state = STATE_SUSPEND + self.doWaitSuspend(thread, frame, event, arg) + return + + + elif info.pydev_step_cmd == CMD_STEP_RETURN: + back_frame = frame.f_back + if back_frame is not None: + #steps back to the same frame (in a return call it will stop in the 'back frame' for the user) + info.pydev_step_stop = frame + if frame.f_trace is None: + frame.f_trace = self.trace_dispatch + SetTraceForParents(frame, self.trace_dispatch) + else: + #No back frame?!? -- this happens in jython when we have some frame created from an awt event + #(the previous frame would be the awt event, but this doesn't make part of 'jython', only 'java') + #so, if we're doing a step return in this situation, it's the same as just making it run + info.pydev_step_stop = None + info.pydev_step_cmd = None + info.pydev_state = STATE_RUN + + del frame + cmd = self.cmdFactory.makeThreadRunMessage(GetThreadId(thread), info.pydev_step_cmd) + self.writer.addCommand(cmd) + + + + def trace_dispatch(self, frame, event, arg): + ''' This is the callback used when we enter some context in the debugger. + + We also decorate the thread we are in with info about the debugging. + The attributes added are: + pydev_state + pydev_step_stop + pydev_step_cmd + pydev_notify_kill + ''' + try: + if self.finishDebuggingSession: + #that was not working very well because jython gave some socket errors + threads = threading.enumerate() + for t in threads: + if hasattr(t, 'doKill'): + t.doKill() + return None + + filename, base = pydevd_file_utils.GetFilenameAndBase(frame) + + if DictContains(DONT_TRACE, base): #we don't want to debug threading or anything related to pydevd + return None + + #print('trace_dispatch', base, frame.f_lineno, event, frame.f_code.co_name) + try: + #this shouldn't give an exception, but it could happen... (python bug) + #see http://mail.python.org/pipermail/python-bugs-list/2007-June/038796.html + #and related bug: http://bugs.python.org/issue1733757 + t = threading.currentThread() + except: + frame.f_trace = self.trace_dispatch + return self.trace_dispatch + + # if thread is not alive, cancel trace_dispatch processing + if not t.isAlive(): + self.processThreadNotAlive(GetThreadId(t)) + return None # suspend tracing + + try: + additionalInfo = t.additionalInfo + except AttributeError: + t.additionalInfo = additionalInfo = pydevd_additional_thread_info.PyDBAdditionalThreadInfo() + + #always keep a reference to the topmost frame so that we're able to start tracing it (if it was untraced) + #that's needed when a breakpoint is added in a current frame for a currently untraced context. + + #each new frame... + dbFrame = additionalInfo.CreateDbFrame(self, filename, additionalInfo, t, frame) + return dbFrame.trace_dispatch(frame, event, arg) + + except SystemExit: + return None + + except Exception: + #Log it + traceback.print_exc() + return None + + if USE_PSYCO_OPTIMIZATION: + try: + import psyco + trace_dispatch = psyco.proxy(trace_dispatch) + processNetCommand = psyco.proxy(processNetCommand) + processInternalCommands = psyco.proxy(processInternalCommands) + doWaitSuspend = psyco.proxy(doWaitSuspend) + getInternalQueue = psyco.proxy(getInternalQueue) + except ImportError: + if hasattr(sys, 'exc_clear'): #jython does not have it + sys.exc_clear() #don't keep the traceback (let's keep it clear for when we go to the point of executing client code) + + if not sys.platform.startswith("java") and not sys.platform.startswith("cli"): + sys.stderr.write("pydev debugger: warning: psyco not available for speedups (the debugger will still work correctly, but a bit slower)\n") + + + def run(self, file, globals=None, locals=None): + + if globals is None: + #patch provided by: Scott Schlesier - when script is run, it does not + #use globals from pydevd: + #This will prevent the pydevd script from contaminating the namespace for the script to be debugged + + #pretend pydevd is not the main module, and + #convince the file to be debugged that it was loaded as main + sys.modules['pydevd'] = sys.modules['__main__'] + sys.modules['pydevd'].__name__ = 'pydevd' + + from imp import new_module + m = new_module('__main__') + sys.modules['__main__'] = m + m.__file__ = file + globals = m.__dict__ + + if locals is None: + locals = globals + + #Predefined (writable) attributes: __name__ is the module's name; + #__doc__ is the module's documentation string, or None if unavailable; + #__file__ is the pathname of the file from which the module was loaded, + #if it was loaded from a file. The __file__ attribute is not present for + #C modules that are statically linked into the interpreter; for extension modules + #loaded dynamically from a shared library, it is the pathname of the shared library file. + + + #I think this is an ugly hack, bug it works (seems to) for the bug that says that sys.path should be the same in + #debug and run. + if m.__file__.startswith(sys.path[0]): + #print >> sys.stderr, 'Deleting: ', sys.path[0] + del sys.path[0] + + #now, the local directory has to be added to the pythonpath + #sys.path.insert(0, os.getcwd()) + #Changed: it's not the local directory, but the directory of the file launched + #The file being run ust be in the pythonpath (even if it was not before) + sys.path.insert(0, os.path.split(file)[0]) + + # for completness, we'll register the pydevd.reader & pydevd.writer threads + net = NetCommand(str(CMD_THREAD_CREATE), 0, '') + self.writer.addCommand(net) + net = NetCommand(str(CMD_THREAD_CREATE), 0, '') + self.writer.addCommand(net) + + pydevd_tracing.SetTrace(self.trace_dispatch) + try: + #not available in jython! + threading.settrace(self.trace_dispatch) # for all future threads + except: + pass + + while not self.readyToRun: + time.sleep(0.1) # busy wait until we receive run command + + PyDBCommandThread(debugger).start() + + if not IS_PY3K: + execfile(file, globals, locals) #execute the script + else: + #We need to compile before so that the module name is correct + obj = compile(open(file).read()+"\n", file, 'exec') + exec(obj, globals, locals) #execute the script + + +def processCommandLine(argv): + """ parses the arguments. + removes our arguments from the command line """ + retVal = {} + retVal['client'] = '' + retVal['server'] = False + retVal['port'] = 0 + retVal['file'] = '' + i = 0 + del argv[0] + while (i < len(argv)): + if (argv[i] == '--port'): + del argv[i] + retVal['port'] = int(argv[i]) + del argv[i] + elif (argv[i] == '--vm_type'): + del argv[i] + retVal['vm_type'] = argv[i] + del argv[i] + elif (argv[i] == '--client'): + del argv[i] + retVal['client'] = argv[i] + del argv[i] + elif (argv[i] == '--server'): + del argv[i] + retVal['server'] = True + elif (argv[i] == '--file'): + del argv[i] + retVal['file'] = argv[i]; + i = len(argv) # pop out, file is our last argument + elif (argv[i] == '--DEBUG_RECORD_SOCKET_READS'): + del argv[i] + retVal['DEBUG_RECORD_SOCKET_READS'] = True + else: + raise ValueError("unexpected option " + argv[i]) + return retVal + +def usage(doExit=0): + sys.stdout.write('Usage:\n') + sys.stdout.write('pydevd.py --port=N [(--client hostname) | --server] --file executable [file_options]\n') + if doExit: + sys.exit(0) + + + +def SetTraceForParents(frame, dispatch_func): + frame = frame.f_back + while frame: + if frame.f_trace is None: + frame.f_trace = dispatch_func + + frame = frame.f_back + del frame + +def settrace(host='localhost', stdoutToServer=False, stderrToServer=False, port=5678, suspend=True): + '''Sets the tracing function with the pydev debug function and initializes needed facilities. + + @param host: the user may specify another host, if the debug server is not in the same machine + @param stdoutToServer: when this is true, the stdout is passed to the debug server + @param stderrToServer: when this is true, the stderr is passed to the debug server + so that they are printed in its console and not in this process console. + @param port: specifies which port to use for communicating with the server (note that the server must be started + in the same port). @note: currently it's hard-coded at 5678 in the client + @param suspend: whether a breakpoint should be emulated as soon as this function is called. + ''' + + global connected + global bufferStdOutToServer + global bufferStdErrToServer + + if not connected : + connected = True + bufferStdOutToServer = stdoutToServer + bufferStdErrToServer = stderrToServer + + pydevd_vm_type.SetupType() + + debugger = PyDB() + debugger.connect(host, port) + + net = NetCommand(str(CMD_THREAD_CREATE), 0, '') + debugger.writer.addCommand(net) + net = NetCommand(str(CMD_THREAD_CREATE), 0, '') + debugger.writer.addCommand(net) + + if bufferStdOutToServer: + sys.stdoutBuf = pydevd_io.IOBuf() + sys.stdout = pydevd_io.IORedirector(sys.stdout, sys.stdoutBuf) #@UndefinedVariable + + if bufferStdErrToServer: + sys.stderrBuf = pydevd_io.IOBuf() + sys.stderr = pydevd_io.IORedirector(sys.stderr, sys.stderrBuf) #@UndefinedVariable + + SetTraceForParents(GetFrame(), debugger.trace_dispatch) + + t = threading.currentThread() + try: + additionalInfo = t.additionalInfo + except AttributeError: + additionalInfo = pydevd_additional_thread_info.PyDBAdditionalThreadInfo() + t.additionalInfo = additionalInfo + + while not debugger.readyToRun: + time.sleep(0.1) # busy wait until we receive run command + + if suspend: + debugger.setSuspend(t, CMD_SET_BREAK) + + #that's right, debug only threads that pass through this function + #(so, we just call sys.settrace and not threading.settrace) + #note that we do that through pydevd_tracing.SetTrace so that the tracing + #is not warned to the user! + pydevd_tracing.SetTrace(debugger.trace_dispatch) + PyDBCommandThread(debugger).start() + + else: + #ok, we're already in debug mode, with all set, so, let's just set the break + debugger = GetGlobalDebugger() + + SetTraceForParents(GetFrame(), debugger.trace_dispatch) + + t = threading.currentThread() + try: + additionalInfo = t.additionalInfo + except AttributeError: + additionalInfo = pydevd_additional_thread_info.PyDBAdditionalThreadInfo() + t.additionalInfo = additionalInfo + + pydevd_tracing.SetTrace(debugger.trace_dispatch) + if suspend: + debugger.setSuspend(t, CMD_SET_BREAK) + + +if __name__ == '__main__': + sys.stderr.write("pydev debugger: starting\n") + # parse the command line. --file is our last argument that is required + try: + setup = processCommandLine(sys.argv) + except ValueError: + traceback.print_exc() + usage(1) + + #as to get here all our imports are already resolved, the psyco module can be + #changed and we'll still get the speedups in the debugger, as those functions + #are already compiled at this time. + try: + import psyco + except ImportError: + if hasattr(sys, 'exc_clear'): #jython does not have it + sys.exc_clear() #don't keep the traceback -- clients don't want to see it + pass #that's ok, no need to mock psyco if it's not available anyways + else: + #if it's available, let's change it for a stub (pydev already made use of it) + import pydevd_psyco_stub + sys.modules['psyco'] = pydevd_psyco_stub + + + PydevdLog(2, "Executing file ", setup['file']) + PydevdLog(2, "arguments:", str(sys.argv)) + + pydevd_vm_type.SetupType(setup.get('vm_type', None)) + + DebugInfoHolder.DEBUG_RECORD_SOCKET_READS = setup.get('DEBUG_RECORD_SOCKET_READS', False) + + debugger = PyDB() + debugger.connect(setup['client'], setup['port']) + debugger.run(setup['file'], None, None) + diff --git a/python/helpers/pydev/pydevd_additional_thread_info.py b/python/helpers/pydev/pydevd_additional_thread_info.py new file mode 100644 index 000000000000..c8152ce0409d --- /dev/null +++ b/python/helpers/pydev/pydevd_additional_thread_info.py @@ -0,0 +1,138 @@ +import sys +from pydevd_constants import * #@UnusedWildImport +import threading +from pydevd_frame import PyDBFrame +import weakref + +#======================================================================================================================= +# AbstractPyDBAdditionalThreadInfo +#======================================================================================================================= +class AbstractPyDBAdditionalThreadInfo: + def __init__(self): + self.pydev_state = STATE_RUN + self.pydev_step_stop = None + self.pydev_step_cmd = None + self.pydev_notify_kill = False + + + def IterFrames(self): + raise NotImplementedError() + + def CreateDbFrame(self, mainDebugger, filename, base, additionalInfo, t, frame): + raise NotImplementedError() + + def __str__(self): + return 'State:%s Stop:%s Cmd: %s Kill:%s' % (self.pydev_state, self.pydev_step_stop, self.pydev_step_cmd, self.pydev_notify_kill) + + +#======================================================================================================================= +# PyDBAdditionalThreadInfoWithCurrentFramesSupport +#======================================================================================================================= +class PyDBAdditionalThreadInfoWithCurrentFramesSupport(AbstractPyDBAdditionalThreadInfo): + + def IterFrames(self): + #sys._current_frames(): dictionary with thread id -> topmost frame + return sys._current_frames().values() #return a copy... don't know if it's changed if we did get an iterator + + #just create the db frame directly + CreateDbFrame = PyDBFrame + +#======================================================================================================================= +# PyDBAdditionalThreadInfoWithoutCurrentFramesSupport +#======================================================================================================================= +class PyDBAdditionalThreadInfoWithoutCurrentFramesSupport(AbstractPyDBAdditionalThreadInfo): + + def __init__(self): + AbstractPyDBAdditionalThreadInfo.__init__(self) + #That's where the last frame entered is kept. That's needed so that we're able to + #trace contexts that were previously untraced and are currently active. So, the bad thing + #is that the frame may be kept alive longer than it would if we go up on the frame stack, + #and is only disposed when some other frame is removed. + #A better way would be if we could get the topmost frame for each thread, but that's + #not possible (until python 2.5 -- which is the PyDBAdditionalThreadInfoWithCurrentFramesSupport version) + #Or if the user compiled threadframe (from http://www.majid.info/mylos/stories/2004/06/10/threadframe.html) + + #NOT RLock!! (could deadlock if it was) + self.lock = threading.Lock() + self._acquire_lock = self.lock.acquire + self._release_lock = self.lock.release + + #collection with the refs + d = {} + self.pydev_existing_frames = d + try: + self._iter_frames = d.iterkeys + except AttributeError: + self._iter_frames = d.keys + + + def _OnDbFrameCollected(self, ref): + ''' + Callback to be called when a given reference is garbage-collected. + ''' + self._acquire_lock() + try: + del self.pydev_existing_frames[ref] + finally: + self._release_lock() + + + def _AddDbFrame(self, db_frame): + self._acquire_lock() + try: + #create the db frame with a callback to remove it from the dict when it's garbage-collected + #(could be a set, but that's not available on all versions we want to target). + r = weakref.ref(db_frame, self._OnDbFrameCollected) + self.pydev_existing_frames[r] = r + finally: + self._release_lock() + + + def CreateDbFrame(self, mainDebugger, filename, additionalInfo, t, frame): + #the frame must be cached as a weak-ref (we return the actual db frame -- which will be kept + #alive until its trace_dispatch method is not referenced anymore). + #that's a large workaround because: + #1. we can't have weak-references to python frame object + #2. only from 2.5 onwards we have _current_frames support from the interpreter + db_frame = PyDBFrame(mainDebugger, filename, additionalInfo, t, frame) + db_frame.frame = frame + self._AddDbFrame(db_frame) + return db_frame + + + def IterFrames(self): + #We cannot use yield (because of the lock) + self._acquire_lock() + try: + ret = [] + + for weak_db_frame in self._iter_frames(): + try: + ret.append(weak_db_frame().frame) + except AttributeError: + pass #ok, garbage-collected already + return ret + finally: + self._release_lock() + + def __str__(self): + return 'State:%s Stop:%s Cmd: %s Kill:%s Frames:%s' % (self.pydev_state, self.pydev_step_stop, self.pydev_step_cmd, self.pydev_notify_kill, len(self.IterFrames())) + +#======================================================================================================================= +# NOW, WE HAVE TO DEFINE WHICH THREAD INFO TO USE +# (whether we have to keep references to the frames or not) +# from version 2.5 onwards, we can use sys._current_frames to get a dict with the threads +# and frames, but to support other versions, we can't rely on that. +#======================================================================================================================= +if hasattr(sys, '_current_frames'): + PyDBAdditionalThreadInfo = PyDBAdditionalThreadInfoWithCurrentFramesSupport +else: + try: + import threadframe + sys._current_frames = threadframe.dict + assert sys._current_frames is threadframe.dict #Just check if it was correctly set + PyDBAdditionalThreadInfo = PyDBAdditionalThreadInfoWithCurrentFramesSupport + except: + #If all fails, let's use the support without frames + PyDBAdditionalThreadInfo = PyDBAdditionalThreadInfoWithoutCurrentFramesSupport + diff --git a/python/helpers/pydev/pydevd_comm.py b/python/helpers/pydev/pydevd_comm.py new file mode 100644 index 000000000000..507ad526d8f3 --- /dev/null +++ b/python/helpers/pydev/pydevd_comm.py @@ -0,0 +1,694 @@ +''' pydevd - a debugging daemon +This is the daemon you launch for python remote debugging. + +Protocol: +each command has a format: + id\tsequence-num\ttext + id: protocol command number + sequence-num: each request has a sequence number. Sequence numbers + originating at the debugger are odd, sequence numbers originating + at the daemon are even. Every response uses the same sequence number + as the request. + payload: it is protocol dependent. When response is a complex structure, it + is returned as XML. Each attribute value is urlencoded, and then the whole + payload is urlencoded again to prevent stray characters corrupting protocol/xml encodings + + Commands: + + NUMBER NAME FROM* ARGUMENTS RESPONSE NOTE +100 series: program execution + 101 RUN JAVA - - + 102 LIST_THREADS JAVA RETURN with XML listing of all threads + 103 THREAD_CREATE PYDB - XML with thread information + 104 THREAD_KILL JAVA id (or * to exit) kills the thread + PYDB id nofies JAVA that thread was killed + 105 THREAD_SUSPEND JAVA XML of the stack, suspends the thread + reason for suspension + PYDB id notifies JAVA that thread was suspended + + 106 CMD_THREAD_RUN JAVA id resume the thread + PYDB id \t reason notifies JAVA that thread was resumed + + 107 STEP_INTO JAVA thread_id + 108 STEP_OVER JAVA thread_id + 109 STEP_RETURN JAVA thread_id + + 110 GET_VARIABLE JAVA thread_id \t frame_id \t GET_VARIABLE with XML of var content + FRAME|GLOBAL \t attributes* + + 111 SET_BREAK JAVA file/line of the breakpoint + 112 REMOVE_BREAK JAVA file/line of the return + 113 CMD_EVALUATE_EXPRESSION JAVA expression result of evaluating the expression + 114 CMD_GET_FRAME JAVA request for frame contents + 115 CMD_EXEC_EXPRESSION JAVA + 116 CMD_WRITE_TO_CONSOLE PYDB + +500 series diagnostics/ok + 901 VERSION either Version string (1.0) Currently just used at startup + 902 RETURN either Depends on caller - + +900 series: errors + 501 ERROR either - This is reserved for unexpected errors. + + * JAVA - remote debugger, the java end + * PYDB - pydevd, the python end +''' +from pydevd_constants import * #@UnusedWildImport + +import time +import threading +try: + import Queue as PydevQueue +except ImportError: + import queue as PydevQueue +from socket import socket +from socket import AF_INET, SOCK_STREAM +try: + from urllib import quote +except: + from urllib.parse import quote +import pydevd_vars +import pydevd_tracing +import pydevd_vm_type +import pydevd_file_utils +import traceback + +from pydevd_tracing import GetExceptionTracebackStr + + +CMD_RUN = 101 +CMD_LIST_THREADS = 102 +CMD_THREAD_CREATE = 103 +CMD_THREAD_KILL = 104 +CMD_THREAD_SUSPEND = 105 +CMD_THREAD_RUN = 106 +CMD_STEP_INTO = 107 +CMD_STEP_OVER = 108 +CMD_STEP_RETURN = 109 +CMD_GET_VARIABLE = 110 +CMD_SET_BREAK = 111 +CMD_REMOVE_BREAK = 112 +CMD_EVALUATE_EXPRESSION = 113 +CMD_GET_FRAME = 114 +CMD_EXEC_EXPRESSION = 115 +CMD_WRITE_TO_CONSOLE = 116 +CMD_CHANGE_VARIABLE = 117 +CMD_RUN_TO_LINE = 118 +CMD_RELOAD_CODE = 119 +CMD_VERSION = 501 +CMD_RETURN = 502 +CMD_ERROR = 901 + +ID_TO_MEANING = { + '101':'CMD_RUN', + '102':'CMD_LIST_THREADS', + '103':'CMD_THREAD_CREATE', + '104':'CMD_THREAD_KILL', + '105':'CMD_THREAD_SUSPEND', + '106':'CMD_THREAD_RUN', + '107':'CMD_STEP_INTO', + '108':'CMD_STEP_OVER', + '109':'CMD_STEP_RETURN', + '110':'CMD_GET_VARIABLE', + '111':'CMD_SET_BREAK', + '112':'CMD_REMOVE_BREAK', + '113':'CMD_EVALUATE_EXPRESSION', + '114':'CMD_GET_FRAME', + '115':'CMD_EXEC_EXPRESSION', + '116':'CMD_WRITE_TO_CONSOLE', + '117':'CMD_CHANGE_VARIABLE', + '118':'CMD_RUN_TO_LINE', + '119':'CMD_RELOAD_CODE', + '501':'CMD_VERSION', + '502':'CMD_RETURN', + '901':'CMD_ERROR', +} + +MAX_IO_MSG_SIZE = 1000 #if the io is too big, we'll not send all (could make the debugger too non-responsive) + #this number can be changed if there's need to do so + +VERSION_STRING = "1.1" + + +#--------------------------------------------------------------------------------------------------- UTILITIES + +#======================================================================================================================= +# PydevdLog +#======================================================================================================================= +def PydevdLog(level, *args): + """ levels are: + 0 most serious warnings/errors + 1 warnings/significant events + 2 informational trace + """ + if level <= DEBUG_TRACE_LEVEL: + #yes, we can have errors printing if the console of the program has been finished (and we're still trying to print something) + try: + sys.stderr.write('%s\n' % (args,)) + except: + pass + +#======================================================================================================================= +# GlobalDebuggerHolder +#======================================================================================================================= +class GlobalDebuggerHolder: + ''' + Holder for the global debugger. + ''' + globalDbg = None + +#======================================================================================================================= +# GetGlobalDebugger +#======================================================================================================================= +def GetGlobalDebugger(): + return GlobalDebuggerHolder.globalDbg + +#======================================================================================================================= +# SetGlobalDebugger +#======================================================================================================================= +def SetGlobalDebugger(dbg): + GlobalDebuggerHolder.globalDbg = dbg + + +#------------------------------------------------------------------- ACTUAL COMM + +#======================================================================================================================= +# PyDBDaemonThread +#======================================================================================================================= +class PyDBDaemonThread(threading.Thread): + + def __init__(self): + threading.Thread.__init__(self) + self.setDaemon(True) + self.killReceived = False + + def run(self): + if sys.platform.startswith("java"): + import org.python.core as PyCore #@UnresolvedImport + ss = PyCore.PySystemState() + # Note: Py.setSystemState() affects only the current thread. + PyCore.Py.setSystemState(ss) + + self.OnRun() + + def OnRun(self): + raise NotImplementedError('Should be reimplemented by: %s' % self.__class__) + + def doKill(self): + #that was not working very well because jython gave some socket errors + self.killReceived = True + +#======================================================================================================================= +# ReaderThread +#======================================================================================================================= +class ReaderThread(PyDBDaemonThread): + """ reader thread reads and dispatches commands in an infinite loop """ + + def __init__(self, sock): + PyDBDaemonThread.__init__(self) + self.sock = sock + self.setName("pydevd.Reader") + + + def doKill(self): + #We must close the socket so that it doesn't stay halted there. + self.killReceived = True + try: + self.sock.close() + except: + #just ignore that + pass + + + def OnRun(self): + pydevd_tracing.SetTrace(None) # no debugging on this thread + buffer = "" + try: + + while not self.killReceived: + try: + r = self.sock.recv(1024) + except: + GlobalDebuggerHolder.globalDbg.finishDebuggingSession = True + break #Finished communication. + if IS_PY3K: + r = r.decode('utf-8') + + buffer += r + if DebugInfoHolder.DEBUG_RECORD_SOCKET_READS: + sys.stdout.write('received >>%s<<\n' % (buffer,)) + + if len(buffer) == 0: + GlobalDebuggerHolder.globalDbg.finishDebuggingSession = True + break + while buffer.find('\n') != -1: + command, buffer = buffer.split('\n', 1) + PydevdLog(1, "received command ", command) + args = command.split('\t', 2) + GlobalDebuggerHolder.globalDbg.processNetCommand(int(args[0]), int(args[1]), args[2]) + except: + traceback.print_exc() + GlobalDebuggerHolder.globalDbg.finishDebuggingSession = True + + +#----------------------------------------------------------------------------------- SOCKET UTILITIES - WRITER +#======================================================================================================================= +# WriterThread +#======================================================================================================================= +class WriterThread(PyDBDaemonThread): + """ writer thread writes out the commands in an infinite loop """ + def __init__(self, sock): + PyDBDaemonThread.__init__(self) + self.sock = sock + self.setName("pydevd.Writer") + self.cmdQueue = PydevQueue.Queue() + if pydevd_vm_type.GetVmType() == 'python': + self.timeout = 0 + else: + self.timeout = 0.1 + + def addCommand(self, cmd): + """ cmd is NetCommand """ + self.cmdQueue.put(cmd) + + def OnRun(self): + """ just loop and write responses """ + + pydevd_tracing.SetTrace(None) # no debugging on this thread + try: + while not self.killReceived: + try: + cmd = self.cmdQueue.get(1) + except: + #PydevdLog(0, 'Finishing debug communication...(1)') + #when liberating the thread here, we could have errors because we were shutting down + #but the thread was still not liberated + return + out = cmd.getOutgoing() + if DEBUG_TRACE_LEVEL >= 1: + out_message = 'sending cmd: ' + out_message += ID_TO_MEANING.get(out[:3], 'UNKNOWN') + out_message += ' ' + out_message += out + try: + sys.stderr.write('%s\n' % (out_message,)) + except: + pass + + if IS_PY3K: + out = bytearray(out, 'utf-8') + self.sock.send(out) #TODO: this does not guarantee that all message are sent (and jython does not have a send all) + if time is None: + break #interpreter shutdown + time.sleep(self.timeout) + except Exception: + GlobalDebuggerHolder.globalDbg.finishDebuggingSession = True + if DEBUG_TRACE_LEVEL >= 0: + traceback.print_exc() + + + + +#--------------------------------------------------- CREATING THE SOCKET THREADS + +#======================================================================================================================= +# StartServer +#======================================================================================================================= +def StartServer(port): + """ binds to a port, waits for the debugger to connect """ + s = socket(AF_INET, SOCK_STREAM) + s.bind(('', port)) + s.listen(1) + newSock, _addr = s.accept() + return newSock + +#======================================================================================================================= +# StartClient +#======================================================================================================================= +def StartClient(host, port): + """ connects to a host/port """ + PydevdLog(1, "Connecting to ", host, ":", str(port)) + try: + s = socket(AF_INET, SOCK_STREAM); + + s.connect((host, port)) + PydevdLog(1, "Connected.") + return s + except: + sys.stderr.write("server timed out after 10 seconds, could not connect to %s: %s\n" % (host, port)) + sys.stderr.write("Exiting. Bye!\n") + sys.exit(1) + + + +#------------------------------------------------------------------------------------ MANY COMMUNICATION STUFF + +#======================================================================================================================= +# NetCommand +#======================================================================================================================= +class NetCommand: + """ Commands received/sent over the network. + + Command can represent command received from the debugger, + or one to be sent by daemon. + """ + next_seq = 0 # sequence numbers + + def __init__(self, id, seq, text): + """ smart handling of paramaters + if sequence is 0, new sequence will be generated + if text has carriage returns they'll be replaced""" + self.id = id + if (seq == 0): seq = self.getNextSeq() + self.seq = seq + self.text = text + self.outgoing = self.makeMessage(id, seq, text) + + def getNextSeq(self): + """ returns next sequence number """ + NetCommand.next_seq += 2 + return NetCommand.next_seq + + def getOutgoing(self): + """ returns the outgoing message""" + return self.outgoing + + def makeMessage(self, cmd, seq, payload): + encoded = quote(str(payload), '/<>_=" \t') + return str(cmd) + '\t' + str(seq) + '\t' + encoded + "\n" + +#======================================================================================================================= +# NetCommandFactory +#======================================================================================================================= +class NetCommandFactory: + + def __init_(self): + self.next_seq = 0 + + def threadToXML(self, thread): + """ thread information as XML """ + name = pydevd_vars.makeValidXmlValue(thread.getName()) + cmdText = '' % (quote(name), GetThreadId(thread)) + return cmdText + + def makeErrorMessage(self, seq, text): + cmd = NetCommand(CMD_ERROR, seq, text) + if DEBUG_TRACE_LEVEL > 2: + sys.stderr.write("Error: %s" % (text,)) + return cmd; + + def makeThreadCreatedMessage(self, thread): + cmdText = "" + self.threadToXML(thread) + "" + return NetCommand(CMD_THREAD_CREATE, 0, cmdText) + + def makeListThreadsMessage(self, seq): + """ returns thread listing as XML """ + try: + t = threading.enumerate() + cmdText = "" + for i in t: + if t.isAlive(): + cmdText += self.threadToXML(i) + cmdText += "" + return NetCommand(CMD_RETURN, seq, cmdText) + except: + return self.makeErrorMessage(seq, GetExceptionTracebackStr()) + + def makeIoMessage(self, v, ctx, dbg=None): + ''' + @param v: the message to pass to the debug server + @param ctx: 1 for stdio 2 for stderr + @param dbg: If not none, add to the writer + ''' + + try: + if len(v) > MAX_IO_MSG_SIZE: + v = v[0:MAX_IO_MSG_SIZE] + v += '...' + + v = pydevd_vars.makeValidXmlValue(quote(v, '/>_= \t')) + net = NetCommand(str(CMD_WRITE_TO_CONSOLE), 0, '' % (v, ctx)) + if dbg: + dbg.writer.addCommand(net) + except: + return self.makeErrorMessage(0, GetExceptionTracebackStr()) + + def makeVersionMessage(self, seq): + try: + return NetCommand(CMD_VERSION, seq, VERSION_STRING) + except: + return self.makeErrorMessage(seq, GetExceptionTracebackStr()) + + def makeThreadKilledMessage(self, id): + try: + return NetCommand(CMD_THREAD_KILL, 0, str(id)) + except: + return self.makeErrorMessage(0, GetExceptionTracebackStr()) + + def makeThreadSuspendMessage(self, thread_id, frame, stop_reason): + + """ + + + + + """ + try: + cmdTextList = [""] + cmdTextList.append('' % (thread_id, stop_reason)) + + curFrame = frame + while curFrame: + #print cmdText + myId = str(id(curFrame)) + #print "id is ", myId + + if curFrame.f_code is None: + break #Iron Python sometimes does not have it! + + myName = curFrame.f_code.co_name #method name (if in method) or ? if global + if myName is None: + break #Iron Python sometimes does not have it! + + #print "name is ", myName + + myFile = pydevd_file_utils.NormFileToClient(curFrame.f_code.co_filename) + #print "file is ", myFile + #myFile = inspect.getsourcefile(curFrame) or inspect.getfile(frame) + + myLine = str(curFrame.f_lineno) + #print "line is ", myLine + + #the variables are all gotten 'on-demand' + #variables = pydevd_vars.frameVarsToXML(curFrame) + + variables = '' + cmdTextList.append('"' % (quote(myFile, '/>_= \t'), myLine)) + cmdTextList.append(variables) + cmdTextList.append("") + curFrame = curFrame.f_back + + cmdTextList.append("") + cmdText = ''.join(cmdTextList) + return NetCommand(CMD_THREAD_SUSPEND, 0, cmdText) + except: + return self.makeErrorMessage(0, GetExceptionTracebackStr()) + + def makeThreadRunMessage(self, id, reason): + try: + return NetCommand(CMD_THREAD_RUN, 0, str(id) + "\t" + str(reason)) + except: + return self.makeErrorMessage(0, GetExceptionTracebackStr()) + + def makeGetVariableMessage(self, seq, payload): + try: + return NetCommand(CMD_GET_VARIABLE, seq, payload) + except Exception: + return self.makeErrorMessage(seq, GetExceptionTracebackStr()) + + def makeGetFrameMessage(self, seq, payload): + try: + return NetCommand(CMD_GET_FRAME, seq, payload) + except Exception: + return self.makeErrorMessage(seq, GetExceptionTracebackStr()) + + + def makeEvaluateExpressionMessage(self, seq, payload): + try: + return NetCommand(CMD_EVALUATE_EXPRESSION, seq, payload) + except Exception: + return self.makeErrorMessage(seq, GetExceptionTracebackStr()) + +INTERNAL_TERMINATE_THREAD = 1 +INTERNAL_SUSPEND_THREAD = 2 + + +#======================================================================================================================= +# InternalThreadCommand +#======================================================================================================================= +class InternalThreadCommand: + """ internal commands are generated/executed by the debugger. + + The reason for their existence is that some commands have to be executed + on specific threads. These are the InternalThreadCommands that get + get posted to PyDB.cmdQueue. + """ + + def canBeExecutedBy(self, thread_id): + '''By default, it must be in the same thread to be executed + ''' + return self.thread_id == thread_id + + def doIt(self, dbg): + raise NotImplementedError("you have to override doIt") + +#======================================================================================================================= +# InternalTerminateThread +#======================================================================================================================= +class InternalTerminateThread(InternalThreadCommand): + def __init__(self, thread_id): + self.thread_id = thread_id + + def doIt(self, dbg): + PydevdLog(1, "killing ", str(self.thread_id)) + cmd = dbg.cmdFactory.makeThreadKilledMessage(self.thread_id) + dbg.writer.addCommand(cmd) + + +#======================================================================================================================= +# InternalGetVariable +#======================================================================================================================= +class InternalGetVariable(InternalThreadCommand): + """ gets the value of a variable """ + def __init__(self, seq, thread_id, frame_id, scope, attrs): + self.sequence = seq + self.thread_id = thread_id + self.frame_id = frame_id + self.scope = scope + self.attributes = attrs + + def doIt(self, dbg): + """ Converts request into python variable """ + try: + xml = "" + valDict = pydevd_vars.resolveCompoundVariable(self.thread_id, self.frame_id, self.scope, self.attributes) + keys = valDict.keys() + if hasattr(keys, 'sort'): + keys.sort() #Python 3.0 does not have it + else: + keys = sorted(keys, key=str) #Jython 2.1 does not have it (and all must be compared as strings). + for k in keys: + xml += pydevd_vars.varToXML(valDict[k], str(k)) + + xml += "" + cmd = dbg.cmdFactory.makeGetVariableMessage(self.sequence, xml) + dbg.writer.addCommand(cmd) + except Exception: + cmd = dbg.cmdFactory.makeErrorMessage(self.sequence, "Error resolving variables " + GetExceptionTracebackStr()) + dbg.writer.addCommand(cmd) + + +#======================================================================================================================= +# InternalChangeVariable +#======================================================================================================================= +class InternalChangeVariable(InternalThreadCommand): + """ changes the value of a variable """ + def __init__(self, seq, thread_id, frame_id, scope, attr, expression): + self.sequence = seq + self.thread_id = thread_id + self.frame_id = frame_id + self.scope = scope + self.attr = attr + self.expression = expression + + def doIt(self, dbg): + """ Converts request into python variable """ + try: + pydevd_vars.changeAttrExpression(self.thread_id, self.frame_id, self.attr, self.expression) + except Exception: + cmd = dbg.cmdFactory.makeErrorMessage(self.sequence, "Error changing variable attr:%s expression:%s traceback:%s" % (self.attr, self.expression, GetExceptionTracebackStr())) + dbg.writer.addCommand(cmd) + + +#======================================================================================================================= +# InternalGetFrame +#======================================================================================================================= +class InternalGetFrame(InternalThreadCommand): + """ gets the value of a variable """ + def __init__(self, seq, thread_id, frame_id): + self.sequence = seq + self.thread_id = thread_id + self.frame_id = frame_id + + def doIt(self, dbg): + """ Converts request into python variable """ + try: + try: + xml = "" + frame = pydevd_vars.findFrame(self.thread_id, self.frame_id) + xml += pydevd_vars.frameVarsToXML(frame) + del frame + xml += "" + cmd = dbg.cmdFactory.makeGetFrameMessage(self.sequence, xml) + dbg.writer.addCommand(cmd) + except pydevd_vars.FrameNotFoundError: + #pydevd_vars.dumpFrames(self.thread_id) + #don't print this error: frame not found: means that the client is not synchronized (but that's ok) + cmd = dbg.cmdFactory.makeErrorMessage(self.sequence, "Frame not found: %s from thread: %s" % (self.frame_id, self.thread_id)) + dbg.writer.addCommand(cmd) + except: + cmd = dbg.cmdFactory.makeErrorMessage(self.sequence, "Error resolving frame: %s from thread: %s" % (self.frame_id, self.thread_id)) + dbg.writer.addCommand(cmd) + + + + +#======================================================================================================================= +# InternalEvaluateExpression +#======================================================================================================================= +class InternalEvaluateExpression(InternalThreadCommand): + """ gets the value of a variable """ + + def __init__(self, seq, thread_id, frame_id, expression, doExec): + self.sequence = seq + self.thread_id = thread_id + self.frame_id = frame_id + self.expression = expression + self.doExec = doExec + + def doIt(self, dbg): + """ Converts request into python variable """ + try: + result = pydevd_vars.evaluateExpression(self.thread_id, self.frame_id, self.expression, self.doExec) + xml = "" + xml += pydevd_vars.varToXML(result, "") + xml += "" + cmd = dbg.cmdFactory.makeEvaluateExpressionMessage(self.sequence, xml) + dbg.writer.addCommand(cmd) + except: + exc = GetExceptionTracebackStr() + sys.stderr.write('%s\n' % (exc,)) + cmd = dbg.cmdFactory.makeErrorMessage(self.sequence, "Error evaluating expression " + exc) + dbg.writer.addCommand(cmd) + + +#======================================================================================================================= +# PydevdFindThreadById +#======================================================================================================================= +def PydevdFindThreadById(thread_id): + try: + # there was a deadlock here when I did not remove the tracing function when thread was dead + threads = threading.enumerate() + for i in threads: + if thread_id == GetThreadId(i): + return i + + sys.stderr.write("Could not find thread %s\n" % thread_id) + sys.stderr.write("Available: %s\n" % [GetThreadId(t) for t in threads]) + except: + traceback.print_exc() + + return None + diff --git a/python/helpers/pydev/pydevd_constants.py b/python/helpers/pydev/pydevd_constants.py new file mode 100644 index 000000000000..fa160381ef73 --- /dev/null +++ b/python/helpers/pydev/pydevd_constants.py @@ -0,0 +1,156 @@ +''' +This module holds the constants used for specifying the states of the debugger. +''' + +STATE_RUN = 1 +STATE_SUSPEND = 2 + +try: + __setFalse = False +except: + import __builtin__ + setattr(__builtin__, 'True', 1) + setattr(__builtin__, 'False', 0) + +DEBUG_TRACE_LEVEL = -1 +DEBUG_TRACE_BREAKPOINTS = -1 + +class DebugInfoHolder: + #we have to put it here because it can be set through the command line (so, the + #already imported references would not have it). + DEBUG_RECORD_SOCKET_READS = False + +#Optimize with psyco? This gave a 50% speedup in the debugger in tests +USE_PSYCO_OPTIMIZATION = True + +#Hold a reference to the original _getframe (because psyco will change that as soon as it's imported) +import sys #Note: the sys import must be here anyways (others depend on it) +GetFrame = sys._getframe + +#Used to determine the maximum size of each variable passed to eclipse -- having a big value here may make +#the communication slower -- as the variables are being gathered lazily in the latest version of eclipse, +#this value was raised from 200 to 1000. +MAXIMUM_VARIABLE_REPRESENTATION_SIZE = 1000 + +import threading +import os + +_nextThreadIdLock = threading.Lock() + +#======================================================================================================================= +# Python 3? +#======================================================================================================================= +IS_PY3K = False +try: + if sys.version_info[0] >= 3: + IS_PY3K = True +except AttributeError: + pass #Not all versions have sys.version_info + + +#======================================================================================================================= +# Jython? +#======================================================================================================================= +try: + import org.python.core.PyDictionary #@UnresolvedImport @UnusedImport -- just to check if it could be valid + def DictContains(d, key): + return d.has_key(key) +except: + try: + #Py3k does not have has_key anymore, and older versions don't have __contains__ + DictContains = dict.__contains__ + except: + DictContains = dict.has_key + +#======================================================================================================================= +# NextId +#======================================================================================================================= +class NextId: + + def __init__(self): + self._id = 0 + + def __call__(self): + #No need to synchronize here + self._id += 1 + return self._id + +_nextThreadId = NextId() + +#======================================================================================================================= +# GetThreadId +#======================================================================================================================= +def GetThreadId(thread): + try: + return thread.__pydevd_id__ + except AttributeError: + _nextThreadIdLock.acquire() + try: + #We do a new check with the lock in place just to be sure that nothing changed + if not hasattr(thread, '__pydevd_id__'): + try: + pid = os.getpid() + except AttributeError: + try: + #Jython does not have it! + import java.lang.management.ManagementFactory #@UnresolvedImport -- just for jython + pid = java.lang.management.ManagementFactory.getRuntimeMXBean().getName() + pid = pid.replace('@', '_') + except: + #ok, no pid available (will be unable to debug multiple processes) + pid = '000001' + + thread.__pydevd_id__ = 'pid%s_seq%s' % (pid, _nextThreadId()) + finally: + _nextThreadIdLock.release() + + return thread.__pydevd_id__ + +#=============================================================================== +# Null +#=============================================================================== +class Null: + """ + Gotten from: http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/68205 + """ + + def __init__(self, *args, **kwargs): + return None + + def __call__(self, *args, **kwargs): + return self + + def __getattr__(self, mname): + return self + + def __setattr__(self, name, value): + return self + + def __delattr__(self, name): + return self + + def __repr__(self): + return "" + + def __str__(self): + return "Null" + + def __len__(self): + return 0 + + def __getitem__(self): + return self + + def __setitem__(self, *args, **kwargs): + pass + + def write(self, *args, **kwargs): + pass + + def __nonzero__(self): + return 0 + +if __name__ == '__main__': + if Null(): + sys.stdout.write('here\n') + diff --git a/python/helpers/pydev/pydevd_file_utils.py b/python/helpers/pydev/pydevd_file_utils.py new file mode 100644 index 000000000000..2e71521ea770 --- /dev/null +++ b/python/helpers/pydev/pydevd_file_utils.py @@ -0,0 +1,215 @@ +''' + This module provides utilities to get the absolute filenames so that we can be sure that: + - The case of a file will match the actual file in the filesystem (otherwise breakpoints won't be hit). + - Providing means for the user to make path conversions when doing a remote debugging session in + one machine and debugging in another. + + To do that, the PATHS_FROM_CLIENT_TO_SERVER constant must be filled with the appropriate paths. + + E.g.: + If the server has the structure + /user/projects/my_project/src/package/module1.py + + and the client has: + c:\my_project\src\package\module1.py + + the PATHS_FROM_CLIENT_TO_SERVER would have to be: + PATHS_FROM_CLIENT_TO_SERVER = [(r'c:\my_project\src', r'/user/projects/my_project/src')] + + @note: DEBUG_CLIENT_SERVER_TRANSLATION can be set to True to debug the result of those translations + + @note: the case of the paths is important! Note that this can be tricky to get right when one machine + uses a case-independent filesystem and the other uses a case-dependent filesystem (if the system being + debugged is case-independent, 'normcase()' should be used on the paths defined in PATHS_FROM_CLIENT_TO_SERVER). + + @note: all the paths with breakpoints must be translated (otherwise they won't be found in the server) + + @note: to enable remote debugging in the target machine (pydev extensions in the eclipse installation) + import pydevd;pydevd.settrace(host, stdoutToServer, stderrToServer, port, suspend) + + see parameter docs on pydevd.py + + @note: for doing a remote debugging session, all the pydevd_ files must be on the server accessible + through the PYTHONPATH (and the PATHS_FROM_CLIENT_TO_SERVER only needs to be set on the target + machine for the paths that'll actually have breakpoints). +''' + + + + +from pydevd_constants import * #@UnusedWildImport +import os.path +import sys +import traceback + +normcase = os.path.normcase +basename = os.path.basename +exists = os.path.exists +join = os.path.join + +try: + rPath = os.path.realpath #@UndefinedVariable +except: + # jython does not support os.path.realpath + # realpath is a no-op on systems without islink support + rPath = os.path.abspath + +#defined as a list of tuples where the 1st element of the tuple is the path in the client machine +#and the 2nd element is the path in the server machine. +#see module docstring for more details. +PATHS_FROM_CLIENT_TO_SERVER = [] + + +#example: +#PATHS_FROM_CLIENT_TO_SERVER = [ +#(normcase(r'd:\temp\temp_workspace_2\test_python\src\yyy\yyy'), +# normcase(r'd:\temp\temp_workspace_2\test_python\src\hhh\xxx'))] + +DEBUG_CLIENT_SERVER_TRANSLATION = False + +#caches filled as requested during the debug session +NORM_FILENAME_CONTAINER = {} +NORM_FILENAME_AND_BASE_CONTAINER = {} +NORM_FILENAME_TO_SERVER_CONTAINER = {} +NORM_FILENAME_TO_CLIENT_CONTAINER = {} + + +def _NormFile(filename): + try: + return NORM_FILENAME_CONTAINER[filename] + except KeyError: + r = normcase(rPath(filename)) + #cache it for fast access later + NORM_FILENAME_CONTAINER[filename] = r + return r + +#Now, let's do a quick test to see if we're working with a version of python that has no problems +#related to the names generated... +try: + try: + code = rPath.func_code + except AttributeError: + code = rPath.__code__ + if not exists(_NormFile(code.co_filename)): + sys.stderr.write('-------------------------------------------------------------------------------\n') + sys.stderr.write('pydev debugger: CRITICAL WARNING: This version of python seems to be incorrectly compiled (internal generated filenames are not absolute)\n') + sys.stderr.write('pydev debugger: The debugger may still function, but it will work slower and may miss breakpoints.\n') + sys.stderr.write('pydev debugger: Related bug: http://bugs.python.org/issue1666807\n') + sys.stderr.write('-------------------------------------------------------------------------------\n') + + initial_norm_file = _NormFile + def _NormFile(filename): #Let's redefine _NormFile to work with paths that may be incorrect + ret = initial_norm_file(filename) + if not exists(ret): + #We must actually go on and check if we can find it as if it was a relative path for some of the paths in the pythonpath + for path in sys.path: + ret = initial_norm_file(join(path, filename)) + if exists(ret): + break + else: + sys.stderr.write('pydev debugger: Unable to find real location for: %s\n' % (filename,)) + ret = filename + + return ret +except: + #Don't fail if there's something not correct here -- but at least print it to the user so that we can correct that + traceback.print_exc() + + +if PATHS_FROM_CLIENT_TO_SERVER: + #Work on the client and server slashes. + client_sep = None + server_sep = None + for client_prefix, server_prefix in PATHS_FROM_CLIENT_TO_SERVER: + if client_sep is not None and server_sep is not None: + break + + if client_sep is None: + for c in client_prefix: + if c in ('/', '\\'): + client_sep = c + break + + if server_sep is None: + for c in server_prefix: + if c in ('/', '\\'): + server_sep = c + break + + #If they're the same or one of them cannot be determined, just make it all None. + if client_sep == server_sep or client_sep is None or server_sep is None: + client_sep = server_sep = None + + + #only setup translation functions if absolutely needed! + def NormFileToServer(filename): + try: + return NORM_FILENAME_TO_SERVER_CONTAINER[filename] + except KeyError: + #used to translate a path from the client to the debug server + translated = normcase(filename) + for client_prefix, server_prefix in PATHS_FROM_CLIENT_TO_SERVER: + if translated.startswith(client_prefix): + if DEBUG_CLIENT_SERVER_TRANSLATION: + sys.stderr.write('pydev debugger: replacing to server: %s\n' % (translated,)) + translated = translated.replace(client_prefix, server_prefix) + if DEBUG_CLIENT_SERVER_TRANSLATION: + sys.stderr.write('pydev debugger: sent to server: %s\n' % (translated,)) + break + else: + if DEBUG_CLIENT_SERVER_TRANSLATION: + sys.stderr.write('pydev debugger: to server: unable to find matching prefix for: %s in %s\n' % \ + (translated, [x[0] for x in PATHS_FROM_CLIENT_TO_SERVER])) + + #Note that when going to the server, we do the replace first and only later do the norm file. + if client_sep is not None: + translated = translated.replace(server_sep, client_sep) + ret = _NormFile(translated) + + NORM_FILENAME_TO_SERVER_CONTAINER[filename] = translated + return ret + + + def NormFileToClient(filename): + try: + return NORM_FILENAME_TO_CLIENT_CONTAINER[filename] + except KeyError: + #used to translate a path from the debug server to the client + translated = normcase(filename) + for client_prefix, server_prefix in PATHS_FROM_CLIENT_TO_SERVER: + if translated.startswith(server_prefix): + if DEBUG_CLIENT_SERVER_TRANSLATION: + sys.stderr.write('pydev debugger: replacing to client: %s\n' % (translated,)) + translated = translated.replace(server_prefix, client_prefix) + if DEBUG_CLIENT_SERVER_TRANSLATION: + sys.stderr.write('pydev debugger: sent to client: %s\n' % (translated,)) + break + else: + if DEBUG_CLIENT_SERVER_TRANSLATION: + sys.stderr.write('pydev debugger: to client: unable to find matching prefix for: %s in %s\n' % \ + (translated, [x[1] for x in PATHS_FROM_CLIENT_TO_SERVER])) + + #When going to the client, first we do the norm file and only later the replace for slashes. + ret = _NormFile(translated) + if client_sep is not None: + ret = ret.replace(client_sep, server_sep) + + NORM_FILENAME_TO_CLIENT_CONTAINER[filename] = ret + return ret + +else: + #no translation step needed (just inline the calls) + NormFileToClient = _NormFile + NormFileToServer = _NormFile + + +def GetFilenameAndBase(frame): + #This one is just internal (so, does not need any kind of client-server translation) + f = frame.f_code.co_filename + try: + return NORM_FILENAME_AND_BASE_CONTAINER[f] + except KeyError: + filename = _NormFile(f) + base = basename(filename) + NORM_FILENAME_AND_BASE_CONTAINER[f] = filename, base + return filename, base diff --git a/python/helpers/pydev/pydevd_frame.py b/python/helpers/pydev/pydevd_frame.py new file mode 100644 index 000000000000..42caaf1d567f --- /dev/null +++ b/python/helpers/pydev/pydevd_frame.py @@ -0,0 +1,194 @@ +from pydevd_comm import * #@UnusedWildImport +from pydevd_constants import * #@UnusedWildImport +import traceback #@Reimport +import os.path +basename = os.path.basename + +#======================================================================================================================= +# PyDBFrame +#======================================================================================================================= +class PyDBFrame: + '''This makes the tracing for a given frame, so, the trace_dispatch + is used initially when we enter into a new context ('call') and then + is reused for the entire context. + ''' + + def __init__(self, *args): + #args = mainDebugger, filename, base, info, t, frame + #yeap, much faster than putting in self and the getting it from self later on + self._args = args[:-1] + + def setSuspend(self, *args, **kwargs): + self._args[0].setSuspend(*args, **kwargs) + + def doWaitSuspend(self, *args, **kwargs): + self._args[0].doWaitSuspend(*args, **kwargs) + + def trace_dispatch(self, frame, event, arg): + if event not in ('line', 'call', 'return', 'exception'): + return None + + mainDebugger, filename, info, thread = self._args + + breakpoint = mainDebugger.breakpoints.get(filename) + + + if info.pydev_state == STATE_RUN: + #we can skip if: + #- we have no stop marked + #- we should make a step return/step over and we're not in the current frame + can_skip = (info.pydev_step_cmd is None and info.pydev_step_stop is None)\ + or (info.pydev_step_cmd in (CMD_STEP_RETURN, CMD_STEP_OVER) and info.pydev_step_stop is not frame) + else: + can_skip = False + + # Let's check to see if we are in a function that has a breakpoint. If we don't have a breakpoint, + # we will return nothing for the next trace + #also, after we hit a breakpoint and go to some other debugging state, we have to force the set trace anyway, + #so, that's why the additional checks are there. + if not breakpoint: + if can_skip: + return None + + else: + #checks the breakpoint to see if there is a context match in some function + curr_func_name = frame.f_code.co_name + + #global context is set with an empty name + if curr_func_name in ('?', ''): + curr_func_name = '' + + for _b, condition, func_name in breakpoint.values(): #jython does not support itervalues() + #will match either global or some function + if func_name in ('None', curr_func_name): + break + + else: # if we had some break, it won't get here (so, that's a context that we want to skip) + if can_skip: + #print 'skipping', frame.f_lineno, info.pydev_state, info.pydev_step_stop, info.pydev_step_cmd + return None + + #We may have hit a breakpoint or we are already in step mode. Either way, let's check what we should do in this frame + #print 'NOT skipped', frame.f_lineno, frame.f_code.co_name + + + try: + line = frame.f_lineno + + #return is not taken into account for breakpoint hit because we'd have a double-hit in this case + #(one for the line and the other for the return). + if event != 'return' and info.pydev_state != STATE_SUSPEND and breakpoint is not None \ + and DictContains(breakpoint, line): + + #ok, hit breakpoint, now, we have to discover if it is a conditional breakpoint + # lets do the conditional stuff here + condition = breakpoint[line][1] + + if condition is not None: + try: + val = eval(condition, frame.f_globals, frame.f_locals) + if not val: + return self.trace_dispatch + + except: + sys.stderr.write('Error while evaluating expression\n') + traceback.print_exc() + return self.trace_dispatch + + self.setSuspend(thread, CMD_SET_BREAK) + + # if thread has a suspend flag, we suspend with a busy wait + if info.pydev_state == STATE_SUSPEND: + self.doWaitSuspend(thread, frame, event, arg) + return self.trace_dispatch + + except: + traceback.print_exc() + raise + + #step handling. We stop when we hit the right frame + try: + + if info.pydev_step_cmd == CMD_STEP_INTO: + + stop = event in ('line', 'return') + + elif info.pydev_step_cmd == CMD_STEP_OVER: + + stop = info.pydev_step_stop is frame and event in ('line', 'return') + + elif info.pydev_step_cmd == CMD_STEP_RETURN: + + stop = event == 'return' and info.pydev_step_stop is frame + + elif info.pydev_step_cmd == CMD_RUN_TO_LINE: + stop = False + if event == 'line': + #Yes, we can only act on line events (weird hum?) + #Note: This code is duplicated at pydevd.py + curr_func_name = frame.f_code.co_name + + #global context is set with an empty name + if curr_func_name in ('?', ''): + curr_func_name = '' + + if curr_func_name == info.pydev_func_name: + line = info.pydev_next_line + if frame.f_lineno == line: + stop = True + else: + if frame.f_trace is None: + frame.f_trace = self.trace_dispatch + frame.f_lineno = line + frame.f_trace = None + stop = True + + else: + stop = False + + if stop: + #event is always == line or return at this point + if event == 'line': + self.setSuspend(thread, info.pydev_step_cmd) + self.doWaitSuspend(thread, frame, event, arg) + else: #return event + back = frame.f_back + if back is not None: + + #When we get to the pydevd run function, the debugging has actually finished for the main thread + #(note that it can still go on for other threads, but for this one, we just make it finish) + #So, just setting it to None should be OK + if basename(back.f_code.co_filename) == 'pydevd.py' and back.f_code.co_name == 'run': + back = None + + + if back is not None: + #if we're in a return, we want it to appear to the user in the previous frame! + self.setSuspend(thread, info.pydev_step_cmd) + self.doWaitSuspend(thread, back, event, arg) + else: + #in jython we may not have a back frame + info.pydev_step_stop = None + info.pydev_step_cmd = None + info.pydev_state = STATE_RUN + + + except: + traceback.print_exc() + info.pydev_step_cmd = None + + #if we are quitting, let's stop the tracing + retVal = None + if not mainDebugger.quitting: + retVal = self.trace_dispatch + + return retVal + + if USE_PSYCO_OPTIMIZATION: + try: + import psyco + trace_dispatch = psyco.proxy(trace_dispatch) + except ImportError: + if hasattr(sys, 'exc_clear'): #jython does not have it + sys.exc_clear() #don't keep the traceback + pass #ok, psyco not available diff --git a/python/helpers/pydev/pydevd_io.py b/python/helpers/pydev/pydevd_io.py new file mode 100644 index 000000000000..9ac8a94482cb --- /dev/null +++ b/python/helpers/pydev/pydevd_io.py @@ -0,0 +1,31 @@ +class IORedirector: + '''This class works to redirect the write function to many streams + ''' + + def __init__(self, *args): + self._redirectTo = args + + def write(self, s): + for r in self._redirectTo: + try: + r.write(s) + except: + pass + +class IOBuf: + '''This class works as a replacement for stdio and stderr. + It is a buffer and when its contents are requested, it will erase what + + it has so far so that the next return will not return the same contents again. + ''' + def __init__(self): + self.buflist = [] + + def getvalue(self): + b = self.buflist + self.buflist = [] #clear it + return ''.join(b) + + def write(self, s): + self.buflist.append(s) + diff --git a/python/helpers/pydev/pydevd_psyco_stub.py b/python/helpers/pydev/pydevd_psyco_stub.py new file mode 100644 index 000000000000..f196d88a113c --- /dev/null +++ b/python/helpers/pydev/pydevd_psyco_stub.py @@ -0,0 +1,36 @@ +''' + Psyco stub: should implement all the external API from psyco. +''' + +def proxy(func, *args, **kwargs): + return func + +def bind(func, *args, **kwargs): + return func + +def unbind(func, *args, **kwargs): + return func + +def unproxy(func, *args, **kwargs): + return func + +def full(*args, **kwargs): + pass + +def log(*args, **kwargs): + pass + +def runonly(*args, **kwargs): + pass + +def background(*args, **kwargs): + pass + +def cannotcompile(*args, **kwargs): + pass + +def profile(*args, **kwargs): + pass + +def stop(*args, **kwargs): + pass diff --git a/python/helpers/pydev/pydevd_reload.py b/python/helpers/pydev/pydevd_reload.py new file mode 100644 index 000000000000..4bc9348a42a6 --- /dev/null +++ b/python/helpers/pydev/pydevd_reload.py @@ -0,0 +1,200 @@ +""" +Copied from the python xreload (available for change) + +Alternative to reload(). + +This works by executing the module in a scratch namespace, and then +patching classes, methods and functions in place. This avoids the +need to patch instances. New objects are copied into the target +namespace. + +Some of the many limitations include: + +- Global mutable objects other than classes are simply replaced, not patched + +- Code using metaclasses is not handled correctly + +- Code creating global singletons is not handled correctly + +- Functions and methods using decorators (other than classmethod and + staticmethod) is not handled correctly + +- Renamings are not handled correctly + +- Dependent modules are not reloaded + +- When a dependent module contains 'from foo import bar', and + reloading foo deletes foo.bar, the dependent module continues to use + the old foo.bar object rather than failing + +- Frozen modules and modules loaded from zip files aren't handled + correctly + +- Classes involving __slots__ are not handled correctly +""" + +import imp +import sys +import types + + +def xreload(mod): + """Reload a module in place, updating classes, methods and functions. + + Args: + mod: a module object + + Returns: + The (updated) input object itself. + """ + # Get the module name, e.g. 'foo.bar.whatever' + modname = mod.__name__ + # Get the module namespace (dict) early; this is part of the type check + modns = mod.__dict__ + # Parse it into package name and module name, e.g. 'foo.bar' and 'whatever' + i = modname.rfind(".") + if i >= 0: + pkgname, modname = modname[:i], modname[i+1:] + else: + pkgname = None + # Compute the search path + if pkgname: + # We're not reloading the package, only the module in it + pkg = sys.modules[pkgname] + path = pkg.__path__ # Search inside the package + else: + # Search the top-level module path + pkg = None + path = None # Make find_module() uses the default search path + # Find the module; may raise ImportError + (stream, filename, (suffix, mode, kind)) = imp.find_module(modname, path) + # Turn it into a code object + try: + # Is it Python source code or byte code read from a file? + if kind not in (imp.PY_COMPILED, imp.PY_SOURCE): + # Fall back to built-in reload() + return reload(mod) + if kind == imp.PY_SOURCE: + source = stream.read() + code = compile(source, filename, "exec") + else: + import marshal + code = marshal.load(stream) + finally: + if stream: + stream.close() + # Execute the code. We copy the module dict to a temporary; then + # clear the module dict; then execute the new code in the module + # dict; then swap things back and around. This trick (due to + # Glyph Lefkowitz) ensures that the (readonly) __globals__ + # attribute of methods and functions is set to the correct dict + # object. + tmpns = modns.copy() + modns.clear() + modns["__name__"] = tmpns["__name__"] + exec(code, modns) + # Now we get to the hard part + oldnames = set(tmpns) + newnames = set(modns) + # Update attributes in place + for name in oldnames & newnames: + modns[name] = _update(tmpns[name], modns[name]) + # Done! + return mod + + +def _update(oldobj, newobj): + """Update oldobj, if possible in place, with newobj. + + If oldobj is immutable, this simply returns newobj. + + Args: + oldobj: the object to be updated + newobj: the object used as the source for the update + + Returns: + either oldobj, updated in place, or newobj. + """ + if oldobj is newobj: + # Probably something imported + return newobj + if type(oldobj) is not type(newobj): + # Cop-out: if the type changed, give up + return newobj + if hasattr(newobj, "__reload_update__"): + # Provide a hook for updating + return newobj.__reload_update__(oldobj) + + if hasattr(types, 'ClassType'): + classtype = types.ClassType + else: + classtype = type + + if isinstance(newobj, classtype): + return _update_class(oldobj, newobj) + if isinstance(newobj, types.FunctionType): + return _update_function(oldobj, newobj) + if isinstance(newobj, types.MethodType): + return _update_method(oldobj, newobj) + if isinstance(newobj, classmethod): + return _update_classmethod(oldobj, newobj) + if isinstance(newobj, staticmethod): + return _update_staticmethod(oldobj, newobj) + # Not something we recognize, just give up + return newobj + + +# All of the following functions have the same signature as _update() + + +def _update_function(oldfunc, newfunc): + """Update a function object.""" + oldfunc.__doc__ = newfunc.__doc__ + oldfunc.__dict__.update(newfunc.__dict__) + oldfunc.__code__ = newfunc.__code__ + oldfunc.__defaults__ = newfunc.__defaults__ + return oldfunc + + +def _update_method(oldmeth, newmeth): + """Update a method object.""" + # XXX What if im_func is not a function? + _update(oldmeth.im_func, newmeth.im_func) + return oldmeth + + +def _update_class(oldclass, newclass): + """Update a class object.""" + olddict = oldclass.__dict__ + newdict = newclass.__dict__ + oldnames = set(olddict) + newnames = set(newdict) + for name in newnames - oldnames: + setattr(oldclass, name, newdict[name]) + for name in oldnames - newnames: + delattr(oldclass, name) + for name in oldnames & newnames - set(['__dict__', '__doc__']): + setattr(oldclass, name, _update(olddict[name], newdict[name])) + return oldclass + + +def _update_classmethod(oldcm, newcm): + """Update a classmethod update.""" + # While we can't modify the classmethod object itself (it has no + # mutable attributes), we *can* extract the underlying function + # (by calling __get__(), which returns a method object) and update + # it in-place. We don't have the class available to pass to + # __get__() but any object except None will do. + _update(oldcm.__get__(0), newcm.__get__(0)) + return newcm + + +def _update_staticmethod(oldsm, newsm): + """Update a staticmethod update.""" + # While we can't modify the staticmethod object itself (it has no + # mutable attributes), we *can* extract the underlying function + # (by calling __get__(), which returns it) and update it in-place. + # We don't have the class available to pass to __get__() but any + # object except None will do. + _update(oldsm.__get__(0), newsm.__get__(0)) + return newsm diff --git a/python/helpers/pydev/pydevd_resolver.py b/python/helpers/pydev/pydevd_resolver.py new file mode 100644 index 000000000000..930e023f91f6 --- /dev/null +++ b/python/helpers/pydev/pydevd_resolver.py @@ -0,0 +1,352 @@ +try: + import StringIO +except: + import io as StringIO +import traceback + +try: + __setFalse = False +except: + import __builtin__ + setattr(__builtin__, 'True', 1) + setattr(__builtin__, 'False', 0) + + +MAX_ITEMS_TO_HANDLE = 500 +TOO_LARGE_MSG = 'Too large to show contents. Max items to show: ' + str(MAX_ITEMS_TO_HANDLE) +TOO_LARGE_ATTR = 'Unable to handle:' + +#======================================================================================================================= +# UnableToResolveVariableException +#======================================================================================================================= +class UnableToResolveVariableException(Exception): + pass + + +#======================================================================================================================= +# InspectStub +#======================================================================================================================= +class InspectStub: + def isbuiltin(self, _args): + return False + def isroutine(self, object): + return False + +try: + import inspect +except: + inspect = InspectStub() + +try: + import java.lang #@UnresolvedImport +except: + pass + +#types does not include a MethodWrapperType +try: + MethodWrapperType = type([].__str__) +except: + MethodWrapperType = None + + +#======================================================================================================================= +# AbstractResolver +#======================================================================================================================= +class AbstractResolver: + ''' + This class exists only for documentation purposes to explain how to create a resolver. + + Some examples on how to resolve things: + - list: getDictionary could return a dict with index->item and use the index to resolve it later + - set: getDictionary could return a dict with id(object)->object and reiterate in that array to resolve it later + - arbitrary instance: getDictionary could return dict with attr_name->attr and use getattr to resolve it later + ''' + + def resolve(self, var, attribute): + ''' + In this method, we'll resolve some child item given the string representation of the item in the key + representing the previously asked dictionary. + + @param var: this is the actual variable to be resolved. + @param attribute: this is the string representation of a key previously returned in getDictionary. + ''' + raise NotImplementedError + + def getDictionary(self, var): + ''' + @param var: this is the variable that should have its children gotten. + + @return: a dictionary where each pair key, value should be shown to the user as children items + in the variables view for the given var. + ''' + raise NotImplementedError + + +#======================================================================================================================= +# DefaultResolver +#======================================================================================================================= +class DefaultResolver: + ''' + DefaultResolver is the class that'll actually resolve how to show some variable. + ''' + + def resolve(self, var, attribute): + return getattr(var, attribute) + + def getDictionary(self, var): + if MethodWrapperType: + return self._getPyDictionary(var) + else: + return self._getJyDictionary(var) + + def _getJyDictionary(self, obj): + ret = {} + found = java.util.HashMap() + + original = obj + if hasattr(obj, '__class__') and obj.__class__ == java.lang.Class: + + #get info about superclasses + classes = [] + classes.append(obj) + c = obj.getSuperclass() + while c != None: + classes.append(c) + c = c.getSuperclass() + + #get info about interfaces + interfs = [] + for obj in classes: + interfs.extend(obj.getInterfaces()) + classes.extend(interfs) + + #now is the time when we actually get info on the declared methods and fields + for obj in classes: + + declaredMethods = obj.getDeclaredMethods() + declaredFields = obj.getDeclaredFields() + for i in range(len(declaredMethods)): + name = declaredMethods[i].getName() + ret[name] = declaredMethods[i].toString() + found.put(name, 1) + + for i in range(len(declaredFields)): + name = declaredFields[i].getName() + found.put(name, 1) + #if declaredFields[i].isAccessible(): + declaredFields[i].setAccessible(True) + #ret[name] = declaredFields[i].get( declaredFields[i] ) + try: + ret[name] = declaredFields[i].get(original) + except: + ret[name] = declaredFields[i].toString() + + #this simple dir does not always get all the info, that's why we have the part before + #(e.g.: if we do a dir on String, some methods that are from other interfaces such as + #charAt don't appear) + try: + d = dir(original) + for name in d: + if found.get(name) is not 1: + ret[name] = getattr(original, name) + except: + #sometimes we're unable to do a dir + pass + + return ret + + def _getPyDictionary(self, var): + filterPrivate = False + filterSpecial = True + filterFunction = True + filterBuiltIn = True + + names = dir(var) + d = {} + + #Be aware that the order in which the filters are applied attempts to + #optimize the operation by removing as many items as possible in the + #first filters, leaving fewer items for later filters + + if filterBuiltIn or filterFunction: + for n in names: + if filterSpecial: + if n.startswith('__') and n.endswith('__'): + continue + + if filterPrivate: + if n.startswith('_') or n.endswith('__'): + continue + + try: + attr = getattr(var, n) + + #filter builtins? + if filterBuiltIn: + if inspect.isbuiltin(attr): + continue + + #filter functions? + if filterFunction: + if inspect.isroutine(attr) or isinstance(attr, MethodWrapperType): + continue + except: + #if some error occurs getting it, let's put it to the user. + strIO = StringIO.StringIO() + traceback.print_exc(file=strIO) + attr = strIO.getvalue() + + d[ n ] = attr + + return d + + +#======================================================================================================================= +# DictResolver +#======================================================================================================================= +class DictResolver: + + def resolve(self, dict, key): + if key == '__len__': + return None + + if '(' not in key: + #we have to treat that because the dict resolver is also used to directly resolve the global and local + #scopes (which already have the items directly) + return dict[key] + + #ok, we have to iterate over the items to find the one that matches the id, because that's the only way + #to actually find the reference from the string we have before. + expected_id = int(key.split('(')[-1][:-1]) + for key, val in dict.items(): + if id(key) == expected_id: + return val + + raise UnableToResolveVariableException() + + def getDictionary(self, dict): + ret = {} + + for key, val in dict.items(): + #we need to add the id because otherwise we cannot find the real object to get its contents later on. + key = '%s (%s)' % (key, id(key)) + ret[key] = val + + ret['__len__'] = len(dict) + return ret + + + +#======================================================================================================================= +# TupleResolver +#======================================================================================================================= +class TupleResolver: #to enumerate tuples and lists + + def resolve(self, var, attribute): + ''' + @param var: that's the original attribute + @param attribute: that's the key passed in the dict (as a string) + ''' + if attribute == '__len__' or attribute == TOO_LARGE_ATTR: + return None + return var[int(attribute)] + + def getDictionary(self, var): + #return dict( [ (i, x) for i, x in enumerate(var) ] ) + # modified 'cause jython does not have enumerate support + l = len(var) + d = {} + + if l < MAX_ITEMS_TO_HANDLE: + format = '%0' + str(int(len(str(l)))) + 'd' + + + for i, item in zip(range(l), var): + d[ format % i ] = item + else: + d[TOO_LARGE_ATTR] = TOO_LARGE_MSG + d['__len__'] = len(var) + return d + + + +#======================================================================================================================= +# SetResolver +#======================================================================================================================= +class SetResolver: + ''' + Resolves a set as dict id(object)->object + ''' + + def resolve(self, var, attribute): + if attribute == '__len__': + return None + + attribute = int(attribute) + for v in var: + if id(v) == attribute: + return v + + raise UnableToResolveVariableException('Unable to resolve %s in %s' % (attribute, var)) + + def getDictionary(self, var): + d = {} + for item in var: + d[ id(item) ] = item + d['__len__'] = len(var) + return d + + +#======================================================================================================================= +# InstanceResolver +#======================================================================================================================= +class InstanceResolver: + + def resolve(self, var, attribute): + field = var.__class__.getDeclaredField(attribute) + field.setAccessible(True) + return field.get(var) + + def getDictionary(self, obj): + ret = {} + + declaredFields = obj.__class__.getDeclaredFields() + for i in range(len(declaredFields)): + name = declaredFields[i].getName() + try: + declaredFields[i].setAccessible(True) + ret[name] = declaredFields[i].get(obj) + except: + traceback.print_exc() + + return ret + + +#======================================================================================================================= +# JyArrayResolver +#======================================================================================================================= +class JyArrayResolver: + ''' + This resolves a regular Object[] array from java + ''' + + def resolve(self, var, attribute): + if attribute == '__len__': + return None + return var[int(attribute)] + + def getDictionary(self, obj): + ret = {} + + for i in range(len(obj)): + ret[ i ] = obj[i] + + ret['__len__'] = len(obj) + return ret + +defaultResolver = DefaultResolver() +dictResolver = DictResolver() +tupleResolver = TupleResolver() +instanceResolver = InstanceResolver() +jyArrayResolver = JyArrayResolver() +setResolver = SetResolver() diff --git a/python/helpers/pydev/pydevd_tracing.py b/python/helpers/pydev/pydevd_tracing.py new file mode 100644 index 000000000000..202854a599f4 --- /dev/null +++ b/python/helpers/pydev/pydevd_tracing.py @@ -0,0 +1,78 @@ +from pydevd_constants import * #@UnusedWildImport + +try: + import cStringIO as StringIO #may not always be available @UnusedImport +except: + try: + import StringIO #@Reimport + except: + import io as StringIO + +import threading +import sys #@Reimport +import traceback + +class TracingFunctionHolder: + '''This class exists just to keep some variables (so that we don't keep them in the global namespace). + ''' + _original_tracing = None + _warn = True + _lock = threading.Lock() + _traceback_limit = 1 + _warnings_shown = {} + + +def GetExceptionTracebackStr(): + exc_info = sys.exc_info() + s = StringIO.StringIO() + traceback.print_exception(exc_info[0], exc_info[1], exc_info[2], file=s) + return s.getvalue() + +def _GetStackStr(frame): + + msg = '\nIf this is needed, please check: ' + \ + '\nhttp://pydev.blogspot.com/2007/06/why-cant-pydev-debugger-work-with.html' + \ + '\nto see how to restore the debug tracing back correctly.\n' + + if TracingFunctionHolder._traceback_limit: + s = StringIO.StringIO() + s.write('Call Location:\n') + traceback.print_stack(f=frame, limit=TracingFunctionHolder._traceback_limit, file=s) + msg = msg + s.getvalue() + + return msg + +def _InternalSetTrace(tracing_func): + if TracingFunctionHolder._warn: + frame = GetFrame() + if frame is not None and frame.f_back is not None: + if not frame.f_back.f_code.co_filename.lower().endswith('threading.py'): + + message = \ + '\nPYDEV DEBUGGER WARNING:' + \ + '\nsys.settrace() should not be used when the debugger is being used.' + \ + '\nThis may cause the debugger to stop working correctly.' + \ + '%s' % _GetStackStr(frame.f_back) + + if message not in TracingFunctionHolder._warnings_shown: + #only warn about each message once... + TracingFunctionHolder._warnings_shown[message] = 1 + sys.stderr.write('%s\n' % (message,)) + + TracingFunctionHolder._original_tracing(tracing_func) + +def SetTrace(tracing_func): + TracingFunctionHolder._lock.acquire() + try: + TracingFunctionHolder._warn = False + _InternalSetTrace(tracing_func) + TracingFunctionHolder._warn = True + finally: + TracingFunctionHolder._lock.release() + + +def ReplaceSysSetTraceFunc(): + if TracingFunctionHolder._original_tracing is None: + TracingFunctionHolder._original_tracing = sys.settrace + sys.settrace = _InternalSetTrace + diff --git a/python/helpers/pydev/pydevd_vars.py b/python/helpers/pydev/pydevd_vars.py new file mode 100644 index 000000000000..6cc31d454a64 --- /dev/null +++ b/python/helpers/pydev/pydevd_vars.py @@ -0,0 +1,389 @@ +""" pydevd_vars deals with variables: + resolution/conversion to XML. +""" +from pydevd_constants import * #@UnusedWildImport +from types import * #@UnusedWildImport +try: + from StringIO import StringIO +except ImportError: + from io import StringIO +import sys #@Reimport +try: + from urllib import quote +except: + from urllib.parse import quote #@UnresolvedImport +import threading +import pydevd_resolver +import traceback + +#-------------------------------------------------------------------------- defining true and false for earlier versions + +try: + __setFalse = False +except: + import __builtin__ + setattr(__builtin__, 'True', 1) + setattr(__builtin__, 'False', 0) + +#------------------------------------------------------------------------------------------------------ class for errors + +class VariableError(RuntimeError):pass +class FrameNotFoundError(RuntimeError):pass + + +#------------------------------------------------------------------------------------------------------ resolvers in map + +if not sys.platform.startswith("java"): + typeMap = [ + #None means that it should not be treated as a compound variable + + #isintance does not accept a tuple on some versions of python, so, we must declare it expanded + (type(None), None,), + (int, None), + (float, None), + (complex, None), + (str, None), + (tuple, pydevd_resolver.tupleResolver), + (list, pydevd_resolver.tupleResolver), + (dict, pydevd_resolver.dictResolver), + ] + + try: + typeMap.append((long, None)) + except: + pass #not available on all python versions + + try: + typeMap.append((unicode, None)) + except: + pass #not available on all python versions + + try: + typeMap.append((set, pydevd_resolver.setResolver)) + except: + pass #not available on all python versions + + try: + typeMap.append((frozenset, pydevd_resolver.setResolver)) + except: + pass #not available on all python versions + +else: #platform is java + from org.python import core #@UnresolvedImport + typeMap = [ + (core.PyNone, None), + (core.PyInteger, None), + (core.PyLong, None), + (core.PyFloat, None), + (core.PyComplex, None), + (core.PyString, None), + (core.PyTuple, pydevd_resolver.tupleResolver), + (core.PyList, pydevd_resolver.tupleResolver), + (core.PyDictionary, pydevd_resolver.dictResolver), + (core.PyStringMap, pydevd_resolver.dictResolver), + ] + + if hasattr(core, 'PyJavaInstance'): + #Jython 2.5b3 removed it. + typeMap.append((core.PyJavaInstance, pydevd_resolver.instanceResolver)) + + +def getType(o): + """ returns a triple (typeObject, typeString, resolver + resolver != None means that variable is a container, + and should be displayed as a hierarchy. + Use the resolver to get its attributes. + + All container objects should have a resolver. + """ + + try: + type_object = type(o) + type_name = type_object.__name__ + except: + #This happens for org.python.core.InitModule + return 'Unable to get Type', 'Unable to get Type', None + + try: + + if type_name == 'org.python.core.PyJavaInstance': + return (type_object, type_name, pydevd_resolver.instanceResolver) + + if type_name == 'org.python.core.PyArray': + return (type_object, type_name, pydevd_resolver.jyArrayResolver) + + for t in typeMap: + if isinstance(o, t[0]): + return (type_object, type_name, t[1]) + except: + traceback.print_exc() + + #no match return default + return (type_object, type_name, pydevd_resolver.defaultResolver) + + +def makeValidXmlValue(s): + return s.replace('<', '<').replace('>', '>') + + +def varToXML(v, name): + """ single variable or dictionary to xml representation """ + type, typeName, resolver = getType(v) + + try: + if hasattr(v, '__class__'): + try: + cName = str(v.__class__) + if cName.find('.') != -1: + cName = cName.split('.')[-1] + + elif cName.find("'") != -1: #does not have '.' (could be something like ) + cName = cName[cName.index("'") + 1:] + + if cName.endswith("'>"): + cName = cName[:-2] + except: + cName = str(v.__class__) + value = '%s: %s' % (cName, v) + else: + value = str(v) + except: + try: + value = repr(v) + except: + value = 'Unable to get repr for %s' % v.__class__ + + xml = ' MAXIMUM_VARIABLE_REPRESENTATION_SIZE: + value = value[0:MAXIMUM_VARIABLE_REPRESENTATION_SIZE] + value += '...' + + #fix to work with unicode values + try: + if not IS_PY3K: + if isinstance(value, unicode): + value = value.encode('utf-8') + else: + if isinstance(value, bytes): + value = value.encode('utf-8') + except TypeError: #in java, unicode is a function + pass + + xmlValue = ' value="%s"' % (makeValidXmlValue(quote(value, '/>_= \t'))) + else: + xmlValue = '' + + if resolver is not None: + xmlCont = ' isContainer="True"' + else: + xmlCont = '' + + return ''.join((xml, xmlValue, xmlCont, ' />\n')) + + +if USE_PSYCO_OPTIMIZATION: + try: + import psyco + varToXML = psyco.proxy(varToXML) + except ImportError: + if hasattr(sys, 'exc_clear'): #jython does not have it + sys.exc_clear() #don't keep the traceback -- clients don't want to see it + + +def frameVarsToXML(frame): + """ dumps frame variables to XML + + """ + xml = "" + + keys = frame.f_locals.keys() + if hasattr(keys, 'sort'): + keys.sort() #Python 3.0 does not have it + else: + keys = sorted(keys) #Jython 2.1 does not have it + + for k in keys: + try: + v = frame.f_locals[k] + xml += varToXML(v, str(k)) + except Exception: + traceback.print_exc() + sys.stderr.write("Unexpected error, recovered safely.\n") + return xml + +def iterFrames(initialFrame): + '''NO-YIELD VERSION: Iterates through all the frames starting at the specified frame (which will be the first returned item)''' + #cannot use yield + frames = [] + + while initialFrame is not None: + frames.append(initialFrame) + initialFrame = initialFrame.f_back + + return frames + +def dumpFrames(thread_id): + sys.stdout.write('dumping frames\n') + if thread_id != GetThreadId(threading.currentThread()) : + raise VariableError("findFrame: must execute on same thread") + + curFrame = GetFrame() + for frame in iterFrames(curFrame): + sys.stdout.write('%s\n' % id(frame)) + +def findFrame(thread_id, frame_id): + """ returns a frame on the thread that has a given frame_id """ + if thread_id != GetThreadId(threading.currentThread()) : + raise VariableError("findFrame: must execute on same thread") + + curFrame = GetFrame() + if frame_id == "*": + return curFrame # any frame is specified with "*" + + frameFound = None + lookingFor = int(frame_id) + + for frame in iterFrames(curFrame): + if lookingFor == id(frame): + frameFound = frame + del frame + break + + del frame + + #for some reason unknown to me, python was holding a reference to the frame + #if we didn't explicitly add those deletes (even after ending this context) + #so, those dels are here for a reason (but still doesn't seem to fix everything) + + #Reason: sys.exc_info holding reference to frame that raises exception (so, other places + #need to call sys.exc_clear()) + del curFrame + + if frameFound is None: + msgFrames = '' + i = 0 + + for frame in iterFrames(GetFrame()): + i += 1 + msgFrames += str(id(frame)) + if i % 5 == 0: + msgFrames += '\n' + else: + msgFrames += ' - ' + + errMsg = '''findFrame: frame not found. +Looking for thread_id:%s, frame_id:%s +Current thread_id:%s, available frames: +%s +''' % (thread_id, lookingFor, GetThreadId(threading.currentThread()), msgFrames) + + raise FrameNotFoundError(errMsg) + + return frameFound + +def resolveCompoundVariable(thread_id, frame_id, scope, attrs): + """ returns the value of the compound variable as a dictionary""" + frame = findFrame(thread_id, frame_id) + attrList = attrs.split('\t') + if scope == "GLOBAL": + var = frame.f_globals + del attrList[0] # globals are special, and they get a single dummy unused attribute + else: + var = frame.f_locals + + for k in attrList: + type, _typeName, resolver = getType(var) + var = resolver.resolve(var, k) + + try: + type, _typeName, resolver = getType(var) + return resolver.getDictionary(var) + except: + traceback.print_exc() + +def evaluateExpression(thread_id, frame_id, expression, doExec): + '''returns the result of the evaluated expression + @param doExec: determines if we should do an exec or an eval + ''' + frame = findFrame(thread_id, frame_id) + + expression = expression.replace('@LINE@', '\n') + + + #Not using frame.f_globals because of https://sourceforge.net/tracker2/?func=detail&aid=2541355&group_id=85796&atid=577329 + #(Names not resolved in generator expression in method) + #See message: http://mail.python.org/pipermail/python-list/2009-January/526522.html + updated_globals = dict() + updated_globals.update(frame.f_globals) + updated_globals.update(frame.f_locals) #locals later because it has precedence over the actual globals + + try: + + if doExec: + try: + #try to make it an eval (if it is an eval we can print it, otherwise we'll exec it and + #it will have whatever the user actually did) + compiled = compile(expression, '', 'eval') + except: + exec(expression, updated_globals, frame.f_locals) + else: + result = eval(compiled, updated_globals, frame.f_locals) + sys.stdout.write('%s\n' % (result,)) + return + + else: + result = None + try: + result = eval(expression, updated_globals, frame.f_locals) + except Exception: + s = StringIO() + traceback.print_exc(file=s) + result = s.getvalue() + return result + finally: + #Should not be kept alive if an exception happens and this frame is kept in the stack. + del updated_globals + del frame + + +def changeAttrExpression(thread_id, frame_id, attr, expression): + '''Changes some attribute in a given frame. + @note: it will not (currently) work if we're not in the topmost frame (that's a python + deficiency -- and it appears that there is no way of making it currently work -- + will probably need some change to the python internals) + ''' + frame = findFrame(thread_id, frame_id) + + try: + expression = expression.replace('@LINE@', '\n') +#tests (needs proposed patch in python accepted) +# if hasattr(frame, 'savelocals'): +# if attr in frame.f_locals: +# frame.f_locals[attr] = eval(expression, frame.f_globals, frame.f_locals) +# frame.savelocals() +# return +# +# elif attr in frame.f_globals: +# frame.f_globals[attr] = eval(expression, frame.f_globals, frame.f_locals) +# return + + + if attr[:7] == "Globals": + attr = attr[8:] + if attr in frame.f_globals: + frame.f_globals[attr] = eval(expression, frame.f_globals, frame.f_locals) + else: + #default way (only works for changing it in the topmost frame) + exec('%s=%s' % (attr, expression), frame.f_globals, frame.f_locals) + + + except Exception: + traceback.print_exc() + + + + + diff --git a/python/helpers/pydev/pydevd_vm_type.py b/python/helpers/pydev/pydevd_vm_type.py new file mode 100644 index 000000000000..76aa8906f626 --- /dev/null +++ b/python/helpers/pydev/pydevd_vm_type.py @@ -0,0 +1,41 @@ +import sys + +#======================================================================================================================= +# PydevdVmType +#======================================================================================================================= +class PydevdVmType: + + PYTHON = 'python' + JYTHON = 'jython' + vm_type = None + + +#======================================================================================================================= +# SetVmType +#======================================================================================================================= +def SetVmType(vm_type): + PydevdVmType.vm_type = vm_type + + +#======================================================================================================================= +# GetVmType +#======================================================================================================================= +def GetVmType(): + if PydevdVmType.vm_type is None: + SetupType() + return PydevdVmType.vm_type + + +#======================================================================================================================= +# SetupType +#======================================================================================================================= +def SetupType(str=None): + if str is not None: + PydevdVmType.vm_type = str + return + + if sys.platform.startswith("java"): + PydevdVmType.vm_type = PydevdVmType.JYTHON + else: + PydevdVmType.vm_type = PydevdVmType.PYTHON + diff --git a/python/helpers/pydev/runfiles.py b/python/helpers/pydev/runfiles.py new file mode 100644 index 000000000000..4a25469c1fec --- /dev/null +++ b/python/helpers/pydev/runfiles.py @@ -0,0 +1,530 @@ +import fnmatch +import os.path +import re +import sys +import unittest + + + + +try: + __setFalse = False +except: + import __builtin__ + setattr(__builtin__, 'True', 1) + setattr(__builtin__, 'False', 0) + + + + +#======================================================================================================================= +# Jython? +#======================================================================================================================= +try: + import org.python.core.PyDictionary #@UnresolvedImport @UnusedImport -- just to check if it could be valid + def DictContains(d, key): + return d.has_key(key) +except: + try: + #Py3k does not have has_key anymore, and older versions don't have __contains__ + DictContains = dict.__contains__ + except: + DictContains = dict.has_key + +try: + xrange +except: + #Python 3k does not have it + xrange = range + +try: + enumerate +except: + def enumerate(lst): + ret = [] + i=0 + for element in lst: + ret.append((i, element)) + i+=1 + return ret + + + +#======================================================================================================================= +# getopt code copied since gnu_getopt is not available on jython 2.1 +#======================================================================================================================= +class GetoptError(Exception): + opt = '' + msg = '' + def __init__(self, msg, opt=''): + self.msg = msg + self.opt = opt + Exception.__init__(self, msg, opt) + + def __str__(self): + return self.msg + + +def gnu_getopt(args, shortopts, longopts=[]): + """getopt(args, options[, long_options]) -> opts, args + + This function works like getopt(), except that GNU style scanning + mode is used by default. This means that option and non-option + arguments may be intermixed. The getopt() function stops + processing options as soon as a non-option argument is + encountered. + + If the first character of the option string is `+', or if the + environment variable POSIXLY_CORRECT is set, then option + processing stops as soon as a non-option argument is encountered. + """ + + opts = [] + prog_args = [] + if isinstance(longopts, ''.__class__): + longopts = [longopts] + else: + longopts = list(longopts) + + # Allow options after non-option arguments? + if shortopts.startswith('+'): + shortopts = shortopts[1:] + all_options_first = True + elif os.environ.get("POSIXLY_CORRECT"): + all_options_first = True + else: + all_options_first = False + + while args: + if args[0] == '--': + prog_args += args[1:] + break + + if args[0][:2] == '--': + opts, args = do_longs(opts, args[0][2:], longopts, args[1:]) + elif args[0][:1] == '-': + opts, args = do_shorts(opts, args[0][1:], shortopts, args[1:]) + else: + if all_options_first: + prog_args += args + break + else: + prog_args.append(args[0]) + args = args[1:] + + return opts, prog_args + +def do_longs(opts, opt, longopts, args): + try: + i = opt.index('=') + except ValueError: + optarg = None + else: + opt, optarg = opt[:i], opt[i + 1:] + + has_arg, opt = long_has_args(opt, longopts) + if has_arg: + if optarg is None: + if not args: + raise GetoptError('option --%s requires argument' % opt, opt) + optarg, args = args[0], args[1:] + elif optarg: + raise GetoptError('option --%s must not have an argument' % opt, opt) + opts.append(('--' + opt, optarg or '')) + return opts, args + +# Return: +# has_arg? +# full option name +def long_has_args(opt, longopts): + possibilities = [o for o in longopts if o.startswith(opt)] + if not possibilities: + raise GetoptError('option --%s not recognized' % opt, opt) + # Is there an exact match? + if opt in possibilities: + return False, opt + elif opt + '=' in possibilities: + return True, opt + # No exact match, so better be unique. + if len(possibilities) > 1: + # XXX since possibilities contains all valid continuations, might be + # nice to work them into the error msg + raise GetoptError('option --%s not a unique prefix' % opt, opt) + assert len(possibilities) == 1 + unique_match = possibilities[0] + has_arg = unique_match.endswith('=') + if has_arg: + unique_match = unique_match[:-1] + return has_arg, unique_match + +def do_shorts(opts, optstring, shortopts, args): + while optstring != '': + opt, optstring = optstring[0], optstring[1:] + if short_has_arg(opt, shortopts): + if optstring == '': + if not args: + raise GetoptError('option -%s requires argument' % opt, + opt) + optstring, args = args[0], args[1:] + optarg, optstring = optstring, '' + else: + optarg = '' + opts.append(('-' + opt, optarg)) + return opts, args + +def short_has_arg(opt, shortopts): + for i in range(len(shortopts)): + if opt == shortopts[i] != ':': + return shortopts.startswith(':', i + 1) + raise GetoptError('option -%s not recognized' % opt, opt) + + +#======================================================================================================================= +# End getopt code +#======================================================================================================================= + + + + + + + + + + +#======================================================================================================================= +# parse_cmdline +#======================================================================================================================= +def parse_cmdline(): + """ parses command line and returns test directories, verbosity, test filter and test suites + usage: + runfiles.py -v|--verbosity -f|--filter -t|--tests dirs|files + """ + verbosity = 2 + test_filter = None + tests = None + + optlist, dirs = gnu_getopt(sys.argv[1:], "v:f:t:", ["verbosity=", "filter=", "tests="]) + for opt, value in optlist: + if opt in ("-v", "--verbosity"): + verbosity = value + + elif opt in ("-f", "--filter"): + test_filter = value.split(',') + + elif opt in ("-t", "--tests"): + tests = value.split(',') + + if type([]) != type(dirs): + dirs = [dirs] + + ret_dirs = [] + for d in dirs: + if '|' in d: + #paths may come from the ide separated by | + ret_dirs.extend(d.split('|')) + else: + ret_dirs.append(d) + + return ret_dirs, int(verbosity), test_filter, tests + + +#======================================================================================================================= +# PydevTestRunner +#======================================================================================================================= +class PydevTestRunner: + """ finds and runs a file or directory of files as a unit test """ + + __py_extensions = ["*.py", "*.pyw"] + __exclude_files = ["__init__.*"] + + def __init__(self, test_dir, test_filter=None, verbosity=2, tests=None): + self.test_dir = test_dir + self.__adjust_path() + self.test_filter = self.__setup_test_filter(test_filter) + self.verbosity = verbosity + self.tests = tests + + + def __adjust_path(self): + """ add the current file or directory to the python path """ + path_to_append = None + for n in xrange(len(self.test_dir)): + dir_name = self.__unixify(self.test_dir[n]) + if os.path.isdir(dir_name): + if not dir_name.endswith("/"): + self.test_dir[n] = dir_name + "/" + path_to_append = os.path.normpath(dir_name) + elif os.path.isfile(dir_name): + path_to_append = os.path.dirname(dir_name) + else: + msg = ("unknown type. \n%s\nshould be file or a directory.\n" % (dir_name)) + raise RuntimeError(msg) + if path_to_append is not None: + #Add it as the last one (so, first things are resolved against the default dirs and + #if none resolves, then we try a relative import). + sys.path.append(path_to_append) + return + + def __setup_test_filter(self, test_filter): + """ turn a filter string into a list of filter regexes """ + if test_filter is None or len(test_filter) == 0: + return None + return [re.compile("test%s" % f) for f in test_filter] + + def __is_valid_py_file(self, fname): + """ tests that a particular file contains the proper file extension + and is not in the list of files to exclude """ + is_valid_fname = 0 + for invalid_fname in self.__class__.__exclude_files: + is_valid_fname += int(not fnmatch.fnmatch(fname, invalid_fname)) + if_valid_ext = 0 + for ext in self.__class__.__py_extensions: + if_valid_ext += int(fnmatch.fnmatch(fname, ext)) + return is_valid_fname > 0 and if_valid_ext > 0 + + def __unixify(self, s): + """ stupid windows. converts the backslash to forwardslash for consistency """ + return os.path.normpath(s).replace(os.sep, "/") + + def __importify(self, s, dir=False): + """ turns directory separators into dots and removes the ".py*" extension + so the string can be used as import statement """ + if not dir: + dirname, fname = os.path.split(s) + + if fname.count('.') > 1: + #if there's a file named xxx.xx.py, it is not a valid module, so, let's not load it... + return + + imp_stmt_pieces = [dirname.replace("\\", "/").replace("/", "."), os.path.splitext(fname)[0]] + + if len(imp_stmt_pieces[0]) == 0: + imp_stmt_pieces = imp_stmt_pieces[1:] + + return ".".join(imp_stmt_pieces) + + else: #handle dir + return s.replace("\\", "/").replace("/", ".") + + def __add_files(self, pyfiles, root, files): + """ if files match, appends them to pyfiles. used by os.path.walk fcn """ + for fname in files: + if self.__is_valid_py_file(fname): + name_without_base_dir = self.__unixify(os.path.join(root, fname)) + pyfiles.append(name_without_base_dir) + return + + + def find_import_files(self): + """ return a list of files to import """ + pyfiles = [] + + for base_dir in self.test_dir: + if os.path.isdir(base_dir): + if hasattr(os, 'walk'): + for root, dirs, files in os.walk(base_dir): + self.__add_files(pyfiles, root, files) + else: + # jython2.1 is too old for os.walk! + os.path.walk(base_dir, self.__add_files, pyfiles) + + elif os.path.isfile(base_dir): + pyfiles.append(base_dir) + + return pyfiles + + def __get_module_from_str(self, modname, print_exception): + """ Import the module in the given import path. + * Returns the "final" module, so importing "coilib40.subject.visu" + returns the "visu" module, not the "coilib40" as returned by __import__ """ + try: + mod = __import__(modname) + for part in modname.split('.')[1:]: + mod = getattr(mod, part) + return mod + except: + if print_exception: + import traceback;traceback.print_exc() + sys.stderr.write('ERROR: Module: %s could not be imported.\n' % (modname,)) + return None + + def find_modules_from_files(self, pyfiles): + """ returns a lisst of modules given a list of files """ + #let's make sure that the paths we want are in the pythonpath... + imports = [self.__importify(s) for s in pyfiles] + + system_paths = [] + for s in sys.path: + system_paths.append(self.__importify(s, True)) + + + ret = [] + for imp in imports: + if imp is None: + continue #can happen if a file is not a valid module + choices = [] + for s in system_paths: + if imp.startswith(s): + add = imp[len(s) + 1:] + if add: + choices.append(add) + #sys.stdout.write(' ' + add + ' ') + + if not choices: + sys.stdout.write('PYTHONPATH not found for file: %s\n' % imp) + else: + for i, import_str in enumerate(choices): + mod = self.__get_module_from_str(import_str, print_exception=i == len(choices) - 1) + if mod is not None: + ret.append(mod) + break + + + return ret + + def find_tests_from_modules(self, modules): + """ returns the unittests given a list of modules """ + loader = unittest.TestLoader() + + ret = [] + if self.tests: + accepted_classes = {} + accepted_methods = {} + + for t in self.tests: + splitted = t.split('.') + if len(splitted) == 1: + accepted_classes[t] = t + + elif len(splitted) == 2: + accepted_methods[t] = t + + #=========================================================================================================== + # GetTestCaseNames + #=========================================================================================================== + class GetTestCaseNames: + """Yes, we need a class for that (cannot use outer context on jython 2.1)""" + + def __init__(self, accepted_classes, accepted_methods): + self.accepted_classes = accepted_classes + self.accepted_methods = accepted_methods + + def __call__(self, testCaseClass): + """Return a sorted sequence of method names found within testCaseClass""" + testFnNames = [] + className = testCaseClass.__name__ + + if DictContains(self.accepted_classes, className): + for attrname in dir(testCaseClass): + #If a class is chosen, we select all the 'test' methods' + if attrname.startswith('test') and hasattr(getattr(testCaseClass, attrname), '__call__'): + testFnNames.append(attrname) + + else: + for attrname in dir(testCaseClass): + #If we have the class+method name, we must do a full check and have an exact match. + if DictContains(self.accepted_methods, className + '.' + attrname): + if hasattr(getattr(testCaseClass, attrname), '__call__'): + testFnNames.append(attrname) + + #sorted() is not available in jython 2.1 + testFnNames.sort() + return testFnNames + + + loader.getTestCaseNames = GetTestCaseNames(accepted_classes, accepted_methods) + + + ret.extend([loader.loadTestsFromModule(m) for m in modules]) + + return ret + + + def filter_tests(self, test_objs): + """ based on a filter name, only return those tests that have + the test case names that match """ + test_suite = [] + for test_obj in test_objs: + + if isinstance(test_obj, unittest.TestSuite): + if test_obj._tests: + test_obj._tests = self.filter_tests(test_obj._tests) + if test_obj._tests: + test_suite.append(test_obj) + + elif isinstance(test_obj, unittest.TestCase): + test_cases = [] + for tc in test_objs: + try: + testMethodName = tc._TestCase__testMethodName + except AttributeError: + #changed in python 2.5 + testMethodName = tc._testMethodName + + if self.__match(self.test_filter, testMethodName) and self.__match_tests(self.tests, tc, testMethodName): + test_cases.append(tc) + return test_cases + return test_suite + + + def __match_tests(self, tests, test_case, test_method_name): + if not tests: + return 1 + + for t in tests: + class_and_method = t.split('.') + if len(class_and_method) == 1: + #only class name + if class_and_method[0] == test_case.__class__.__name__: + return 1 + + elif len(class_and_method) == 2: + if class_and_method[0] == test_case.__class__.__name__ and class_and_method[1] == test_method_name: + return 1 + + return 0 + + + + + def __match(self, filter_list, name): + """ returns whether a test name matches the test filter """ + if filter_list is None: + return 1 + for f in filter_list: + if re.match(f, name): + return 1 + return 0 + + + def run_tests(self): + """ runs all tests """ + sys.stdout.write("Finding files...\n") + files = self.find_import_files() + sys.stdout.write('%s %s\n' % (self.test_dir, '... done')) + sys.stdout.write("Importing test modules ... ") + modules = self.find_modules_from_files(files) + sys.stdout.write("done.\n") + all_tests = self.find_tests_from_modules(modules) + if self.test_filter or self.tests: + + if self.test_filter: + sys.stdout.write('Test Filter: %s' % ([p.pattern for p in self.test_filter],)) + + if self.tests: + sys.stdout.write('Tests to run: %s' % (self.tests,)) + + all_tests = self.filter_tests(all_tests) + + sys.stdout.write('\n') + runner = unittest.TextTestRunner(stream=sys.stdout, descriptions=1, verbosity=verbosity) + runner.run(unittest.TestSuite(all_tests)) + return + +#======================================================================================================================= +# main +#======================================================================================================================= +if __name__ == '__main__': + dirs, verbosity, test_filter, tests = parse_cmdline() + PydevTestRunner(dirs, test_filter, verbosity, tests).run_tests() diff --git a/python/helpers/pydev/unittest2.py b/python/helpers/pydev/unittest2.py new file mode 100644 index 000000000000..98b28268ec2f --- /dev/null +++ b/python/helpers/pydev/unittest2.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python +from unittest import TestResult + +class TestListener: + """ Simulate a Java interface by providing an abstract class + All methods need to be implemented by classes extending TestListener + """ + + # Test was successful. + def addSuccess(self, test): + raise NotImplementedError, "TestListener.addSuccess()" + + # An error occured + def addError(self, test, err): + raise NotImplementedError, "TestListener.addError()" + + # A failure occurred. + def addFailure(self, test, err): + raise NotImplementedError, "TestListener.addFailure()" + + # A test started. + def startTest(self, test): + raise NotImplementedError, "TestListener.startTest()" + + # A test ended. + def endTest(self, test): + raise NotImplementedError, "TestListener.endTest()" + +class TestResultWithListeners(TestResult): + def __init__(self): + TestResult.__init__(self) + self.listeners = [] + + def startTest(self, test): + TestResult.startTest(self, test) + for listener in self.listeners: + listener.startTest(test) + + def endTest(self, test): + for listener in self.listeners: + listener.endTest(test) + + def addSuccess(self, test): + TestResult.addSuccess(self, test) + for listener in self.listeners: + listener.addSuccess(test) + + def addError(self, test, err): + TestResult.addError(self, test, err) + for listener in self.listeners: + listener.addError(test, err) + + def addFailure(self, test, err): + TestResult.addFailure(self, test, err) + for listener in self.listeners: + listener.addFailure(test, err) + + def addListener(self, listener): + self.listeners.append(listener) + + def removeListener(self, listener): + self.listeners.remove(listener) + diff --git a/python/python.iml b/python/python.iml index ef3ad1488da1..86930cbb6efa 100644 --- a/python/python.iml +++ b/python/python.iml @@ -32,6 +32,7 @@ + diff --git a/python/src/META-INF/python-plugin-common.xml b/python/src/META-INF/python-plugin-common.xml index 4b247cde2d91..5772ab69756b 100644 --- a/python/src/META-INF/python-plugin-common.xml +++ b/python/src/META-INF/python-plugin-common.xml @@ -41,7 +41,9 @@ + + diff --git a/python/src/com/jetbrains/python/PythonHelpersLocator.java b/python/src/com/jetbrains/python/PythonHelpersLocator.java index 60b005480116..d3bc628be0da 100644 --- a/python/src/com/jetbrains/python/PythonHelpersLocator.java +++ b/python/src/com/jetbrains/python/PythonHelpersLocator.java @@ -1,8 +1,8 @@ package com.jetbrains.python; -import org.jetbrains.annotations.NonNls; -import com.intellij.util.PathUtil; import com.intellij.openapi.diagnostic.Logger; +import com.intellij.util.PathUtil; +import org.jetbrains.annotations.NonNls; import java.io.File; @@ -23,4 +23,8 @@ public class PythonHelpersLocator { } return new File(jarPath); } + + public static String getHelperPath(String scriptName) { + return new File(getHelpersRoot(), scriptName).getAbsolutePath(); + } } diff --git a/python/src/com/jetbrains/python/debugger/PyDebugProcess.java b/python/src/com/jetbrains/python/debugger/PyDebugProcess.java new file mode 100644 index 000000000000..5cdfbb3e26d6 --- /dev/null +++ b/python/src/com/jetbrains/python/debugger/PyDebugProcess.java @@ -0,0 +1,226 @@ +package com.jetbrains.python.debugger; + +import com.intellij.execution.process.ProcessHandler; +import com.intellij.execution.ui.ConsoleView; +import com.intellij.execution.ui.ConsoleViewContentType; +import com.intellij.execution.ui.ExecutionConsole; +import com.intellij.openapi.progress.ProgressIndicator; +import com.intellij.openapi.progress.ProgressManager; +import com.intellij.openapi.progress.Task; +import com.intellij.openapi.ui.Messages; +import com.intellij.xdebugger.XDebugProcess; +import com.intellij.xdebugger.XDebugSession; +import com.intellij.xdebugger.XSourcePosition; +import com.intellij.xdebugger.breakpoints.XBreakpointHandler; +import com.intellij.xdebugger.breakpoints.XLineBreakpoint; +import com.intellij.xdebugger.evaluation.XDebuggerEditorsProvider; +import com.jetbrains.python.debugger.local.PyLocalPositionConverter; +import com.jetbrains.python.debugger.pydev.*; +import org.jetbrains.annotations.NotNull; + +import java.net.ServerSocket; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import static javax.swing.SwingUtilities.invokeLater; + +/** + * @author yole + */ +// todo: bundle messages +// todo: pydevd supports module reloading - look for a way to use the feature +// todo: smart step into +public class PyDebugProcess extends XDebugProcess { + + private final PyPositionConverter myPositionConverter = new PyLocalPositionConverter(); + private final RemoteDebugger myDebugger; + private final XBreakpointHandler[] myBreakpointHandlers; + private final PyDebuggerEditorsProvider myEditorsProvider; + private final ProcessHandler myProcessHandler; + private final ExecutionConsole myExecutionConsole; + private final Map myRegisteredBreakpoints = new ConcurrentHashMap(); + private volatile PyThreadInfo mySuspendedThread = null; + + protected PyDebugProcess(@NotNull XDebugSession session, + final ServerSocket serverSocket, + final ExecutionConsole executionConsole, + final ProcessHandler processHandler) { + super(session); + session.setPauseActionSupported(true); // todo: implement and drop + myDebugger = new RemoteDebugger(this, serverSocket, 10); + myBreakpointHandlers = new XBreakpointHandler[]{new PyLineBreakpointHandler(this)}; + myEditorsProvider = new PyDebuggerEditorsProvider(); + myProcessHandler = processHandler; + myExecutionConsole = executionConsole; + } + + public PyPositionConverter getPositionConverter() { + return myPositionConverter; + } + + public XBreakpointHandler[] getBreakpointHandlers() { + return myBreakpointHandlers; + } + + @Override + public XDebuggerEditorsProvider getEditorsProvider() { + return myEditorsProvider; + } + + protected ProcessHandler doGetProcessHandler() { + return myProcessHandler; + } + + @NotNull + public ExecutionConsole createConsole() { + return myExecutionConsole; + } + + @Override + public void sessionInitialized() { + super.sessionInitialized(); + ProgressManager.getInstance().run(new Task.Backgroundable(null, "Connecting to debugger", false) { + public void run(@NotNull final ProgressIndicator indicator) { + indicator.setText("Connecting to debugger..."); + try { + myDebugger.waitForConnect(); + handshake(); + registerBreakpoints(); + myDebugger.sendCommand(new RunCommand()); + } + catch (final Exception e) { + myProcessHandler.destroyProcess(); + invokeLater(new Runnable() { + public void run() { + Messages.showErrorDialog("Unable to establish connection with debugger:\n" + e.getMessage(), "Connecting to debugger"); + } + }); + } + } + }); + } + + private void handshake() throws PyDebuggerException { + final String remoteVersion = myDebugger.handshake(); + ((ConsoleView)myExecutionConsole).print("Connected to pydevd (version " + remoteVersion + ")\n", ConsoleViewContentType.SYSTEM_OUTPUT); + } + + private void registerBreakpoints() { + for (Map.Entry entry : myRegisteredBreakpoints.entrySet()) { + addBreakpoint(entry.getKey(), entry.getValue()); + } + } + + public void startStepOver() { + resume(ResumeCommand.Mode.STEP_OVER); + } + + public void startStepInto() { + resume(ResumeCommand.Mode.STEP_INTO); + } + + public void startStepOut() { + resume(ResumeCommand.Mode.STEP_OUT); + } + + public void stop() { + myDebugger.disconnect(); + } + + public void resume() { + resume(ResumeCommand.Mode.RESUME); + } + + private void resume(final ResumeCommand.Mode mode) { + if (myDebugger.isConnected() && mySuspendedThread != null) { + final ResumeCommand command = new ResumeCommand(mySuspendedThread.getId(), mode); + mySuspendedThread = null; + myDebugger.sendCommand(command); + } + } + + public void runToPosition(@NotNull XSourcePosition position) { + if (myDebugger.isConnected() && mySuspendedThread != null) { + final PySourcePosition pyPosition = myPositionConverter.convert(position); + myDebugger.sendCommand(new SetBreakpointCommand(pyPosition.getFile(), pyPosition.getLine())); // set temp. breakpoint + resume(ResumeCommand.Mode.RESUME); + } + } + + public PyDebugValue evaluate(final String expression) throws PyDebuggerException { + PyStackFrame frame = currentFrame(); + return myDebugger.evaluate(frame.getThreadId(), frame.getFrameId(), expression); + } + + public List loadFrame() throws PyDebuggerException { + PyStackFrame frame = currentFrame(); + return myDebugger.loadFrame(frame.getThreadId(), frame.getFrameId()); + } + + private PyStackFrame currentFrame() throws PyDebuggerException { + if (!myDebugger.isConnected()) { + throw new PyDebuggerException("Disconnected"); + } + + final PyStackFrame frame = (PyStackFrame)getSession().getCurrentStackFrame(); + if (frame == null) { + throw new PyDebuggerException("Process is running"); + } + + return frame; + } + + public void addBreakpoint(PySourcePosition position, XLineBreakpoint breakpoint) { + myRegisteredBreakpoints.put(position, breakpoint); + if (myDebugger.isConnected()) { + myDebugger.sendCommand(new SetBreakpointCommand(position.getFile(), position.getLine())); + } + } + + public void removeBreakpoint(PySourcePosition position) { + myRegisteredBreakpoints.remove(position); + if (myDebugger.isConnected()) { + myDebugger.sendCommand(new RemoveBreakpointCommand(position.getFile(), position.getLine())); + } + } + + public Collection getThreads() { + return myDebugger.getThreads(); + } + + public void threadSuspended(final PyThreadInfo threadInfo) { + if (mySuspendedThread != null) { + // todo: XDebugSession supports only one suspend context + final ResumeCommand command = new ResumeCommand(threadInfo.getId(), ResumeCommand.Mode.RESUME); + myDebugger.sendCommand(command); + return; + } + mySuspendedThread = threadInfo; + + final List frames = threadInfo.getFrames(); + if (frames != null) { + final PySuspendContext suspendContext = new PySuspendContext(this, threadInfo); + + XLineBreakpoint breakpoint = null; + if (threadInfo.isStopOnBreakpoint()) { + final PySourcePosition position = frames.get(0).getPosition(); + breakpoint = myRegisteredBreakpoints.get(position); + if (breakpoint == null) { + myDebugger.sendCommand(new RemoveBreakpointCommand(position.getFile(), position.getLine())); // remove temp. breakpoint + } + } + + if (breakpoint != null) { + if (!getSession().breakpointReached(breakpoint, suspendContext)) { + resume(); + } + } + else { + getSession().positionReached(suspendContext); + } + } + } + +} diff --git a/python/src/com/jetbrains/python/debugger/PyDebugRunner.java b/python/src/com/jetbrains/python/debugger/PyDebugRunner.java new file mode 100644 index 000000000000..3107d1fa66c3 --- /dev/null +++ b/python/src/com/jetbrains/python/debugger/PyDebugRunner.java @@ -0,0 +1,81 @@ +package com.jetbrains.python.debugger; + +import com.intellij.execution.ExecutionException; +import com.intellij.execution.ExecutionResult; +import com.intellij.execution.Executor; +import com.intellij.execution.configurations.GeneralCommandLine; +import com.intellij.execution.configurations.RunProfile; +import com.intellij.execution.configurations.RunProfileState; +import com.intellij.execution.executors.DefaultDebugExecutor; +import com.intellij.execution.runners.ExecutionEnvironment; +import com.intellij.execution.runners.GenericProgramRunner; +import com.intellij.execution.ui.RunContentDescriptor; +import com.intellij.openapi.fileEditor.FileDocumentManager; +import com.intellij.openapi.project.Project; +import com.intellij.xdebugger.XDebugProcess; +import com.intellij.xdebugger.XDebugProcessStarter; +import com.intellij.xdebugger.XDebugSession; +import com.intellij.xdebugger.XDebuggerManager; +import com.jetbrains.python.PythonHelpersLocator; +import com.jetbrains.python.run.AbstractPythonRunConfiguration; +import com.jetbrains.python.run.CommandLinePatcher; +import com.jetbrains.python.run.PythonCommandLineState; +import org.jetbrains.annotations.NotNull; + +import java.io.IOException; +import java.net.ServerSocket; + +/** + * @author yole + */ +public class PyDebugRunner extends GenericProgramRunner { + @NotNull + public String getRunnerId() { + return "PyDebugRunner"; + } + + public boolean canRun(@NotNull String executorId, @NotNull RunProfile profile) { + return DefaultDebugExecutor.EXECUTOR_ID.equals(executorId) && profile instanceof AbstractPythonRunConfiguration; + } + + protected RunContentDescriptor doExecute(Project project, Executor executor, RunProfileState state, + RunContentDescriptor contentToReuse, + ExecutionEnvironment env) throws ExecutionException { + FileDocumentManager.getInstance().saveAllDocuments(); + + final ServerSocket serverSocket; + try { + //noinspection SocketOpenedButNotSafelyClosed + serverSocket = new ServerSocket(0); + } + catch (IOException e) { + throw new ExecutionException("Failed to find free socket port", e); + } + + PythonCommandLineState pyState = (PythonCommandLineState) state; + final ExecutionResult result = pyState.execute(new CommandLinePatcher() { + public void patchCommandLine(GeneralCommandLine commandLine) { + final String[] args = new String[]{ + PythonHelpersLocator.getHelperPath("pydev/pydevd.py"), + "--client", + "127.0.0.1", + "--port", + String.valueOf(serverSocket.getLocalPort()), + "--file" + }; + for (int i=0; i frames = myThreadInfo.getFrames(); + return frames != null ? convert(myDebugProcess, frames.get(0)) : null; + } + + @Override + public void computeStackFrames(int firstFrameIndex, XStackFrameContainer container) { + if (myThreadInfo.getState() != PyThreadInfo.State.SUSPENDED) { + container.errorOccured("Frames not available in non-suspended state"); + return; + } + + final List frames = myThreadInfo.getFrames(); + if (frames != null && firstFrameIndex <= frames.size()) { + final List xFrames = new LinkedList(); + for (int i = firstFrameIndex; i < frames.size(); i++) { + xFrames.add(convert(myDebugProcess, frames.get(i))); + } + container.addStackFrames(xFrames, true); + } + else { + container.addStackFrames(Collections.emptyList(), true); + } + } + + private static PyStackFrame convert(final PyDebugProcess debugProcess, final PyStackFrameInfo frameInfo) { + return new PyStackFrame(debugProcess, frameInfo); + } + +} diff --git a/python/src/com/jetbrains/python/debugger/PyLineBreakpointHandler.java b/python/src/com/jetbrains/python/debugger/PyLineBreakpointHandler.java new file mode 100644 index 000000000000..742ddecc4f46 --- /dev/null +++ b/python/src/com/jetbrains/python/debugger/PyLineBreakpointHandler.java @@ -0,0 +1,33 @@ +package com.jetbrains.python.debugger; + +import com.intellij.xdebugger.XSourcePosition; +import com.intellij.xdebugger.breakpoints.XBreakpointHandler; +import com.intellij.xdebugger.breakpoints.XBreakpointProperties; +import com.intellij.xdebugger.breakpoints.XLineBreakpoint; +import org.jetbrains.annotations.NotNull; + + +public class PyLineBreakpointHandler extends XBreakpointHandler> { + + private final PyDebugProcess myDebugProcess; + + public PyLineBreakpointHandler(@NotNull final PyDebugProcess debugProcess) { + super(PyLineBreakpointType.class); + myDebugProcess = debugProcess; + } + + public void registerBreakpoint(@NotNull XLineBreakpoint breakpoint) { + final XSourcePosition position = breakpoint.getSourcePosition(); + if (position != null) { + myDebugProcess.addBreakpoint(myDebugProcess.getPositionConverter().convert(position), breakpoint); + } + } + + public void unregisterBreakpoint(@NotNull XLineBreakpoint breakpoint, boolean temporary) { + final XSourcePosition position = breakpoint.getSourcePosition(); + if (position != null) { + myDebugProcess.removeBreakpoint(myDebugProcess.getPositionConverter().convert(position)); + } + } + +} diff --git a/python/src/com/jetbrains/python/debugger/PyLineBreakpointType.java b/python/src/com/jetbrains/python/debugger/PyLineBreakpointType.java new file mode 100644 index 000000000000..88e65eab70b0 --- /dev/null +++ b/python/src/com/jetbrains/python/debugger/PyLineBreakpointType.java @@ -0,0 +1,49 @@ +package com.jetbrains.python.debugger; + +import com.intellij.openapi.editor.Document; +import com.intellij.openapi.fileEditor.FileDocumentManager; +import com.intellij.openapi.project.Project; +import com.intellij.openapi.util.Ref; +import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.psi.PsiComment; +import com.intellij.psi.PsiElement; +import com.intellij.psi.PsiWhiteSpace; +import com.intellij.util.Processor; +import com.intellij.xdebugger.XDebuggerUtil; +import com.intellij.xdebugger.breakpoints.XBreakpointProperties; +import com.intellij.xdebugger.breakpoints.XLineBreakpointType; +import com.jetbrains.python.PythonFileType; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + + +public class PyLineBreakpointType extends XLineBreakpointType { + + public PyLineBreakpointType() { + super("python-line", "Python Line Breakpoint"); + } + + public boolean canPutAt(@NotNull VirtualFile file, int line, @NotNull Project project) { + final Ref stoppable = Ref.create(false); + if (file.getFileType() == PythonFileType.INSTANCE) { + final Document document = FileDocumentManager.getInstance().getDocument(file); + if (document != null) { + XDebuggerUtil.getInstance().iterateLine(project, document, line, new Processor() { + public boolean process(PsiElement psiElement) { + if (psiElement instanceof PsiWhiteSpace || psiElement instanceof PsiComment) return true; + // Python debugger seems to be able to stop on pretty much everything + stoppable.set(true); + return false; + } + }); + } + } + return stoppable.get(); + } + + @Nullable + public XBreakpointProperties createBreakpointProperties(@NotNull VirtualFile file, int line) { + return null; + } + +} diff --git a/python/src/com/jetbrains/python/debugger/PyPositionConverter.java b/python/src/com/jetbrains/python/debugger/PyPositionConverter.java new file mode 100644 index 000000000000..8fc629b219a7 --- /dev/null +++ b/python/src/com/jetbrains/python/debugger/PyPositionConverter.java @@ -0,0 +1,19 @@ +package com.jetbrains.python.debugger; + +import com.intellij.xdebugger.XSourcePosition; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + + +public interface PyPositionConverter { + + @NotNull + PySourcePosition create(@NotNull final String file, final int line); + + @NotNull + PySourcePosition convert(@NotNull final XSourcePosition position); + + @Nullable + XSourcePosition convert(@NotNull final PySourcePosition position); + +} diff --git a/python/src/com/jetbrains/python/debugger/PySourcePosition.java b/python/src/com/jetbrains/python/debugger/PySourcePosition.java new file mode 100644 index 000000000000..9ea979f735f3 --- /dev/null +++ b/python/src/com/jetbrains/python/debugger/PySourcePosition.java @@ -0,0 +1,41 @@ +package com.jetbrains.python.debugger; + +public abstract class PySourcePosition { + + private final String file; + private final int line; + + protected PySourcePosition(final String file, final int line) { + this.file = file; + this.line = line; + } + + public String getFile() { + return file; + } + + public int getLine() { + return line; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof PySourcePosition)) return false; + + PySourcePosition that = (PySourcePosition)o; + + if (line != that.line) return false; + if (file != null ? !file.equals(that.file) : that.file != null) return false; + + return true; + } + + @Override + public int hashCode() { + int result = file != null ? file.hashCode() : 0; + result = 31 * result + line; + return result; + } + +} diff --git a/python/src/com/jetbrains/python/debugger/PyStackFrame.java b/python/src/com/jetbrains/python/debugger/PyStackFrame.java new file mode 100644 index 000000000000..86d03ac4b705 --- /dev/null +++ b/python/src/com/jetbrains/python/debugger/PyStackFrame.java @@ -0,0 +1,104 @@ +package com.jetbrains.python.debugger; + +import com.intellij.openapi.application.ApplicationManager; +import com.intellij.openapi.diagnostic.Logger; +import com.intellij.openapi.editor.Document; +import com.intellij.openapi.fileEditor.FileDocumentManager; +import com.intellij.openapi.project.Project; +import com.intellij.openapi.roots.ProjectRootManager; +import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.ui.SimpleColoredComponent; +import com.intellij.ui.SimpleTextAttributes; +import com.intellij.xdebugger.XSourcePosition; +import com.intellij.xdebugger.evaluation.XDebuggerEvaluator; +import com.intellij.xdebugger.frame.XCompositeNode; +import com.intellij.xdebugger.frame.XStackFrame; +import com.intellij.xdebugger.ui.DebuggerIcons; +import org.jetbrains.annotations.NotNull; + +import java.util.List; + + +public class PyStackFrame extends XStackFrame { + + private static final Logger LOG = Logger.getInstance("#com.jetbrains.python.pydev.PyStackFrame"); + + private final PyDebugProcess myDebugProcess; + private final PyStackFrameInfo myFrameInfo; + private final XSourcePosition myPosition; + + public PyStackFrame(@NotNull final PyDebugProcess debugProcess, @NotNull final PyStackFrameInfo frameInfo) { + myDebugProcess = debugProcess; + myFrameInfo = frameInfo; + myPosition = myDebugProcess.getPositionConverter().convert(frameInfo.getPosition()); + } + + @Override + public XSourcePosition getSourcePosition() { + return myPosition; + } + + @Override + public XDebuggerEvaluator getEvaluator() { + return new PyDebuggerEvaluator(myDebugProcess); + } + + @Override + public void customizePresentation(SimpleColoredComponent component) { + component.setIcon(DebuggerIcons.STACK_FRAME_ICON); + + if (myPosition == null) { + component.append("", SimpleTextAttributes.GRAY_ATTRIBUTES); + return; + } + + boolean isExternal = true; + final VirtualFile file = myPosition.getFile(); + final Document document = FileDocumentManager.getInstance().getDocument(file); + if (document != null) { + final Project project = myDebugProcess.getSession().getProject(); + isExternal = !ProjectRootManager.getInstance(project).getFileIndex().isInContent(file); + } + + component.append(myFrameInfo.getName(), gray(SimpleTextAttributes.REGULAR_ATTRIBUTES, isExternal)); + component.append(", ", gray(SimpleTextAttributes.REGULAR_ATTRIBUTES, isExternal)); + component.append(myPosition.getFile().getName(), gray(SimpleTextAttributes.REGULAR_ATTRIBUTES, isExternal)); + component.append(":", gray(SimpleTextAttributes.REGULAR_ATTRIBUTES, isExternal)); + component.append(Integer.toString(myPosition.getLine() + 1), gray(SimpleTextAttributes.REGULAR_ATTRIBUTES, isExternal)); + } + + private static SimpleTextAttributes gray(SimpleTextAttributes attributes, boolean gray) { + if (!gray) { + return attributes; + } + else { + return (attributes.getStyle() & SimpleTextAttributes.STYLE_ITALIC) != 0 + ? SimpleTextAttributes.GRAY_ITALIC_ATTRIBUTES : SimpleTextAttributes.GRAYED_ATTRIBUTES; + } + } + + @Override + public void computeChildren(@NotNull final XCompositeNode node) { + ApplicationManager.getApplication().executeOnPooledThread(new Runnable() { + public void run() { + try { + final List values = myDebugProcess.loadFrame(); + node.addChildren(values, true); + // todo: globals (?) + } + catch (PyDebuggerException e) { + LOG.error(e); + } + } + }); + } + + public String getThreadId() { + return myFrameInfo.getThreadId(); + } + + public String getFrameId() { + return myFrameInfo.getId(); + } + +} diff --git a/python/src/com/jetbrains/python/debugger/PyStackFrameInfo.java b/python/src/com/jetbrains/python/debugger/PyStackFrameInfo.java new file mode 100644 index 000000000000..b3163d5a937d --- /dev/null +++ b/python/src/com/jetbrains/python/debugger/PyStackFrameInfo.java @@ -0,0 +1,34 @@ +package com.jetbrains.python.debugger; + + +public class PyStackFrameInfo { + + private final String myThreadId; + private final String myId; + private final String myName; + private final PySourcePosition myPosition; + + public PyStackFrameInfo(final String threadId, final String id, final String name, final PySourcePosition position) { + myThreadId = threadId; + myId = id; + myName = name; + myPosition = position; + } + + public String getThreadId() { + return myThreadId; + } + + public String getId() { + return myId; + } + + public String getName() { + return myName; + } + + public PySourcePosition getPosition() { + return myPosition; + } + +} diff --git a/python/src/com/jetbrains/python/debugger/PySuspendContext.java b/python/src/com/jetbrains/python/debugger/PySuspendContext.java new file mode 100644 index 000000000000..2340cc386653 --- /dev/null +++ b/python/src/com/jetbrains/python/debugger/PySuspendContext.java @@ -0,0 +1,41 @@ +package com.jetbrains.python.debugger; + +import com.intellij.xdebugger.frame.XExecutionStack; +import com.intellij.xdebugger.frame.XSuspendContext; +import org.jetbrains.annotations.NotNull; + + +// todo: support for multiple stacks (threads) +public class PySuspendContext extends XSuspendContext { + + private final XExecutionStack myActiveStack; + + public PySuspendContext(@NotNull final PyDebugProcess debugProcess, @NotNull final PyThreadInfo threadInfo) { + myActiveStack = new PyExecutionStack(debugProcess, threadInfo); + } + + @Override + public XExecutionStack getActiveExecutionStack() { + return myActiveStack; + } + + @Override + public XExecutionStack[] getExecutionStacks() { + return super.getExecutionStacks(); + /* + final Collection threads = myDebugProcess.getThreads(); + if (threads.size() < 1) { + return XExecutionStack.EMPTY_ARRAY; + } + else { + XExecutionStack[] stacks = new XExecutionStack[threads.size()]; + int i = 0; + for (PyThreadInfo thread : threads) { + stacks[i++] = new PyExecutionStack(myDebugProcess, thread); + } + return stacks; + } + */ + } + +} diff --git a/python/src/com/jetbrains/python/debugger/PyThreadInfo.java b/python/src/com/jetbrains/python/debugger/PyThreadInfo.java new file mode 100644 index 000000000000..1ae6630e77c4 --- /dev/null +++ b/python/src/com/jetbrains/python/debugger/PyThreadInfo.java @@ -0,0 +1,54 @@ +package com.jetbrains.python.debugger; + +import org.jetbrains.annotations.Nullable; + +import java.util.Collections; +import java.util.List; + + +public class PyThreadInfo { + + public enum State { + RUNNING, SUSPENDED, KILLED + } + + private final String myId; + private final String myName; + private List myFrames; + private State myState; + private final boolean myStopOnBreakpoint; // todo: remove + + public PyThreadInfo(final String id, final String name, final List frames, final boolean stopOnBreakpoint) { + myId = id; + myName = name; + myFrames = (frames != null && frames.size() > 0 ? Collections.unmodifiableList(frames) : null); + myStopOnBreakpoint = stopOnBreakpoint; + } + + public String getId() { + return myId; + } + + public String getName() { + return myName; + } + + @Nullable + public synchronized List getFrames() { + return myFrames; + } + + public synchronized State getState() { + return myState; + } + + public synchronized void updateState(final State state, final List frames) { + myState = state; + myFrames = (frames != null && frames.size() > 0 ? Collections.unmodifiableList(frames) : null); + } + + public boolean isStopOnBreakpoint() { + return myStopOnBreakpoint; + } + +} diff --git a/python/src/com/jetbrains/python/psi/PyExpressionCodeFragment.java b/python/src/com/jetbrains/python/psi/PyExpressionCodeFragment.java new file mode 100644 index 000000000000..195ef2d14662 --- /dev/null +++ b/python/src/com/jetbrains/python/psi/PyExpressionCodeFragment.java @@ -0,0 +1,5 @@ +package com.jetbrains.python.psi; + + +public interface PyExpressionCodeFragment extends PyFile { +} diff --git a/python/src/com/jetbrains/python/psi/impl/PyExpressionCodeFragmentImpl.java b/python/src/com/jetbrains/python/psi/impl/PyExpressionCodeFragmentImpl.java new file mode 100644 index 000000000000..3cd926d7ba95 --- /dev/null +++ b/python/src/com/jetbrains/python/psi/impl/PyExpressionCodeFragmentImpl.java @@ -0,0 +1,69 @@ +package com.jetbrains.python.psi.impl; + +import com.intellij.openapi.fileTypes.FileTypeManager; +import com.intellij.openapi.project.Project; +import com.intellij.psi.FileViewProvider; +import com.intellij.psi.PsiElement; +import com.intellij.psi.PsiManager; +import com.intellij.psi.SingleRootFileViewProvider; +import com.intellij.psi.impl.PsiManagerEx; +import com.intellij.psi.impl.file.impl.FileManager; +import com.intellij.psi.impl.source.tree.FileElement; +import com.intellij.testFramework.LightVirtualFile; +import com.jetbrains.python.psi.PyExpressionCodeFragment; +import org.jetbrains.annotations.NonNls; +import org.jetbrains.annotations.NotNull; + +/** + * clone of JSExpressionCodeFragment + */ +public class PyExpressionCodeFragmentImpl extends PyFileImpl implements PyExpressionCodeFragment { + + private PsiElement myContext; + private boolean myPhysical; + private FileViewProvider myViewProvider; + + public PyExpressionCodeFragmentImpl(Project project, @NonNls String name, CharSequence text, boolean isPhysical) { + super(((PsiManagerEx)PsiManager.getInstance(project)).getFileManager().createFileViewProvider( + new LightVirtualFile(name, FileTypeManager.getInstance().getFileTypeByFileName(name), text), isPhysical) + ); + myPhysical = isPhysical; + ((SingleRootFileViewProvider)getViewProvider()).forceCachedPsi(this); + } + + protected PyExpressionCodeFragmentImpl clone() { + final PyExpressionCodeFragmentImpl clone = (PyExpressionCodeFragmentImpl)cloneImpl((FileElement)calcTreeElement().clone()); + clone.myPhysical = false; + clone.myOriginalFile = this; + FileManager fileManager = ((PsiManagerEx)getManager()).getFileManager(); + SingleRootFileViewProvider cloneViewProvider = (SingleRootFileViewProvider)fileManager.createFileViewProvider(new LightVirtualFile(getName(), getLanguage(), getText()), false); + cloneViewProvider.forceCachedPsi(clone); + clone.myViewProvider = cloneViewProvider; + return clone; + } + + public PsiElement getContext() { + return myContext; + } + + @NotNull + public FileViewProvider getViewProvider() { + if(myViewProvider != null) return myViewProvider; + return super.getViewProvider(); + } + + public boolean isValid() { + if (!super.isValid()) return false; + if (myContext != null && !myContext.isValid()) return false; + return true; + } + + public boolean isPhysical() { + return myPhysical; + } + + public void setContext(PsiElement context) { + myContext = context; + } + +} diff --git a/python/src/com/jetbrains/python/run/CommandLinePatcher.java b/python/src/com/jetbrains/python/run/CommandLinePatcher.java new file mode 100644 index 000000000000..2a0a4e38976f --- /dev/null +++ b/python/src/com/jetbrains/python/run/CommandLinePatcher.java @@ -0,0 +1,10 @@ +package com.jetbrains.python.run; + +import com.intellij.execution.configurations.GeneralCommandLine; + +/** + * @author yole + */ +public interface CommandLinePatcher { + void patchCommandLine(GeneralCommandLine commandLine); +} diff --git a/python/src/com/jetbrains/python/run/PythonCommandLineState.java b/python/src/com/jetbrains/python/run/PythonCommandLineState.java index f7ed4387dcaf..25fcd37df105 100644 --- a/python/src/com/jetbrains/python/run/PythonCommandLineState.java +++ b/python/src/com/jetbrains/python/run/PythonCommandLineState.java @@ -40,7 +40,11 @@ public class PythonCommandLineState extends CommandLineState { @Override public ExecutionResult execute(@NotNull Executor executor, @NotNull ProgramRunner runner) throws ExecutionException { - final ProcessHandler processHandler = startProcess(); + return execute(null); + } + + public ExecutionResult execute(CommandLinePatcher patcher) throws ExecutionException { + final ProcessHandler processHandler = startProcess(patcher); final ConsoleView console = createAndAttachConsole(getConfig().getProject(), processHandler); return new DefaultExecutionResult(console, processHandler, createActions(console, processHandler)); @@ -59,7 +63,14 @@ public class PythonCommandLineState extends CommandLineState { } protected OSProcessHandler startProcess() throws ExecutionException { + return startProcess(null); + } + + protected OSProcessHandler startProcess(CommandLinePatcher patcher) throws ExecutionException { GeneralCommandLine commandLine = generateCommandLine(); + if (patcher != null) { + patcher.patchCommandLine(commandLine); + } final OSProcessHandler processHandler = new OSProcessHandler(commandLine.createProcess(), commandLine.getCommandLineString()); ProcessTerminatedListener.attach(processHandler); From 408bc8b2f36b2b85d3946c3b0c4b73a581318649 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Thu, 24 Dec 2009 16:52:34 +0300 Subject: [PATCH 06/32] convert test to PyLightFixtureTestCase --- .../com/jetbrains/python/PythonHighlightingTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/testSrc/com/jetbrains/python/PythonHighlightingTest.java b/python/testSrc/com/jetbrains/python/PythonHighlightingTest.java index ad372d90a314..6e0b5ea4c02b 100644 --- a/python/testSrc/com/jetbrains/python/PythonHighlightingTest.java +++ b/python/testSrc/com/jetbrains/python/PythonHighlightingTest.java @@ -1,11 +1,11 @@ package com.jetbrains.python; -import com.intellij.codeInsight.daemon.DaemonAnalyzerTestCase; import com.intellij.openapi.editor.colors.EditorColorsManager; import com.intellij.openapi.editor.colors.EditorColorsScheme; import com.intellij.openapi.editor.colors.TextAttributesKey; import com.intellij.openapi.editor.markup.EffectType; import com.intellij.openapi.editor.markup.TextAttributes; +import com.jetbrains.python.fixtures.PyLightFixtureTestCase; import org.jetbrains.annotations.NonNls; import java.awt.*; @@ -14,7 +14,7 @@ import java.awt.*; * Test highlighting added by annotators. * @author yole */ -public class PythonHighlightingTest extends DaemonAnalyzerTestCase { +public class PythonHighlightingTest extends PyLightFixtureTestCase { @NonNls protected String getTestDataPath() { return PythonTestUtil.getTestDataPath() + "/highlighting/"; @@ -101,11 +101,11 @@ public class PythonHighlightingTest extends DaemonAnalyzerTestCase { } private void doTest() throws Exception { - doTest(getTestName(true) + PyNames.DOT_PY, true, true); + myFixture.testHighlighting(true, true, false, getTestName(true) + PyNames.DOT_PY); } private void doTest(boolean checkWarnings, boolean checkInfos) throws Exception { - doTest(getTestName(true) + PyNames.DOT_PY, checkWarnings, checkInfos); + myFixture.testHighlighting(checkWarnings, checkInfos, false, getTestName(true) + PyNames.DOT_PY); } } From 7cc74acdcbf28592dbd03bfe41a295d495419a7b Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Thu, 24 Dec 2009 17:15:10 +0300 Subject: [PATCH 07/32] convert test to PyLightFixtureTestCase --- .../PythonDemorganLawIntentionTest.java | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/python/testSrc/com/jetbrains/python/PythonDemorganLawIntentionTest.java b/python/testSrc/com/jetbrains/python/PythonDemorganLawIntentionTest.java index ddd07222195c..0b9dbff4c9fd 100644 --- a/python/testSrc/com/jetbrains/python/PythonDemorganLawIntentionTest.java +++ b/python/testSrc/com/jetbrains/python/PythonDemorganLawIntentionTest.java @@ -4,21 +4,29 @@ */ package com.jetbrains.python; -import com.intellij.codeInsight.daemon.LightIntentionActionTestCase; +import com.intellij.codeInsight.intention.IntentionAction; +import com.jetbrains.python.fixtures.PyLightFixtureTestCase; -public class PythonDemorganLawIntentionTest extends LightIntentionActionTestCase { - public void test() throws Exception { doAllTests(); } - - protected String getBasePath() { - return "/demorgan"; +public class PythonDemorganLawIntentionTest extends PyLightFixtureTestCase { + public void test1() throws Exception { + doTest(); } + public void test2() throws Exception { + doTest(); + } + + public void test3() throws Exception { + doTest(); + } + + private void doTest() throws Exception { + myFixture.configureByFile("before" + getTestName(false) + ".py"); + final IntentionAction action = myFixture.findSingleIntention("DeMorgan Law"); + myFixture.launchAction(action); + + } protected String getTestDataPath() { - return PythonTestUtil.getTestDataPath() + "/intentions"; - } - - @Override - protected boolean shouldBeAvailableAfterExecution() { - return true; + return PythonTestUtil.getTestDataPath() + "/intentions/demorgan"; } } \ No newline at end of file From cca9d5cd41e3611dd1eecc14b41097935042ddff Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Thu, 24 Dec 2009 17:18:27 +0300 Subject: [PATCH 08/32] convert test to PyLightFixtureTestCase --- .../com/jetbrains/python/PythonCompletionTest.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/python/testSrc/com/jetbrains/python/PythonCompletionTest.java b/python/testSrc/com/jetbrains/python/PythonCompletionTest.java index 2a1316f66298..c135392734c3 100644 --- a/python/testSrc/com/jetbrains/python/PythonCompletionTest.java +++ b/python/testSrc/com/jetbrains/python/PythonCompletionTest.java @@ -4,17 +4,15 @@ */ package com.jetbrains.python; -import com.intellij.codeInsight.completion.CodeCompletionHandlerBase; -import com.intellij.codeInsight.completion.CompletionType; -import com.intellij.testFramework.LightCodeInsightTestCase; +import com.jetbrains.python.fixtures.PyLightFixtureTestCase; -public class PythonCompletionTest extends LightCodeInsightTestCase{ +public class PythonCompletionTest extends PyLightFixtureTestCase { private void doTest() throws Exception { final String testName = getTestName(true); - configureByFile(testName + ".py"); - new CodeCompletionHandlerBase(CompletionType.BASIC).invoke(getProject(), getEditor(), getFile()); - checkResultByFile(testName + ".after.py"); + myFixture.configureByFile(testName + ".py"); + myFixture.completeBasic(); + myFixture.checkResultByFile(testName + ".after.py"); } protected String getTestDataPath() { From 98f13556b3be3914a40b0452674993f33f00ca66 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Thu, 24 Dec 2009 17:29:15 +0300 Subject: [PATCH 09/32] convert test to PyLightFixtureTestCase --- .../com/jetbrains/python/PySelectWordTest.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/python/testSrc/com/jetbrains/python/PySelectWordTest.java b/python/testSrc/com/jetbrains/python/PySelectWordTest.java index 0a637ce2a032..8a2723eef3b3 100644 --- a/python/testSrc/com/jetbrains/python/PySelectWordTest.java +++ b/python/testSrc/com/jetbrains/python/PySelectWordTest.java @@ -2,7 +2,7 @@ package com.jetbrains.python; import com.intellij.codeInsight.editorActions.SelectWordHandler; import com.intellij.ide.DataManager; -import com.intellij.testFramework.LightCodeInsightTestCase; +import com.jetbrains.python.fixtures.PyLightFixtureTestCase; import org.jetbrains.annotations.NonNls; import java.io.File; @@ -10,33 +10,34 @@ import java.io.File; /** * @author yole */ -public class PySelectWordTest extends LightCodeInsightTestCase { +public class PySelectWordTest extends PyLightFixtureTestCase { public void testWord() throws Exception { doTest(); } private void doTest() throws Exception { + myFixture.copyDirectoryToProject("", ""); @NonNls final String path = getTestName(true); - configureByFile(path + "/before.py"); + myFixture.configureByFile(path + "/before.py"); int i = 1; while (true) { @NonNls String resultPath = path + "/after" + i + ".py"; if (new File(getTestDataPath() + resultPath).exists()) { performAction(); //System.out.println("comparing with "+resultPath); - checkResultByFile("Step " + i, resultPath, false); + myFixture.checkResultByFile(resultPath, false); i++; } else { break; } } - assertTrue(i>1); + assertTrue(i > 1); } private void performAction() { SelectWordHandler action = new SelectWordHandler(null); - action.execute(getEditor(), DataManager.getInstance().getDataContext()); + action.execute(myFixture.getEditor(), DataManager.getInstance().getDataContext()); } @Override From cf08a86367843eba8718b00c108662edec345ea7 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Thu, 24 Dec 2009 17:41:48 +0300 Subject: [PATCH 10/32] convert test to PyLightFixtureTestCase --- .../com/jetbrains/python/PyDecoratorTest.java | 4 +- .../python/fixtures/PyResolveTestCase.java | 43 +++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 python/testSrc/com/jetbrains/python/fixtures/PyResolveTestCase.java diff --git a/python/testSrc/com/jetbrains/python/PyDecoratorTest.java b/python/testSrc/com/jetbrains/python/PyDecoratorTest.java index 9d5dee76bde1..8c7dccf30559 100644 --- a/python/testSrc/com/jetbrains/python/PyDecoratorTest.java +++ b/python/testSrc/com/jetbrains/python/PyDecoratorTest.java @@ -2,7 +2,7 @@ package com.jetbrains.python; import com.intellij.psi.PsiElement; import com.intellij.psi.PsiReference; -import com.intellij.testFramework.ResolveTestCase; +import com.jetbrains.python.fixtures.PyResolveTestCase; import com.jetbrains.python.psi.PyArgumentList; import com.jetbrains.python.psi.PyDecorator; import com.jetbrains.python.psi.PyExpression; @@ -13,7 +13,7 @@ import com.jetbrains.python.psi.PyFunction; * User: dcheryasov * Date: Dec 28, 2008 3:50:23 AM */ -public class PyDecoratorTest extends ResolveTestCase { +public class PyDecoratorTest extends PyResolveTestCase { private PsiElement find() throws Exception { PsiReference ref = configureByFile(getTestName(false) + ".py"); return ref.getElement(); diff --git a/python/testSrc/com/jetbrains/python/fixtures/PyResolveTestCase.java b/python/testSrc/com/jetbrains/python/fixtures/PyResolveTestCase.java new file mode 100644 index 000000000000..4ceac0ac2e68 --- /dev/null +++ b/python/testSrc/com/jetbrains/python/fixtures/PyResolveTestCase.java @@ -0,0 +1,43 @@ +package com.jetbrains.python.fixtures; + +import com.intellij.openapi.application.ApplicationManager; +import com.intellij.openapi.util.text.StringUtil; +import com.intellij.openapi.vfs.LocalFileSystem; +import com.intellij.openapi.vfs.VfsUtil; +import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.psi.PsiReference; +import org.jetbrains.annotations.NonNls; + +import java.io.File; +import java.io.IOException; + +/** + * @author yole + */ +public abstract class PyResolveTestCase extends PyLightFixtureTestCase { + @NonNls protected static final String MARKER = ""; + + protected PsiReference configureByFile(final String filePath) throws Exception { + VirtualFile testDataRoot = LocalFileSystem.getInstance().refreshAndFindFileByIoFile(new File(getTestDataPath())); + final VirtualFile file = testDataRoot.findFileByRelativePath(filePath); + assertNotNull(file); + + String fileText = StringUtil.convertLineSeparators(VfsUtil.loadText(file)); + int offset = fileText.indexOf(MARKER); + assertTrue(offset >= 0); + fileText = fileText.substring(0, offset) + fileText.substring(offset + MARKER.length()); + final String finalFileText = fileText; + ApplicationManager.getApplication().runWriteAction(new Runnable() { + public void run() { + try { + myFixture.configureByText(new File(filePath).getName(), finalFileText); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + }); + final PsiReference reference = myFixture.getFile().findReferenceAt(offset); + return reference; + } +} From c47b89b90d61de932832f476abec2063e665f391 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Thu, 24 Dec 2009 17:46:16 +0300 Subject: [PATCH 11/32] convert test to PyLightFixtureTestCase --- python/testSrc/com/jetbrains/python/PyResolveTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/testSrc/com/jetbrains/python/PyResolveTest.java b/python/testSrc/com/jetbrains/python/PyResolveTest.java index 7330643f9338..8ec83a77600d 100644 --- a/python/testSrc/com/jetbrains/python/PyResolveTest.java +++ b/python/testSrc/com/jetbrains/python/PyResolveTest.java @@ -9,10 +9,10 @@ import com.intellij.psi.PsiPolyVariantReference; import com.intellij.psi.PsiReference; import com.intellij.psi.ResolveResult; import com.intellij.psi.util.PsiTreeUtil; -import com.intellij.testFramework.ResolveTestCase; +import com.jetbrains.python.fixtures.PyResolveTestCase; import com.jetbrains.python.psi.*; -public class PyResolveTest extends ResolveTestCase { +public class PyResolveTest extends PyResolveTestCase { private PsiElement resolve() throws Exception { PsiReference ref = configureByFile(getTestName(false) + ".py"); return ref.resolve(); From 41188793e21cfb9d45a25cbe7bb377d0432c5b24 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Thu, 24 Dec 2009 17:48:49 +0300 Subject: [PATCH 12/32] convert test to PyLightFixtureTestCase --- python/testSrc/com/jetbrains/python/PyResolveCalleeTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/testSrc/com/jetbrains/python/PyResolveCalleeTest.java b/python/testSrc/com/jetbrains/python/PyResolveCalleeTest.java index 3b655634b144..22e581b682ac 100644 --- a/python/testSrc/com/jetbrains/python/PyResolveCalleeTest.java +++ b/python/testSrc/com/jetbrains/python/PyResolveCalleeTest.java @@ -2,7 +2,7 @@ package com.jetbrains.python; import com.intellij.psi.PsiReference; import com.intellij.psi.util.PsiTreeUtil; -import com.intellij.testFramework.ResolveTestCase; +import com.jetbrains.python.fixtures.PyResolveTestCase; import com.jetbrains.python.psi.PyCallExpression; import com.jetbrains.python.psi.PyFunction; @@ -13,7 +13,7 @@ import java.util.EnumSet; * User: dcheryasov * Date: Aug 21, 2008 */ -public class PyResolveCalleeTest extends ResolveTestCase { +public class PyResolveCalleeTest extends PyResolveTestCase { private PyCallExpression.PyMarkedFunction resolveCallee() throws Exception { PsiReference ref = configureByFile(getTestName(false) + ".py"); From 4300cb604b6048998ca83c83a1a46866e62e68a2 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Thu, 24 Dec 2009 17:53:04 +0300 Subject: [PATCH 13/32] convert test to PyLightFixtureTestCase --- .../python/PyInheritorsSearchTest.java | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/python/testSrc/com/jetbrains/python/PyInheritorsSearchTest.java b/python/testSrc/com/jetbrains/python/PyInheritorsSearchTest.java index b0a26c5b2dbe..5667db6a6b2a 100644 --- a/python/testSrc/com/jetbrains/python/PyInheritorsSearchTest.java +++ b/python/testSrc/com/jetbrains/python/PyInheritorsSearchTest.java @@ -1,11 +1,10 @@ package com.jetbrains.python; -import com.intellij.codeInsight.CodeInsightTestCase; -import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.openapi.project.Project; import com.intellij.psi.PsiDocumentManager; import com.intellij.psi.search.ProjectScope; import com.intellij.psi.stubs.StubIndex; -import com.intellij.testFramework.PsiTestUtil; +import com.jetbrains.python.fixtures.PyLightFixtureTestCase; import com.jetbrains.python.psi.PyClass; import com.jetbrains.python.psi.search.PyClassInheritorsSearch; import com.jetbrains.python.psi.stubs.PyClassNameIndex; @@ -15,7 +14,7 @@ import java.util.Collection; /** * @author yole */ -public class PyInheritorsSearchTest extends CodeInsightTestCase { +public class PyInheritorsSearchTest extends PyLightFixtureTestCase { public void testSimple() throws Exception { setupProject(); final PyClass pyClass = findClass("A"); @@ -39,16 +38,20 @@ public class PyInheritorsSearchTest extends CodeInsightTestCase { private void setupProject() throws Exception { String testName = getTestName(true); - String root = PythonTestUtil.getTestDataPath() + "/inheritors/" + testName; - VirtualFile rootDir = PsiTestUtil.createTestProjectStructure(myProject, myModule, root, myFilesToDelete, false); - PsiTestUtil.addSourceContentToRoots(myModule, rootDir); - PsiDocumentManager.getInstance(myProject).commitAllDocuments(); + myFixture.copyDirectoryToProject(testName, ""); + PsiDocumentManager.getInstance(myFixture.getProject()).commitAllDocuments(); } private PyClass findClass(final String name) { - final Collection classes = StubIndex.getInstance().get(PyClassNameIndex.KEY, name, myProject, - ProjectScope.getProjectScope(myProject)); + final Project project = myFixture.getProject(); + final Collection classes = StubIndex.getInstance().get(PyClassNameIndex.KEY, name, project, + ProjectScope.getProjectScope(project)); assertEquals(1, classes.size()); return classes.iterator().next(); } + + @Override + protected String getTestDataPath() { + return PythonTestUtil.getTestDataPath() + "/inheritors/"; + } } From 5904980b7e7798081b90ccf53c06cb3f931d5ab7 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Thu, 24 Dec 2009 18:01:05 +0300 Subject: [PATCH 14/32] convert test to PyLightFixtureTestCase --- .../com/jetbrains/python/PyIndentTest.java | 260 ++++++++---------- 1 file changed, 108 insertions(+), 152 deletions(-) diff --git a/python/testSrc/com/jetbrains/python/PyIndentTest.java b/python/testSrc/com/jetbrains/python/PyIndentTest.java index 8ccb7210eac4..8408b6cd2936 100644 --- a/python/testSrc/com/jetbrains/python/PyIndentTest.java +++ b/python/testSrc/com/jetbrains/python/PyIndentTest.java @@ -1,185 +1,141 @@ package com.jetbrains.python; -import com.intellij.ide.DataManager; import com.intellij.idea.Bombed; import com.intellij.openapi.actionSystem.IdeActions; -import com.intellij.openapi.editor.actionSystem.EditorActionHandler; -import com.intellij.openapi.editor.actionSystem.EditorActionManager; -import com.intellij.testFramework.LightCodeInsightTestCase; +import com.intellij.openapi.application.ApplicationManager; +import com.intellij.openapi.command.CommandProcessor; +import com.jetbrains.python.fixtures.PyLightFixtureTestCase; +import java.io.IOException; import java.util.Calendar; /** * @author yole */ -public class PyIndentTest extends LightCodeInsightTestCase { - private void doTest(String before, String after) throws Exception { - final String name = getTestName(false); +public class PyIndentTest extends PyLightFixtureTestCase { + private void doTest(final String before, String after) throws Exception { + final String name = getTestName(false); - configureFromFileText(name + ".py", before); + ApplicationManager.getApplication().runWriteAction(new Runnable() { + public void run() { + try { + myFixture.configureByText(name + ".py", before); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + }); + CommandProcessor.getInstance().executeCommand(myFixture.getProject(), new Runnable() { + public void run() { + myFixture.performEditorAction(IdeActions.ACTION_EDITOR_ENTER); + } + }, "", null); + String s = myFixture.getFile().getText(); + myFixture.checkResult(after); + } - EditorActionManager actionManager = EditorActionManager.getInstance(); - EditorActionHandler actionHandler = actionManager.getActionHandler(IdeActions.ACTION_EDITOR_ENTER); + public void testSimpleIndent() throws Exception { + doTest("a=1", "a=1\n"); + } - actionHandler.execute(getEditor(), DataManager.getInstance().getDataContext()); + public void testIndentColon() throws Exception { + doTest("if a:", "if a:\n "); + } - String s = myFile.getText(); - checkResultByText(null, after, false); - } + public void testIndentStatementList() throws Exception { + doTest("if a:\n print a", "if a:\n \n print a"); + } - public void testSimpleIndent() throws Exception { - doTest("a=1", "a=1\n"); - } + public void testIndentStatementList2() throws Exception { + doTest("while a:\n print a", "while a:\n print a\n "); + } - public void testIndentColon() throws Exception { - doTest("if a:", "if a:\n "); - } + public void testIndentStatementList3() throws Exception { + doTest("if a:\n print a\n\nprint b", "if a:\n print a\n \n\nprint b"); + } - public void testIndentStatementList() throws Exception { - doTest("if a:\n print a", "if a:\n \n print a"); - } + public void testIndentOneLineStatementList() throws Exception { + doTest("if a:\n if b: print c\n print d", "if a:\n if b: print c\n \n print d"); + } - public void testIndentStatementList2() throws Exception { - doTest("while a:\n print a", "while a:\n print a\n "); - } + public void testIndentOneLineStatementListBreak() throws Exception { + doTest("if a:\n if b: print c\n print d", "if a:\n if b:\n print c\n print d"); + } - public void testIndentStatementList3() throws Exception { - doTest("if a:\n print a\n\nprint b", - "if a:\n print a\n \n\nprint b"); - } + @Bombed(month = Calendar.FEBRUARY, day = 17, user = "yole") + public void testAlignInList() throws Exception { + doTest("__all__ = [a,", "__all__ = [a,\n" + " "); + } - public void testIndentOneLineStatementList() throws Exception { - doTest("if a:\n if b: print c\n print d", - "if a:\n if b: print c\n \n print d"); - } + public void testAlignInListMiddle() throws Exception { + doTest("__all__ = [a,\n" + " c]", "__all__ = [a,\n" + " \n" + " c]"); + } - public void testIndentOneLineStatementListBreak() throws Exception { - doTest("if a:\n if b: print c\n print d", - "if a:\n if b:\n print c\n print d"); - } + public void testAlignInListMiddle2() throws Exception { + doTest("__all__ = [a,\n" + " b,\n" + " c]", + "__all__ = [a,\n" + " b,\n" + " \n" + " c]"); + } - @Bombed(month = Calendar.FEBRUARY, day = 17, user="yole") - public void testAlignInList() throws Exception { - doTest("__all__ = [a,", - "__all__ = [a,\n" + - " "); - } + @Bombed(month = Calendar.FEBRUARY, day = 17, user = "yole") + public void testAlignInListComp() throws Exception { + doTest("__all__ = [a for", "__all__ = [a for\n" + " "); + } - public void testAlignInListMiddle() throws Exception { - doTest("__all__ = [a,\n" + - " c]", - "__all__ = [a,\n" + - " \n" + - " c]"); - } + public void testClass() throws Exception { + doTest("class A:\n" + " print a", "class A:\n" + " print a\n" + " "); + } - public void testAlignInListMiddle2() throws Exception { - doTest("__all__ = [a,\n" + - " b,\n" + - " c]", - "__all__ = [a,\n" + - " b,\n" + - " \n" + - " c]"); - } - @Bombed(month = Calendar.FEBRUARY, day = 17, user="yole") - public void testAlignInListComp() throws Exception { - doTest("__all__ = [a for", - "__all__ = [a for\n" + - " "); - } + public void testClass2() throws Exception { + doTest("class CombatExpertiseFeat(Ability):\n" + " if a: print b\n" + " def getAvailableActions(self):", + "class CombatExpertiseFeat(Ability):\n" + " if a: print b\n" + " def getAvailableActions(self):\n" + " "); + } - public void testClass() throws Exception { - doTest("class A:\n" + - " print a", - "class A:\n" + - " print a\n" + - " "); - } + public void testClass2_1() throws Exception { + doTest( + "class CombatExpertiseFeat(Ability):\n" + " if a: print b\n" + " def getAvailableActions(self):\n" + "class C2: pass", - public void testClass2() throws Exception { - doTest("class CombatExpertiseFeat(Ability):\n" + - " if a: print b\n" + - " def getAvailableActions(self):", - "class CombatExpertiseFeat(Ability):\n" + - " if a: print b\n" + - " def getAvailableActions(self):\n" + - " "); - } + "class CombatExpertiseFeat(Ability):\n" + + " if a: print b\n" + + " def getAvailableActions(self):\n" + + " \n" + + "class C2: pass"); + } - public void testClass2_1() throws Exception { - doTest("class CombatExpertiseFeat(Ability):\n" + - " if a: print b\n" + - " def getAvailableActions(self):\n" + - "class C2: pass", + public void testMultiDedent() throws Exception { + doTest("class CombatExpertiseFeat(Ability):\n" + " def getAvailableActions(self):\n" + " result = ArrayList()", + "class CombatExpertiseFeat(Ability):\n" + + " def getAvailableActions(self):\n" + + " result = ArrayList()\n" + + " "); + } - "class CombatExpertiseFeat(Ability):\n" + - " if a: print b\n" + - " def getAvailableActions(self):\n" + - " \n" + - "class C2: pass"); - } + public void testMultiDedent1() throws Exception { + doTest("class CombatExpertiseFeat(Ability):\n" + " def getAvailableActions(self):\n" + " if a:", + "class CombatExpertiseFeat(Ability):\n" + " def getAvailableActions(self):\n" + " if a:\n" + " "); + } - public void testMultiDedent() throws Exception { - doTest("class CombatExpertiseFeat(Ability):\n" + - " def getAvailableActions(self):\n" + - " result = ArrayList()", - "class CombatExpertiseFeat(Ability):\n" + - " def getAvailableActions(self):\n" + - " result = ArrayList()\n" + - " "); - } + public void testMultiDedent2() throws Exception { + doTest("class CombatExpertiseFeat(Ability):\n" + " def getAvailableActions(self): result = ArrayList()", + "class CombatExpertiseFeat(Ability):\n" + " def getAvailableActions(self): result = ArrayList()\n" + " "); + } - public void testMultiDedent1() throws Exception { - doTest("class CombatExpertiseFeat(Ability):\n" + - " def getAvailableActions(self):\n" + - " if a:", - "class CombatExpertiseFeat(Ability):\n" + - " def getAvailableActions(self):\n" + - " if a:\n" + - " "); - } + public void testIfElse() throws Exception { + doTest("if a:\n" + " b\n" + "else:\n" + " c", "if a:\n" + " \n" + " b\n" + "else:\n" + " c"); + } - public void testMultiDedent2() throws Exception { - doTest("class CombatExpertiseFeat(Ability):\n" + - " def getAvailableActions(self): result = ArrayList()", - "class CombatExpertiseFeat(Ability):\n" + - " def getAvailableActions(self): result = ArrayList()\n" + - " "); - } + public void testIfElse2() throws Exception { + doTest("if a:\n" + " b\n" + "else:\n" + " c", "if a:\n" + " b\n" + "else:\n" + " \n" + " c"); + } - public void testIfElse() throws Exception { - doTest("if a:\n" + - " b\n" + - "else:\n" + - " c", - "if a:\n" + - " \n" + - " b\n" + - "else:\n" + - " c"); - } - - public void testIfElse2() throws Exception { - doTest("if a:\n" + - " b\n" + - "else:\n" + - " c", - "if a:\n" + - " b\n" + - "else:\n" + - " \n" + - " c"); - } - - /* - TODO: formatter core problem? - public void testAlignListBeforeEquals() throws Exception { - doTest("__all__ = [a,\n" + - " b]", - "__all__ \n" + - "= [a,\n" + - " b]"); - } - */ + /* + TODO: formatter core problem? + public void testAlignListBeforeEquals() throws Exception { + doTest("__all__ = [a,\n" + + " b]", + "__all__ \n" + + "= [a,\n" + + " b]"); + } + */ } From 85894d26372c126ff6222981b4415ce7370b66be Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Thu, 24 Dec 2009 18:05:07 +0300 Subject: [PATCH 15/32] convert test to PyLightFixtureTestCase --- .../python/PyMultiFileResolveTest.java | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/python/testSrc/com/jetbrains/python/PyMultiFileResolveTest.java b/python/testSrc/com/jetbrains/python/PyMultiFileResolveTest.java index 67b4dc04830b..42927b7d9975 100644 --- a/python/testSrc/com/jetbrains/python/PyMultiFileResolveTest.java +++ b/python/testSrc/com/jetbrains/python/PyMultiFileResolveTest.java @@ -1,16 +1,15 @@ package com.jetbrains.python; -import com.intellij.codeInsight.CodeInsightTestCase; import com.intellij.openapi.editor.Document; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.psi.*; -import com.intellij.testFramework.PsiTestUtil; +import com.jetbrains.python.fixtures.PyLightFixtureTestCase; import com.jetbrains.python.psi.*; /** * @author yole */ -public class PyMultiFileResolveTest extends CodeInsightTestCase { +public class PyMultiFileResolveTest extends PyLightFixtureTestCase { public void testSimple() throws Exception { PsiElement element = doResolve(); assertTrue(element instanceof PyFile); @@ -111,17 +110,20 @@ public class PyMultiFileResolveTest extends CodeInsightTestCase { private PsiFile prepareFile() throws Exception { String testName = getTestName(true); String fileName = getTestName(false) + ".py"; - String root = PythonTestUtil.getTestDataPath() + "/resolve/multiFile/" + testName; - VirtualFile rootDir = PsiTestUtil.createTestProjectStructure(myProject, myModule, root, myFilesToDelete, false); - PsiTestUtil.addSourceContentToRoots(myModule, rootDir); - PsiDocumentManager.getInstance(myProject).commitAllDocuments(); + myFixture.copyDirectoryToProject(testName, ""); + PsiDocumentManager.getInstance(myFixture.getProject()).commitAllDocuments(); - VirtualFile sourceFile = rootDir.findChild(fileName); + VirtualFile sourceFile = myFixture.findFileInTempDir(fileName); assert sourceFile != null; - PsiFile psiFile = myPsiManager.findFile(sourceFile); + PsiFile psiFile = myFixture.getPsiManager().findFile(sourceFile); return psiFile; } + @Override + protected String getTestDataPath() { + return PythonTestUtil.getTestDataPath() + "/resolve/multiFile/"; + } + private PsiElement doResolve() throws Exception { PsiFile psiFile = prepareFile(); int offset = findMarkerOffset(psiFile); @@ -137,7 +139,7 @@ public class PyMultiFileResolveTest extends CodeInsightTestCase { } private int findMarkerOffset(final PsiFile psiFile) { - Document document = PsiDocumentManager.getInstance(myProject).getDocument(psiFile); + Document document = PsiDocumentManager.getInstance(myFixture.getProject()).getDocument(psiFile); assert document != null; int offset = -1; for (int i=1; i Date: Thu, 24 Dec 2009 18:11:32 +0300 Subject: [PATCH 16/32] convert test to PyLightFixtureTestCase --- .../python/PyAssignmentMappingTest.java | 6 +- .../python/fixtures/LightMarkedTestCase.java | 114 ++++++++++++++++++ 2 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 python/testSrc/com/jetbrains/python/fixtures/LightMarkedTestCase.java diff --git a/python/testSrc/com/jetbrains/python/PyAssignmentMappingTest.java b/python/testSrc/com/jetbrains/python/PyAssignmentMappingTest.java index 3b149567039b..22602c3b8fa1 100644 --- a/python/testSrc/com/jetbrains/python/PyAssignmentMappingTest.java +++ b/python/testSrc/com/jetbrains/python/PyAssignmentMappingTest.java @@ -2,10 +2,12 @@ package com.jetbrains.python; import com.intellij.openapi.util.Pair; import com.intellij.psi.PsiElement; +import com.jetbrains.python.fixtures.LightMarkedTestCase; import com.jetbrains.python.psi.PyAssignmentStatement; import com.jetbrains.python.psi.PyExpression; -import com.jetbrains.python.psi.PyTargetExpression; import com.jetbrains.python.psi.PySubscriptionExpression; +import com.jetbrains.python.psi.PyTargetExpression; + import java.util.List; import java.util.Map; @@ -14,7 +16,7 @@ import java.util.Map; * User: dcheryasov * Date: Dec 11, 2009 2:13:51 AM */ -public class PyAssignmentMappingTest extends MarkedTestCase { +public class PyAssignmentMappingTest extends LightMarkedTestCase { public String getTestDataPath() { return PythonTestUtil.getTestDataPath() + "/psi/assignment/"; diff --git a/python/testSrc/com/jetbrains/python/fixtures/LightMarkedTestCase.java b/python/testSrc/com/jetbrains/python/fixtures/LightMarkedTestCase.java new file mode 100644 index 000000000000..5da87975466c --- /dev/null +++ b/python/testSrc/com/jetbrains/python/fixtures/LightMarkedTestCase.java @@ -0,0 +1,114 @@ +package com.jetbrains.python.fixtures; + +import com.intellij.openapi.application.ApplicationManager; +import com.intellij.openapi.util.text.StringUtil; +import com.intellij.openapi.vfs.LocalFileSystem; +import com.intellij.openapi.vfs.VfsUtil; +import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.psi.PsiElement; +import com.intellij.psi.PsiFile; +import com.intellij.util.containers.HashMap; +import org.jetbrains.annotations.NonNls; + +import java.io.File; +import java.io.IOException; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Base for cases that need marked PSI elements. + * User: dcheryasov + * Date: Mar 14, 2009 11:57:52 PM + */ +public abstract class LightMarkedTestCase extends PyLightFixtureTestCase { + protected PsiFile myFile; + + /** + * Marker "as expected", any alphanumeric sting in angle brackets. + */ + public final @NonNls String MARKER = "<[a-zA-Z0-9_]+>"; + + /** + * Uses MARKER as regexp. + * @see #configureByFileText(String, String, String) + * @param filePath file to load and parse + * @return a mapping of markers to PSI elements + * @throws Exception + */ + protected Map configureByFile(@NonNls String filePath) throws Exception { + return configureByFile(filePath, MARKER); + } + + /** + * Like configureByFileText, but with a file to be read. + * @param filePath file to read and parse + * @param markerRegexp regexp for markers + * @return a mapping of markers to PSI elements + * @throws Exception + */ + protected Map configureByFile(@NonNls String filePath, @NonNls String markerRegexp) + throws Exception + { + final String fullPath = getTestDataPath() + filePath; + final VirtualFile vFile = LocalFileSystem.getInstance().findFileByPath(fullPath.replace(File.separatorChar, '/')); + assertNotNull("file " + filePath + " not found", vFile); + + String fileText = StringUtil.convertLineSeparators(VfsUtil.loadText(vFile), "\n"); + + final String fileName = vFile.getName(); + + return configureByFileText(fileText, fileName, markerRegexp); + } + + /** + * Typically a text is marked with patterns: "foo <ref1>bar() + <ref2>baz", etc, and the result is + * a map where strings "<ref1>" and "<ref2>" are mapped to PSI elements for "bar" and "baz". + * @param fileText text to parse + * @param fileName name to give to the PSI file + * @param markerRegexp regexp to detect markers in the text + * @return mapping of markers to the PSI elements + * @throws Exception + */ + protected Map configureByFileText(String fileText, final String fileName, @NonNls String markerRegexp) + throws Exception + { + // build a map of marks to positions, and the text with marks stripped + Pattern pat = Pattern.compile(markerRegexp); + Matcher mat = pat.matcher(fileText); + int rest_index = 0; // from here on fileText is not yet looked at + Map offsets = new HashMap(); + final StringBuffer text = new StringBuffer(); + while (mat.find(rest_index)) { + String mark = mat.group(); + CharSequence prev_part = fileText.subSequence(rest_index, mat.start()); + text.append(prev_part); + offsets.put(mark, text.length()); + rest_index = mat.end(); + } + if (rest_index < fileText.length()) text.append(fileText.substring(rest_index)); + + // create a file and map marks to PSI elements + Map result = new HashMap(); + ApplicationManager.getApplication().runWriteAction(new Runnable() { + public void run() { + try { + myFile = myFixture.configureByText(fileName, text.toString()); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + }); for (Map.Entry entry : offsets.entrySet()) { + result.put(entry.getKey(), myFile.findElementAt(entry.getValue())); + } + return result; + } + + protected Map loadTest() throws Exception { + String fname = getTestName(false) + ".py"; + return configureByFile(fname); + } + + protected abstract String getTestDataPath(); +} \ No newline at end of file From 544b3292c28af8e23824a5adacaa3c928b2cf332 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Thu, 24 Dec 2009 18:14:28 +0300 Subject: [PATCH 17/32] convert test to PyLightFixtureTestCase --- .../testSrc/com/jetbrains/python/PyParameterInfoTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/testSrc/com/jetbrains/python/PyParameterInfoTest.java b/python/testSrc/com/jetbrains/python/PyParameterInfoTest.java index 244ddf8c2cbe..cf8eccc44507 100644 --- a/python/testSrc/com/jetbrains/python/PyParameterInfoTest.java +++ b/python/testSrc/com/jetbrains/python/PyParameterInfoTest.java @@ -4,12 +4,12 @@ import com.intellij.lang.parameterInfo.CreateParameterInfoContext; import com.intellij.lang.parameterInfo.ParameterInfoHandler; import com.intellij.lang.parameterInfo.ParameterInfoUIContextEx; import com.intellij.lang.parameterInfo.UpdateParameterInfoContext; -import com.intellij.openapi.application.PathManager; import com.intellij.openapi.editor.Editor; import com.intellij.openapi.project.Project; import com.intellij.psi.PsiElement; import com.intellij.psi.PsiFile; import com.intellij.util.containers.HashSet; +import com.jetbrains.python.fixtures.LightMarkedTestCase; import com.jetbrains.python.psi.PyArgumentList; import com.jetbrains.python.psi.PyUtil; import org.jetbrains.annotations.NotNull; @@ -25,7 +25,7 @@ import java.util.Set; *
User: dcheryasov * Date: Jul 14, 2009 3:42:44 AM */ -public class PyParameterInfoTest extends MarkedTestCase { +public class PyParameterInfoTest extends LightMarkedTestCase { protected String getTestDataPath() { return PythonTestUtil.getTestDataPath()+ "/paramInfo/"; } @@ -247,7 +247,7 @@ public class PyParameterInfoTest extends MarkedTestCase { * @throws Exception if it fails */ private Collector feignCtrlP(int offset) throws Exception { - Collector collector = new Collector(getProject(), getFile(), offset); + Collector collector = new Collector(myFixture.getProject(), myFixture.getFile(), offset); PyParameterInfoHandler handler = new PyParameterInfoHandler(); collector.setParameterOwner(handler.findElementForParameterInfo(collector)); // finds arglist, sets items to show if (collector.getParameterOwner() != null) { From 882e334c847a1e9b075e93ce2fbc3b24fd231552 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Thu, 24 Dec 2009 18:26:52 +0300 Subject: [PATCH 18/32] convert test to PyLightFixtureTestCase --- python/testSrc/com/jetbrains/python/PyQuickDocTest.java | 3 ++- .../com/jetbrains/python/fixtures/LightMarkedTestCase.java | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/python/testSrc/com/jetbrains/python/PyQuickDocTest.java b/python/testSrc/com/jetbrains/python/PyQuickDocTest.java index e0084b6cb17e..511ae3acce9a 100644 --- a/python/testSrc/com/jetbrains/python/PyQuickDocTest.java +++ b/python/testSrc/com/jetbrains/python/PyQuickDocTest.java @@ -5,6 +5,7 @@ import com.intellij.openapi.vfs.LocalFileSystem; import com.intellij.openapi.vfs.VfsUtil; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.psi.PsiElement; +import com.jetbrains.python.fixtures.LightMarkedTestCase; import com.jetbrains.python.psi.PyDocStringOwner; import com.jetbrains.python.psi.PyReferenceExpression; import com.jetbrains.python.psi.PyStringLiteralExpression; @@ -17,7 +18,7 @@ import java.util.Map; * User: dcheryasov * Date: Jun 7, 2009 12:31:07 PM */ -public class PyQuickDocTest extends MarkedTestCase { +public class PyQuickDocTest extends LightMarkedTestCase { private PythonDocumentationProvider myProvider; @Override diff --git a/python/testSrc/com/jetbrains/python/fixtures/LightMarkedTestCase.java b/python/testSrc/com/jetbrains/python/fixtures/LightMarkedTestCase.java index 5da87975466c..888b51e6bf5b 100644 --- a/python/testSrc/com/jetbrains/python/fixtures/LightMarkedTestCase.java +++ b/python/testSrc/com/jetbrains/python/fixtures/LightMarkedTestCase.java @@ -93,13 +93,15 @@ public abstract class LightMarkedTestCase extends PyLightFixtureTestCase { ApplicationManager.getApplication().runWriteAction(new Runnable() { public void run() { try { - myFile = myFixture.configureByText(fileName, text.toString()); + myFile = myFixture.addFileToProject(fileName, text.toString()); + myFixture.configureFromExistingVirtualFile(myFile.getVirtualFile()); } catch (IOException e) { throw new RuntimeException(e); } } - }); for (Map.Entry entry : offsets.entrySet()) { + }); + for (Map.Entry entry : offsets.entrySet()) { result.put(entry.getKey(), myFile.findElementAt(entry.getValue())); } return result; From 8ed733e7b4d0adb44c66de3475df93a8b9fc017c Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Thu, 24 Dec 2009 18:30:40 +0300 Subject: [PATCH 19/32] convert test to PyLightFixtureTestCase --- .../com/jetbrains/python/MarkedTestCase.java | 103 ------------------ .../python/PyStatementPartsTest.java | 3 +- 2 files changed, 2 insertions(+), 104 deletions(-) delete mode 100644 python/testSrc/com/jetbrains/python/MarkedTestCase.java diff --git a/python/testSrc/com/jetbrains/python/MarkedTestCase.java b/python/testSrc/com/jetbrains/python/MarkedTestCase.java deleted file mode 100644 index 65ef04e16820..000000000000 --- a/python/testSrc/com/jetbrains/python/MarkedTestCase.java +++ /dev/null @@ -1,103 +0,0 @@ -package com.jetbrains.python; - -import com.intellij.openapi.util.text.StringUtil; -import com.intellij.openapi.vfs.LocalFileSystem; -import com.intellij.openapi.vfs.VfsUtil; -import com.intellij.openapi.vfs.VirtualFile; -import com.intellij.psi.PsiElement; -import com.intellij.testFramework.PsiTestCase; -import com.intellij.util.containers.HashMap; -import org.jetbrains.annotations.NonNls; - -import java.io.File; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Base for cases that need marked PSI elements. - * User: dcheryasov - * Date: Mar 14, 2009 11:57:52 PM - */ -public abstract class MarkedTestCase extends PsiTestCase { - - /** - * Marker "as expected", any alphanumeric sting in angle brackets. - */ - public final @NonNls String MARKER = "<[a-zA-Z0-9_]+>"; - - /** - * Uses MARKER as regexp. - * @see #configureByFileText(String, String, String) - * @param filePath file to load and parse - * @return a mapping of markers to PSI elements - * @throws Exception - */ - protected Map configureByFile(@NonNls String filePath) throws Exception { - return configureByFile(filePath, MARKER); - } - - /** - * Like configureByFileText, but with a file to be read. - * @param filePath file to read and parse - * @param markerRegexp regexp for markers - * @return a mapping of markers to PSI elements - * @throws Exception - */ - protected Map configureByFile(@NonNls String filePath, @NonNls String markerRegexp) - throws Exception - { - final String fullPath = getTestDataPath() + filePath; - final VirtualFile vFile = LocalFileSystem.getInstance().findFileByPath(fullPath.replace(File.separatorChar, '/')); - assertNotNull("file " + filePath + " not found", vFile); - - String fileText = StringUtil.convertLineSeparators(VfsUtil.loadText(vFile), "\n"); - - final String fileName = vFile.getName(); - - return configureByFileText(fileText, fileName, markerRegexp); - } - - /** - * Typically a text is marked with patterns: "foo <ref1>bar() + <ref2>baz", etc, and the result is - * a map where strings "<ref1>" and "<ref2>" are mapped to PSI elements for "bar" and "baz". - * @param fileText text to parse - * @param fileName name to give to the PSI file - * @param markerRegexp regexp to detect markers in the text - * @return mapping of markers to the PSI elements - * @throws Exception - */ - protected Map configureByFileText(String fileText, String fileName, @NonNls String markerRegexp) - throws Exception - { - // build a map of marks to positions, and the text with marks stripped - Pattern pat = Pattern.compile(markerRegexp); - Matcher mat = pat.matcher(fileText); - int rest_index = 0; // from here on fileText is not yet looked at - Map offsets = new HashMap(); - StringBuffer text = new StringBuffer(); - while (mat.find(rest_index)) { - String mark = mat.group(); - CharSequence prev_part = fileText.subSequence(rest_index, mat.start()); - text.append(prev_part); - offsets.put(mark, text.length()); - rest_index = mat.end(); - } - if (rest_index < fileText.length()) text.append(fileText.substring(rest_index)); - - // create a file and map marks to PSI elements - Map result = new HashMap(); - myFile = createFile(myModule, fileName, text.toString()); - for (Map.Entry entry : offsets.entrySet()) { - result.put(entry.getKey(), myFile.findElementAt(entry.getValue())); - } - return result; - } - - protected Map loadTest() throws Exception { - String fname = getTestName(false) + ".py"; - return configureByFile(fname); - } - - protected abstract String getTestDataPath(); -} diff --git a/python/testSrc/com/jetbrains/python/PyStatementPartsTest.java b/python/testSrc/com/jetbrains/python/PyStatementPartsTest.java index ad410c7ea87b..3a0fc3ab0ba8 100644 --- a/python/testSrc/com/jetbrains/python/PyStatementPartsTest.java +++ b/python/testSrc/com/jetbrains/python/PyStatementPartsTest.java @@ -1,6 +1,7 @@ package com.jetbrains.python; import com.intellij.psi.PsiElement; +import com.jetbrains.python.fixtures.LightMarkedTestCase; import com.jetbrains.python.psi.*; import java.util.Map; @@ -10,7 +11,7 @@ import java.util.Map; * User: dcheryasov * Date: Mar 15, 2009 3:11:01 AM */ -public class PyStatementPartsTest extends MarkedTestCase { +public class PyStatementPartsTest extends LightMarkedTestCase { protected String getTestDataPath() { return PythonTestUtil.getTestDataPath() + "/psi/parts/"; From c6ab7f76f3cd0dd12b633eaabded5367b11b35a3 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Thu, 24 Dec 2009 19:44:45 +0300 Subject: [PATCH 20/32] convert test to PyLightFixtureTestCase --- .../PyUnresolvedReferencesInspection.java | 8 +- .../inspections/AddFieldAddConstructor.py | 2 +- .../inspections/AddFieldFromInstance.py | 2 +- .../inspections/AddFieldFromMethod.py | 2 +- python/testData/inspections/AddImportDoc.py | 2 +- .../inspections/AddMethodFromInstance.py | 2 +- .../inspections/AddMethodFromMethod.py | 2 +- python/testData/inspections/AddSelf.py | 2 +- .../inspections/AddToImportFromList.py | 2 +- .../testData/inspections/QualifyByImport.py | 2 +- .../inspections/RemoveTrailingSemicolon.py | 2 +- python/testData/inspections/RenameToSelf.py | 2 +- .../com/jetbrains/python/PyQuickFixTest.java | 115 ++++++------------ 13 files changed, 52 insertions(+), 93 deletions(-) diff --git a/python/src/com/jetbrains/python/inspections/PyUnresolvedReferencesInspection.java b/python/src/com/jetbrains/python/inspections/PyUnresolvedReferencesInspection.java index 13a3c9d776ac..957354d67fed 100644 --- a/python/src/com/jetbrains/python/inspections/PyUnresolvedReferencesInspection.java +++ b/python/src/com/jetbrains/python/inspections/PyUnresolvedReferencesInspection.java @@ -87,10 +87,10 @@ public class PyUnresolvedReferencesInspection extends LocalInspectionTool { } @NotNull - static List proposeImportFixes(final PyElement node, String ref_text) { + static Collection proposeImportFixes(final PyElement node, String ref_text) { PsiFile exisitng_import_file = null; // if there's a matching existing import, this it the file it imports ImportFromExistingFix fix = null; - List fixes = new ArrayList(2); + Collection fixes = new HashSet(2); Set seen_file_names = new HashSet(); // true import names // maybe the name is importable via some existing 'import foo' statement, and only needs a qualifier. // walk up collecting all such statements and analyzing @@ -279,10 +279,10 @@ public class PyUnresolvedReferencesInspection extends LocalInspectionTool { } // look in other imported modules for this whole name if (! ref_in_import) { - List import_fixes = proposeImportFixes(node, ref_text); + Collection import_fixes = proposeImportFixes(node, ref_text); if (import_fixes.size() > 0) { actions.addAll(import_fixes); - Object first_action = import_fixes.get(0); + Object first_action = import_fixes.iterator().next(); if (first_action instanceof HintAction) { hint_action = ((HintAction)first_action); } diff --git a/python/testData/inspections/AddFieldAddConstructor.py b/python/testData/inspections/AddFieldAddConstructor.py index d68c345ea44c..142be452fa33 100644 --- a/python/testData/inspections/AddFieldAddConstructor.py +++ b/python/testData/inspections/AddFieldAddConstructor.py @@ -5,4 +5,4 @@ class A: class B(A): def foo(self): - return self.x + return self.x diff --git a/python/testData/inspections/AddFieldFromInstance.py b/python/testData/inspections/AddFieldFromInstance.py index 3d1290dd4379..85f7724d6b0c 100644 --- a/python/testData/inspections/AddFieldFromInstance.py +++ b/python/testData/inspections/AddFieldFromInstance.py @@ -4,4 +4,4 @@ class A: a = A() -a.y+1 +a.y+1 diff --git a/python/testData/inspections/AddFieldFromMethod.py b/python/testData/inspections/AddFieldFromMethod.py index 897bd9b0383d..6680c07a9282 100644 --- a/python/testData/inspections/AddFieldFromMethod.py +++ b/python/testData/inspections/AddFieldFromMethod.py @@ -3,4 +3,4 @@ class A: self.x = 1 def foo(self): - a = self.y + a = self.y diff --git a/python/testData/inspections/AddImportDoc.py b/python/testData/inspections/AddImportDoc.py index e8b8a9fa6e83..21eacb04e0da 100644 --- a/python/testData/inspections/AddImportDoc.py +++ b/python/testData/inspections/AddImportDoc.py @@ -1,4 +1,4 @@ "Doc comment stays intact" -AddImportDoc.foo +AddImportDoc.foo foo = 1 diff --git a/python/testData/inspections/AddMethodFromInstance.py b/python/testData/inspections/AddMethodFromInstance.py index e114cf9f5666..07024b2fa6e5 100644 --- a/python/testData/inspections/AddMethodFromInstance.py +++ b/python/testData/inspections/AddMethodFromInstance.py @@ -4,4 +4,4 @@ class A: a = A() -a.y() +a.y() diff --git a/python/testData/inspections/AddMethodFromMethod.py b/python/testData/inspections/AddMethodFromMethod.py index bee15e3c1190..84cb7101c4f0 100644 --- a/python/testData/inspections/AddMethodFromMethod.py +++ b/python/testData/inspections/AddMethodFromMethod.py @@ -3,4 +3,4 @@ class A: self.x = 1 def foo(self, a): - self.y(1, a) + self.y(1, a) diff --git a/python/testData/inspections/AddSelf.py b/python/testData/inspections/AddSelf.py index 7affecd6841e..06916a26f40c 100644 --- a/python/testData/inspections/AddSelf.py +++ b/python/testData/inspections/AddSelf.py @@ -1,3 +1,3 @@ class A: - def foo(): # Add 'self' + def foo(): # Add 'self' pass diff --git a/python/testData/inspections/AddToImportFromList.py b/python/testData/inspections/AddToImportFromList.py index 0e908235b701..373ff05ff3af 100644 --- a/python/testData/inspections/AddToImportFromList.py +++ b/python/testData/inspections/AddToImportFromList.py @@ -1,3 +1,3 @@ from AddToImportFromFoo import bar -foo # must get imported +foo # must get imported diff --git a/python/testData/inspections/QualifyByImport.py b/python/testData/inspections/QualifyByImport.py index 5b3905668ac5..70e969c8998f 100644 --- a/python/testData/inspections/QualifyByImport.py +++ b/python/testData/inspections/QualifyByImport.py @@ -1,3 +1,3 @@ import QualifyByImportFoo -foo # must be qualified +foo # must be qualified diff --git a/python/testData/inspections/RemoveTrailingSemicolon.py b/python/testData/inspections/RemoveTrailingSemicolon.py index 464e5991ca8a..5dc39b9dc2bb 100644 --- a/python/testData/inspections/RemoveTrailingSemicolon.py +++ b/python/testData/inspections/RemoveTrailingSemicolon.py @@ -1 +1 @@ -a = 4; b = 5; \ No newline at end of file +a = 4; b = 5; \ No newline at end of file diff --git a/python/testData/inspections/RenameToSelf.py b/python/testData/inspections/RenameToSelf.py index b8fed41e3a2f..e4a2a2ab4b08 100644 --- a/python/testData/inspections/RenameToSelf.py +++ b/python/testData/inspections/RenameToSelf.py @@ -1,3 +1,3 @@ class A: - def foo(slf): # Rename to 'self' + def foo(slf): # Rename to 'self' pass diff --git a/python/testSrc/com/jetbrains/python/PyQuickFixTest.java b/python/testSrc/com/jetbrains/python/PyQuickFixTest.java index 0e5d5ec77b25..fd44fa387ce6 100644 --- a/python/testSrc/com/jetbrains/python/PyQuickFixTest.java +++ b/python/testSrc/com/jetbrains/python/PyQuickFixTest.java @@ -1,30 +1,26 @@ package com.jetbrains.python; -import com.intellij.codeInsight.daemon.DaemonAnalyzerTestCase; -import com.intellij.codeInsight.daemon.impl.HighlightInfo; -import com.intellij.codeInsight.daemon.quickFix.LightQuickFixTestCase; import com.intellij.codeInsight.intention.IntentionAction; -import com.intellij.codeInspection.LocalInspectionTool; -import com.intellij.openapi.editor.markup.GutterIconRenderer; -import com.intellij.openapi.util.Pair; -import com.intellij.openapi.util.TextRange; -import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.openapi.application.PathManager; +import com.intellij.openapi.projectRoots.Sdk; +import com.intellij.openapi.projectRoots.SdkModificator; +import com.intellij.openapi.projectRoots.impl.ProjectJdkImpl; +import com.intellij.testFramework.LightProjectDescriptor; +import com.jetbrains.python.fixtures.PyLightFixtureTestCase; import com.jetbrains.python.inspections.PyMethodParametersInspection; import com.jetbrains.python.inspections.PyTrailingSemicolonInspection; import com.jetbrains.python.inspections.PyUnresolvedReferencesInspection; -import gnu.trove.TIntObjectHashMap; +import com.jetbrains.python.sdk.PythonSdkType; import org.jetbrains.annotations.NonNls; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; +import java.io.File; /** * Test actions that various inspections add. * User: dcheryasov * Date: Nov 29, 2008 12:47:08 AM */ -public class PyQuickFixTest extends DaemonAnalyzerTestCase { +public class PyQuickFixTest extends PyLightFixtureTestCase { public void testAddImport() throws Exception { doInspectionTest("AddImport.py", PyUnresolvedReferencesInspection.class, PyBundle.message("ACT.NAME.add.import"), true, true); @@ -106,20 +102,6 @@ public class PyQuickFixTest extends DaemonAnalyzerTestCase { ); } - protected VirtualFile[] loadFiles(String[] names) { - VirtualFile[] ret = new VirtualFile[names.length]; - String prefix = getTestDataPath(); - for (int i=0; i < names.length; i += 1) { - ret[i] = getVirtualFile(prefix+names[i]); - } - return ret; - } - - protected void doTest(VirtualFile[] vFile, boolean checkWarnings, boolean checkInfos) throws Exception { - configureByFiles(null, vFile); - doDoTest(checkWarnings, checkInfos); - } - protected @NonNls String getTestDataPath() { return PythonTestUtil.getTestDataPath() + "/inspections/"; } @@ -148,68 +130,32 @@ public class PyQuickFixTest extends DaemonAnalyzerTestCase { boolean applyFix, boolean available ) throws Exception { - final LocalInspectionTool tool = (LocalInspectionTool)inspectionClass.newInstance(); - enableInspectionTool(tool); - configureByFiles(null, testFiles); - Collection infos = doDoTest(true, false); - - doQuickFixTest(infos, quickFixName, applyFix, available, testFiles[0]); - disableInspectionTool(tool.getShortName()); - } - - protected void doQuickFixTest(Collection infos, - String quickFixName, - boolean applyFix, - boolean shouldBeAvailable, - String s) throws Exception { - final List availableActions = new ArrayList(1); - - TIntObjectHashMap map = new TIntObjectHashMap(infos.size()); - for (HighlightInfo info : infos) { - final GutterIconRenderer renderer = info.getGutterIconRenderer(); - if (renderer == null) { - assertFalse("There should be one intention for highlight info", map.containsKey(info.startOffset)); - map.put(info.startOffset, info); - - if (info.quickFixActionRanges != null) { - for (Pair pair : info.quickFixActionRanges) { - IntentionAction action = pair.first.getAction(); - availableActions.add(action); - } - } - } - } - - final IntentionAction intentionAction = LightQuickFixTestCase.findActionWithText(availableActions, quickFixName); - if (shouldBeAvailable) { + myFixture.enableInspections(inspectionClass); + myFixture.configureByFiles(testFiles); + myFixture.checkHighlighting(true, false, false); + final IntentionAction intentionAction = myFixture.findSingleIntention(quickFixName); + if (available) { assertNotNull(intentionAction); + if (applyFix) { + myFixture.launchAction(intentionAction); + + myFixture.checkResultByFile(graftBeforeExt(testFiles [0], "_after")); + } } else { assertNull(intentionAction); } - - if (applyFix && shouldBeAvailable) { - intentionAction.invoke(myProject, myEditor, myFile); - - checkResultByFile(graftBeforeExt(s, "_after")); - } } // Turns "name.ext" to "name_insertion.ext" - private @NonNls String graftBeforeExt(String name, String insertion) { + + @NonNls + private static String graftBeforeExt(String name, String insertion) { int dotpos = name.indexOf('.'); if (dotpos < 0) dotpos = name.length(); return name.substring(0, dotpos) + insertion + name.substring(dotpos, name.length()); } - /* - @Override - protected Sdk getProjectJDK() { - return createMockJdk(PathManager.getHomePath() + "/plugins/python/testData/mockPythonJDK"); - } - */ - - /* private static Sdk createMockJdk(String jdkHome) { File jdkHomeFile = new File(jdkHome); if (!jdkHomeFile.exists()) return null; @@ -226,5 +172,18 @@ public class PyQuickFixTest extends DaemonAnalyzerTestCase { jdk.setVersionString("2.5"); return jdk; } - */ -} \ No newline at end of file + + protected static class PyWithSdkProjectDescriptor extends PyLightProjectDescriptor { + @Override + public Sdk getSdk() { + return createMockJdk(PathManager.getHomePath() + "/plugins/python/testData/mockPythonJDK"); + } + } + + private static final LightProjectDescriptor ourProjectDescriptor = new PyWithSdkProjectDescriptor(); + + @Override + protected LightProjectDescriptor getProjectDescriptor() { + return ourProjectDescriptor; + } +} From 84bb87e04da4951ca04d77577d2fb1abc20b55d0 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Thu, 24 Dec 2009 19:49:38 +0300 Subject: [PATCH 21/32] convert test to PyLightFixtureTestCase --- .../com/jetbrains/python/PyStubsTest.java | 39 ++++++------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/python/testSrc/com/jetbrains/python/PyStubsTest.java b/python/testSrc/com/jetbrains/python/PyStubsTest.java index 3352c4392a97..a3532ebd7faf 100644 --- a/python/testSrc/com/jetbrains/python/PyStubsTest.java +++ b/python/testSrc/com/jetbrains/python/PyStubsTest.java @@ -3,49 +3,34 @@ */ package com.jetbrains.python; -import com.intellij.codeInsight.CodeInsightTestCase; import com.intellij.openapi.application.Result; -import com.intellij.openapi.application.WriteAction; import com.intellij.openapi.command.WriteCommandAction; import com.intellij.openapi.vfs.VirtualFile; -import com.intellij.psi.PsiDocumentManager; import com.intellij.psi.PsiElement; import com.intellij.psi.PsiFile; import com.intellij.psi.stubs.StubElement; import com.intellij.psi.stubs.StubUpdatingIndex; -import com.intellij.testFramework.PsiTestUtil; import com.intellij.util.indexing.FileBasedIndex; +import com.jetbrains.python.fixtures.PyLightFixtureTestCase; import com.jetbrains.python.psi.*; import com.jetbrains.python.psi.impl.PyFileImpl; import com.jetbrains.python.psi.stubs.PyClassStub; +import java.io.IOException; import java.util.List; -public class PyStubsTest extends CodeInsightTestCase { - private VirtualFile myRootDir; +public class PyStubsTest extends PyLightFixtureTestCase { private static final String PARSED_ERROR_MSG = "Operations should have been performed on stubs but caused file to be parsed"; - protected void setUp() throws Exception { - myRunCommandForTest = false; - super.setUp(); - prepareRoots(); + @Override + protected String getTestDataPath() { + return PythonTestUtil.getTestDataPath() + "/stubs/"; } - private void assertNotParsed(PyFile file) { + private static void assertNotParsed(PyFile file) { assertNull(PARSED_ERROR_MSG, ((PyFileImpl)file).getTreeElement()); } - private void prepareRoots() throws Exception { - new WriteAction() { - protected void run(final Result result) throws Throwable { - String root = PythonTestUtil.getTestDataPath() + "/stubs/"; - myRootDir = PsiTestUtil.createTestProjectStructure(myProject, myModule, root, myFilesToDelete, false); - PsiTestUtil.addSourceContentToRoots(myModule, myRootDir); - PsiDocumentManager.getInstance(myProject).commitAllDocuments(); - } - }.execute(); - } - public void testStubStructure() throws Exception { final PyFile file = getTestFile(); final List classes = file.getTopLevelClasses(); @@ -152,7 +137,7 @@ public class PyStubsTest extends CodeInsightTestCase { assertEquals(1, children.length); assertSame(pyClass, children[0]); - new WriteCommandAction(myProject, fileImpl) { + new WriteCommandAction(myFixture.getProject(), fileImpl) { protected void run(final Result result) throws Throwable { pyClass.setName("RenamedClass"); assertEquals("RenamedClass", pyClass.getName()); @@ -162,7 +147,7 @@ public class PyStubsTest extends CodeInsightTestCase { StubElement fileStub = fileImpl.getStub(); assertNull("There should be no stub if file holds tree element", fileStub); - FileBasedIndex.getInstance().ensureUpToDate(StubUpdatingIndex.INDEX_ID, getProject(), null); + FileBasedIndex.getInstance().ensureUpToDate(StubUpdatingIndex.INDEX_ID, myFixture.getProject(), null); fileImpl.unloadContent(); assertNull(fileImpl.getTreeElement()); // Test unload successed. @@ -177,10 +162,10 @@ public class PyStubsTest extends CodeInsightTestCase { return getTestFile(getTestName(false) + ".py"); } - private PyFile getTestFile(final String fileName) { - VirtualFile sourceFile = myRootDir.findChild(fileName); + private PyFile getTestFile(final String fileName) throws IOException { + VirtualFile sourceFile = myFixture.copyFileToProject(fileName); assert sourceFile != null; - PsiFile psiFile = myPsiManager.findFile(sourceFile); + PsiFile psiFile = myFixture.getPsiManager().findFile(sourceFile); return (PyFile)psiFile; } } From 9ab912bb763cc73a1b79df7eba7df4f21ad04595 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Thu, 24 Dec 2009 19:58:21 +0300 Subject: [PATCH 22/32] extract Java-related tests to a separate module --- python/python-plugin-tests.iml | 16 ++++++ .../jetbrains/python/PyToJavaResolveTest.java | 49 ------------------- .../jetbrains/python/PythonAllTestsSuite.java | 2 +- 3 files changed, 17 insertions(+), 50 deletions(-) create mode 100644 python/python-plugin-tests.iml delete mode 100644 python/testSrc/com/jetbrains/python/PyToJavaResolveTest.java diff --git a/python/python-plugin-tests.iml b/python/python-plugin-tests.iml new file mode 100644 index 000000000000..f7444bf21dc5 --- /dev/null +++ b/python/python-plugin-tests.iml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/python/testSrc/com/jetbrains/python/PyToJavaResolveTest.java b/python/testSrc/com/jetbrains/python/PyToJavaResolveTest.java deleted file mode 100644 index b060a6dcd258..000000000000 --- a/python/testSrc/com/jetbrains/python/PyToJavaResolveTest.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.jetbrains.python; - -import com.intellij.openapi.projectRoots.Sdk; -import com.intellij.openapi.projectRoots.impl.JavaSdkImpl; -import com.intellij.psi.*; -import com.intellij.testFramework.ResolveTestCase; - -/** - * @author yole - */ -public class PyToJavaResolveTest extends ResolveTestCase { - protected Sdk getTestProjectJdk() { - return JavaSdkImpl.getMockJdk15("java 1.5"); - } - - private PsiElement resolve() throws Exception { - PsiReference ref = configureByFile(getTestName(false) + ".py"); - return ref.resolve(); - } - - public void testSimple() throws Exception { - PsiElement target = resolve(); - assertTrue(target instanceof PsiClass); - assertEquals("java.util.ArrayList", ((PsiClass) target).getQualifiedName()); - } - - public void testMethod() throws Exception { - PsiElement target = resolve(); - assertTrue(target instanceof PsiMethod); - assertEquals("java.util.ArrayList", ((PsiMethod) target).getContainingClass().getQualifiedName()); - } - - public void testField() throws Exception { - PsiElement target = resolve(); - assertTrue(target instanceof PsiField); - assertEquals("java.lang.System", ((PsiField) target).getContainingClass().getQualifiedName()); - } - - public void testReturnValue() throws Exception { - PsiElement target = resolve(); - assertTrue(target instanceof PsiMethod); - assertEquals("java.util.List", ((PsiMethod) target).getContainingClass().getQualifiedName()); - } - - @Override - protected String getTestDataPath() { - return PythonTestUtil.getTestDataPath() + "/resolve/pyToJava/"; - } -} diff --git a/python/testSrc/com/jetbrains/python/PythonAllTestsSuite.java b/python/testSrc/com/jetbrains/python/PythonAllTestsSuite.java index 0321c1470280..81b970b35265 100644 --- a/python/testSrc/com/jetbrains/python/PythonAllTestsSuite.java +++ b/python/testSrc/com/jetbrains/python/PythonAllTestsSuite.java @@ -19,7 +19,6 @@ public class PythonAllTestsSuite { PyResolveTest.class, PyMultiFileResolveTest.class, PyResolveCalleeTest.class, - PyToJavaResolveTest.class, PyAssignmentMappingTest.class, PythonCompletionTest.class, PyInheritorsSearchTest.class, @@ -30,6 +29,7 @@ public class PythonAllTestsSuite { PythonDemorganLawIntentionTest.class, PyQuickFixTest.class, PySelectWordTest.class, + PySurroundWithTest.class }; public static TestSuite suite() { From e84db7d2bf7ff0166dc33c4666a9de9eee986ec0 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Thu, 24 Dec 2009 20:04:48 +0300 Subject: [PATCH 23/32] remove depenendencies which are no longer necessary --- python/python-tests.iml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/python/python-tests.iml b/python/python-tests.iml index 5f46dba2fc3c..99ac91f5fb60 100644 --- a/python/python-tests.iml +++ b/python/python-tests.iml @@ -7,12 +7,9 @@ - - - - + From 8409d343b52bcdc716c2d7ae1fd7fc7a79fe4b4e Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Thu, 24 Dec 2009 20:21:09 +0300 Subject: [PATCH 24/32] allow running tests with classpath of module python-tests --- python/python-tests.iml | 5 + .../ProductivityFeaturesRegistry.xml | 252 ++++++++++++++++++ .../fixtures/PyLightFixtureTestCase.java | 17 ++ 3 files changed, 274 insertions(+) create mode 100644 python/resources/ProductivityFeaturesRegistry.xml diff --git a/python/python-tests.iml b/python/python-tests.iml index 99ac91f5fb60..58cac4fe5f47 100644 --- a/python/python-tests.iml +++ b/python/python-tests.iml @@ -10,6 +10,11 @@ + + + + + diff --git a/python/resources/ProductivityFeaturesRegistry.xml b/python/resources/ProductivityFeaturesRegistry.xml new file mode 100644 index 000000000000..dc87bf416ab0 --- /dev/null +++ b/python/resources/ProductivityFeaturesRegistry.xml @@ -0,0 +1,252 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/python/fixtures/PyLightFixtureTestCase.java b/python/testSrc/com/jetbrains/python/fixtures/PyLightFixtureTestCase.java index 75c55fe17e1c..3cab76ca553d 100644 --- a/python/testSrc/com/jetbrains/python/fixtures/PyLightFixtureTestCase.java +++ b/python/testSrc/com/jetbrains/python/fixtures/PyLightFixtureTestCase.java @@ -23,10 +23,12 @@ public abstract class PyLightFixtureTestCase extends UsefulTestCase { private static final PyLightProjectDescriptor ourPyDescriptor = new PyLightProjectDescriptor(); protected CodeInsightTestFixture myFixture; + private static boolean ourPlatformPrefixInitialized; @Override protected void setUp() throws Exception { super.setUp(); + initPlatformPrefix(); IdeaTestFixtureFactory factory = IdeaTestFixtureFactory.getFixtureFactory(); TestFixtureBuilder fixtureBuilder = factory.createLightFixtureBuilder(getProjectDescriptor()); final IdeaProjectTestFixture fixture = fixtureBuilder.getFixture(); @@ -66,4 +68,19 @@ public abstract class PyLightFixtureTestCase extends UsefulTestCase { } } + public static void initPlatformPrefix() { + if (!ourPlatformPrefixInitialized) { + ourPlatformPrefixInitialized = true; + boolean isIDEA = true; + try { + PyLightFixtureTestCase.class.getClassLoader().loadClass("com.intellij.openapi.project.impl.IdeaProjectManagerImpl"); + } + catch (ClassNotFoundException e) { + isIDEA = false; + } + if (!isIDEA) { + System.setProperty("idea.platform.prefix", "Python"); + } + } + } } From a5916b197770325ca9d59fc57af831ec6493c6a8 Mon Sep 17 00:00:00 2001 From: nik Date: Fri, 25 Dec 2009 10:22:34 +0300 Subject: [PATCH 25/32] incorrect dependency on junit3 removed --- python/python-tests.iml | 1 - 1 file changed, 1 deletion(-) diff --git a/python/python-tests.iml b/python/python-tests.iml index 58cac4fe5f47..40b970e6a877 100644 --- a/python/python-tests.iml +++ b/python/python-tests.iml @@ -8,7 +8,6 @@ - From f27cd6f234584c3936a4d8414fac15ca3f840204 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Fri, 25 Dec 2009 17:21:34 +0300 Subject: [PATCH 26/32] catch SIOOBE in PythonNamesValidator --- .../refactoring/PythonNamesValidator.java | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/python/src/com/jetbrains/python/refactoring/PythonNamesValidator.java b/python/src/com/jetbrains/python/refactoring/PythonNamesValidator.java index bcb695f6bba0..403e454d503c 100644 --- a/python/src/com/jetbrains/python/refactoring/PythonNamesValidator.java +++ b/python/src/com/jetbrains/python/refactoring/PythonNamesValidator.java @@ -7,29 +7,36 @@ import com.jetbrains.python.lexer.PythonLexer; import org.jetbrains.annotations.NotNull; /** - * Created by IntelliJ IDEA. - * User: Alexey.Ivanov - * Date: Aug 19, 2009 - * Time: 10:23:26 PM + * @author Alexey.Ivanov */ public class PythonNamesValidator implements NamesValidator { private static final PythonLexer ourLexer = new PythonLexer(); public synchronized boolean isKeyword(@NotNull final String name, final Project project) { - ourLexer.start(name); - if (!PyTokenTypes.KEYWORDS.contains(ourLexer.getTokenType())) { + try { + ourLexer.start(name); + if (!PyTokenTypes.KEYWORDS.contains(ourLexer.getTokenType())) { + return false; + } + ourLexer.advance(); + return ourLexer.getTokenType() == null; + } + catch (StringIndexOutOfBoundsException e) { return false; } - ourLexer.advance(); - return ourLexer.getTokenType() == null; } public synchronized boolean isIdentifier(@NotNull final String name, final Project project) { - ourLexer.start(name); - if (ourLexer.getTokenType() != PyTokenTypes.IDENTIFIER) { + try { + ourLexer.start(name); + if (ourLexer.getTokenType() != PyTokenTypes.IDENTIFIER) { + return false; + } + ourLexer.advance(); + return ourLexer.getTokenType() == null; + } + catch (StringIndexOutOfBoundsException e) { return false; } - ourLexer.advance(); - return ourLexer.getTokenType() == null; } } From 0ad756b9359859bbb3dc9fa5f9dc337bfed4224f Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Fri, 25 Dec 2009 17:59:38 +0300 Subject: [PATCH 27/32] tests for forthcoming functionality, cleanup and refactoring --- python/src/com/jetbrains/python/psi/PyFile.java | 5 ++++- .../src/com/jetbrains/python/psi/impl/PyFileImpl.java | 10 ++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/python/src/com/jetbrains/python/psi/PyFile.java b/python/src/com/jetbrains/python/psi/PyFile.java index 6c23bf92b131..db339e85e158 100644 --- a/python/src/com/jetbrains/python/psi/PyFile.java +++ b/python/src/com/jetbrains/python/psi/PyFile.java @@ -36,5 +36,8 @@ public interface PyFile extends PyElement, PsiFile, PyDocStringOwner { @return an URL of file, maybe bogus if virtual file is not present. */ @NotNull - String getUrl(); + String getUrl(); + + @Nullable + PyFunction findTopLevelFunction(String name); } diff --git a/python/src/com/jetbrains/python/psi/impl/PyFileImpl.java b/python/src/com/jetbrains/python/psi/impl/PyFileImpl.java index 3fef824a0d00..c76eeeb5230c 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyFileImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyFileImpl.java @@ -68,6 +68,16 @@ public class PyFileImpl extends PsiFileBase implements PyFile, PyExpression { return fname; } + public PyFunction findTopLevelFunction(String name) { + final List functions = getTopLevelFunctions(); + for (PyFunction function : functions) { + if (name.equals(function.getName())) { + return function; + } + } + return null; + } + public Icon getIcon(int flags) { return PythonFileType.INSTANCE.getIcon(); } From 016389ede712dbe86ca02dc3745f525b93c8d969 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Fri, 25 Dec 2009 18:31:11 +0300 Subject: [PATCH 28/32] do not copy helper scripts to temp dir, run them directly from helpers --- .../python/sdk => helpers}/find_binaries.py | 0 .../python/sdk => helpers}/generator3.py | 0 .../python/sdk => helpers}/pyparsing.py | 0 .../python/sdk => helpers}/pyparsing_py3.py | 0 .../jetbrains/python/sdk/PythonSdkType.java | 145 +++++++----------- 5 files changed, 53 insertions(+), 92 deletions(-) rename python/{src/com/jetbrains/python/sdk => helpers}/find_binaries.py (100%) rename python/{src/com/jetbrains/python/sdk => helpers}/generator3.py (100%) rename python/{src/com/jetbrains/python/sdk => helpers}/pyparsing.py (100%) rename python/{src/com/jetbrains/python/sdk => helpers}/pyparsing_py3.py (100%) diff --git a/python/src/com/jetbrains/python/sdk/find_binaries.py b/python/helpers/find_binaries.py similarity index 100% rename from python/src/com/jetbrains/python/sdk/find_binaries.py rename to python/helpers/find_binaries.py diff --git a/python/src/com/jetbrains/python/sdk/generator3.py b/python/helpers/generator3.py similarity index 100% rename from python/src/com/jetbrains/python/sdk/generator3.py rename to python/helpers/generator3.py diff --git a/python/src/com/jetbrains/python/sdk/pyparsing.py b/python/helpers/pyparsing.py similarity index 100% rename from python/src/com/jetbrains/python/sdk/pyparsing.py rename to python/helpers/pyparsing.py diff --git a/python/src/com/jetbrains/python/sdk/pyparsing_py3.py b/python/helpers/pyparsing_py3.py similarity index 100% rename from python/src/com/jetbrains/python/sdk/pyparsing_py3.py rename to python/helpers/pyparsing_py3.py diff --git a/python/src/com/jetbrains/python/sdk/PythonSdkType.java b/python/src/com/jetbrains/python/sdk/PythonSdkType.java index 901d3647aa36..ebb95ea67e2e 100644 --- a/python/src/com/jetbrains/python/sdk/PythonSdkType.java +++ b/python/src/com/jetbrains/python/sdk/PythonSdkType.java @@ -27,6 +27,7 @@ import com.intellij.openapi.vfs.VfsUtil; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.util.ArrayUtil; import com.jetbrains.python.PythonFileType; +import com.jetbrains.python.PythonHelpersLocator; import com.jetbrains.python.facet.PythonFacetSettings; import org.jdom.Element; import org.jetbrains.annotations.NonNls; @@ -34,7 +35,9 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import javax.swing.*; -import java.io.*; +import java.io.File; +import java.io.IOException; +import java.io.PrintStream; import java.util.Collections; import java.util.List; import java.util.TreeSet; @@ -351,57 +354,29 @@ public class PythonSdkType extends SdkType { return getPythonBinaryPath(sdkHome).getPath(); } - /** - * Copies a number of resources as files to a temporary directory. - * @param resourceNames each file created will have the same name as the resource it's created from. - * @return the temporary directory - * @throws IOException if anything goes wrong. - */ - private static File copyResourcesToTempDir(final String... resourceNames) throws IOException { - final File tempdir = FileUtil.createTempDirectory("pycharm", ""); - for (final String resourceName : resourceNames) { - final String text = FileUtil.loadTextAndClose(new InputStreamReader(PythonSdkType.class.getResourceAsStream(resourceName))); - File target = new File(tempdir.getCanonicalPath(), resourceName); - FileWriter out = new FileWriter(target); - out.write(text); - out.close(); - } - return tempdir; - } - - private final static String GENERATOR3 = "generator3.py"; private final static String FIND_BINARIES = "find_binaries.py"; public static void generateBuiltinStubs(String sdkPath, final String stubsRoot) { new File(stubsRoot).mkdirs(); + + GeneralCommandLine commandLine = new GeneralCommandLine(); + commandLine.setExePath(getInterpreterPath(sdkPath)); // python + commandLine.addParameter(PythonHelpersLocator.getHelperPath(GENERATOR3)); + + commandLine.addParameter("-d"); + commandLine.addParameter(stubsRoot); // -d stubs_root + commandLine.addParameter("-b"); // for builtins + commandLine.addParameter("-u"); // for update-only mode try { - - final File tempDir = copyResourcesToTempDir(GENERATOR3, "pyparsing.py", "pyparsing_py3.py"); - - GeneralCommandLine commandLine = new GeneralCommandLine(); - commandLine.setExePath(getInterpreterPath(sdkPath)); // python - - commandLine.addParameter(tempDir.getAbsolutePath() + File.separatorChar + GENERATOR3); - - commandLine.addParameter("-d"); commandLine.addParameter(stubsRoot); // -d stubs_root - commandLine.addParameter("-b"); // for builtins - commandLine.addParameter("-u"); // for update-only mode - try { - final OSProcessHandler handler = new OSProcessHandler(commandLine.createProcess(), commandLine.getCommandLineString()); - handler.startNotify(); - handler.waitFor(); - handler.destroyProcess(); - } - catch (ExecutionException e) { - LOG.error(e); - } - FileUtil.delete(tempDir); + final OSProcessHandler handler = new OSProcessHandler(commandLine.createProcess(), commandLine.getCommandLineString()); + handler.startNotify(); + handler.waitFor(); + handler.destroyProcess(); } - catch (IOException e) { + catch (ExecutionException e) { LOG.error(e); } - } /** @@ -412,64 +387,50 @@ public class PythonSdkType extends SdkType { * @param indicator ProgressIndicator to update, or null. */ public static void generateBinaryStubs(final String sdkPath, final String stubsRoot, ProgressIndicator indicator) { - if (!new File(stubsRoot).exists()) return; + if (!new File(stubsRoot).exists()) return; if (indicator != null) { indicator.setText("Generating skeletons of binary libs"); } - try { - final int RUN_TIMEOUT = 10*1000; // 10 seconds per call is plenty enough; anything more is clearly wrong. - final String bin_path = getInterpreterPath(sdkPath); + final int RUN_TIMEOUT = 10 * 1000; // 10 seconds per call is plenty enough; anything more is clearly wrong. + final String bin_path = getInterpreterPath(sdkPath); - final File tempDir = copyResourcesToTempDir(GENERATOR3, FIND_BINARIES, "pyparsing.py", "pyparsing_py3.py"); + final ProcessOutput run_result = + SdkUtil.getProcessOutput(sdkPath, new String[]{bin_path, PythonHelpersLocator.getHelperPath(FIND_BINARIES)}); - - try { - final ProcessOutput run_result = SdkUtil.getProcessOutput( - sdkPath, new String[] {bin_path, tempDir.getPath() + File.separatorChar + FIND_BINARIES} - ); - - if (run_result.getExitCode() == 0) { - for (String line : run_result.getStdoutLines()) { - // line = "mod_name path" - int cutpos = line.indexOf(' '); - String modname = line.substring(0, cutpos); - String mod_fname = modname.replace(".", File.separator); // "a.b.c" -> "a/b/c", no ext - String fname = line.substring(cutpos+1); - //String ext = fname.substring(fname.lastIndexOf('.')); // no way ext is absent - // check if it's fresh - File f_orig = new File(fname); - File f_skel = new File(stubsRoot + File.separator + mod_fname + ".py"); - if (f_orig.lastModified() >= f_skel.lastModified()) { - // stale skeleton, rebuild - if (indicator != null) { - indicator.setText2(modname); - } - LOG.info("Skeleton for " + modname); - final ProcessOutput gen_result = SdkUtil.getProcessOutput(sdkPath, - new String[] {bin_path, tempDir.getPath() + File.separatorChar + GENERATOR3, "-d", stubsRoot, modname}, RUN_TIMEOUT - ); - if (gen_result.getExitCode() != 0) { - StringBuffer sb = new StringBuffer("Skeleton for "); - sb.append(modname).append(" failed. stderr: --"); - for (String err_line : gen_result.getStderrLines()) sb.append(err_line).append("\n"); - sb.append("--"); - LOG.warn(sb.toString()); - } - } + if (run_result.getExitCode() == 0) { + for (String line : run_result.getStdoutLines()) { + // line = "mod_name path" + int cutpos = line.indexOf(' '); + String modname = line.substring(0, cutpos); + String mod_fname = modname.replace(".", File.separator); // "a.b.c" -> "a/b/c", no ext + String fname = line.substring(cutpos + 1); + //String ext = fname.substring(fname.lastIndexOf('.')); // no way ext is absent + // check if it's fresh + File f_orig = new File(fname); + File f_skel = new File(stubsRoot + File.separator + mod_fname + ".py"); + if (f_orig.lastModified() >= f_skel.lastModified()) { + // stale skeleton, rebuild + if (indicator != null) { + indicator.setText2(modname); + } + LOG.info("Skeleton for " + modname); + final ProcessOutput gen_result = SdkUtil + .getProcessOutput(sdkPath, new String[]{bin_path, PythonHelpersLocator.getHelperPath(GENERATOR3), "-d", stubsRoot, modname}, + RUN_TIMEOUT); + if (gen_result.getExitCode() != 0) { + StringBuffer sb = new StringBuffer("Skeleton for "); + sb.append(modname).append(" failed. stderr: --"); + for (String err_line : gen_result.getStderrLines()) sb.append(err_line).append("\n"); + sb.append("--"); + LOG.warn(sb.toString()); } } - else { - StringBuffer sb = new StringBuffer(); - for (String err_line : run_result.getStderrLines()) sb.append(err_line).append("\n"); - LOG.error("failed to run " + FIND_BINARIES + ", exit code " + run_result.getExitCode() + ", stderr '" + sb.toString() + "'"); - } - } - finally { - FileUtil.delete(tempDir); } } - catch (IOException e) { - LOG.error(e); + else { + StringBuffer sb = new StringBuffer(); + for (String err_line : run_result.getStderrLines()) sb.append(err_line).append("\n"); + LOG.error("failed to run " + FIND_BINARIES + ", exit code " + run_result.getExitCode() + ", stderr '" + sb.toString() + "'"); } } From 733d9a3736bccd51dfce248dc68e2cbd045eb47d Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Fri, 25 Dec 2009 18:34:54 +0300 Subject: [PATCH 29/32] really-really , we do need to process remaining modules if we fail to generate a skeleton for one --- python/helpers/generator3.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/helpers/generator3.py b/python/helpers/generator3.py index 3ab93c581546..45ad4b88c281 100644 --- a/python/helpers/generator3.py +++ b/python/helpers/generator3.py @@ -1085,5 +1085,6 @@ if __name__ == "__main__": outfile.close() except: sys.stderr.write("Failed to process " + name + " while " + action + "\n") - raise + # Please don't add a 'raise' here. If we fail to process one module, it's not a sufficient reason + # to cancel processing the remaining ones. continue From ef05a7b80b4bf452b2a01c06e6f84f658b88a8c1 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Fri, 25 Dec 2009 18:37:13 +0300 Subject: [PATCH 30/32] PythonHighlightingTest fixed --- python/testData/highlighting/declarations.py | 2 +- python/testData/highlighting/yieldInNestedFunction.py | 2 +- python/testSrc/com/jetbrains/python/PythonHighlightingTest.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/testData/highlighting/declarations.py b/python/testData/highlighting/declarations.py index 7661bce73ba8..3b2f482e2bd6 100644 --- a/python/testData/highlighting/declarations.py +++ b/python/testData/highlighting/declarations.py @@ -14,4 +14,4 @@ class __made_up__(self): - return None + return None diff --git a/python/testData/highlighting/yieldInNestedFunction.py b/python/testData/highlighting/yieldInNestedFunction.py index d366af9e3a25..67628a4e57fb 100644 --- a/python/testData/highlighting/yieldInNestedFunction.py +++ b/python/testData/highlighting/yieldInNestedFunction.py @@ -2,4 +2,4 @@ def foo(): def a(): yield 1 - return False + return False diff --git a/python/testSrc/com/jetbrains/python/PythonHighlightingTest.java b/python/testSrc/com/jetbrains/python/PythonHighlightingTest.java index 6e0b5ea4c02b..3562095b8ac8 100644 --- a/python/testSrc/com/jetbrains/python/PythonHighlightingTest.java +++ b/python/testSrc/com/jetbrains/python/PythonHighlightingTest.java @@ -63,7 +63,7 @@ public class PythonHighlightingTest extends PyLightFixtureTestCase { } public void testContinueInFinallyBlock() throws Exception { - doTest(); + doTest(false, false); } public void testReturnWithArgumentsInGenerator() throws Exception { From 5ead73ef75b5b714a31d65188d5ad6509476eddb Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Fri, 25 Dec 2009 18:54:28 +0300 Subject: [PATCH 31/32] PythonCompletionTest fixed --- python/testData/completion/funcParams.after.py | 4 ++-- python/testData/completion/funcParams.py | 4 ++-- python/testData/completion/funcParamsStar.after.py | 4 ++-- python/testData/completion/funcParamsStar.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/python/testData/completion/funcParams.after.py b/python/testData/completion/funcParams.after.py index b29540ddf81c..044fb3d731d1 100644 --- a/python/testData/completion/funcParams.after.py +++ b/python/testData/completion/funcParams.after.py @@ -1,4 +1,4 @@ -def foo(uno, dos, tres): +def foo(auno, dos, tres): pass -foo(uno=) +foo(auno=) diff --git a/python/testData/completion/funcParams.py b/python/testData/completion/funcParams.py index e5418d88cc80..9b013c10595e 100644 --- a/python/testData/completion/funcParams.py +++ b/python/testData/completion/funcParams.py @@ -1,4 +1,4 @@ -def foo(uno, dos, tres): +def foo(auno, dos, tres): pass -foo(un) +foo(aun) diff --git a/python/testData/completion/funcParamsStar.after.py b/python/testData/completion/funcParamsStar.after.py index f987e86b0dae..c5942aedce36 100644 --- a/python/testData/completion/funcParamsStar.after.py +++ b/python/testData/completion/funcParamsStar.after.py @@ -1,4 +1,4 @@ -def foo(boo1, *boo2, **boo3): +def foo(xboo1, *boo2, **boo3): pass -foo(boo1=) +foo(xboo1=) diff --git a/python/testData/completion/funcParamsStar.py b/python/testData/completion/funcParamsStar.py index 22dc5842e422..7c88a5f9bbf3 100644 --- a/python/testData/completion/funcParamsStar.py +++ b/python/testData/completion/funcParamsStar.py @@ -1,4 +1,4 @@ -def foo(boo1, *boo2, **boo3): +def foo(xboo1, *boo2, **boo3): pass -foo(boo) +foo(xboo) From 72d6cde09c98f3d39a21d7342d0a4b7054b3bd02 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Fri, 25 Dec 2009 19:44:43 +0300 Subject: [PATCH 32/32] reference from render_to_response to template file --- .../jetbrains/python/fixtures/PyLightFixtureTestCase.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/python/testSrc/com/jetbrains/python/fixtures/PyLightFixtureTestCase.java b/python/testSrc/com/jetbrains/python/fixtures/PyLightFixtureTestCase.java index 3cab76ca553d..1f76476ab157 100644 --- a/python/testSrc/com/jetbrains/python/fixtures/PyLightFixtureTestCase.java +++ b/python/testSrc/com/jetbrains/python/fixtures/PyLightFixtureTestCase.java @@ -6,6 +6,8 @@ import com.intellij.openapi.module.ModuleType; import com.intellij.openapi.projectRoots.Sdk; import com.intellij.openapi.roots.ContentEntry; import com.intellij.openapi.roots.ModifiableRootModel; +import com.intellij.psi.PsiDocumentManager; +import com.intellij.psi.PsiReference; import com.intellij.testFramework.LightProjectDescriptor; import com.intellij.testFramework.UsefulTestCase; import com.intellij.testFramework.fixtures.CodeInsightTestFixture; @@ -55,6 +57,11 @@ public abstract class PyLightFixtureTestCase extends UsefulTestCase { return ourPyDescriptor; } + protected PsiReference findReferenceBySignature(final String signature) { + int pos = PsiDocumentManager.getInstance(myFixture.getProject()).getDocument(myFixture.getFile()).getText().indexOf(signature); + return myFixture.getFile().findReferenceAt(pos); + } + protected static class PyLightProjectDescriptor implements LightProjectDescriptor { public ModuleType getModuleType() { return EmptyModuleType.getInstance();