From a3dffc58accf6355077f3e046a00efedc4365dae Mon Sep 17 00:00:00 2001 From: Elizaveta Shashkova Date: Fri, 22 Sep 2017 13:52:10 +0300 Subject: [PATCH] Add ability to load variables in the Variables Pane asynchronously (PY-12987) If in the frame there is at least one very long or slow variable, the frame's variable aren't shown. In order to avoid such situations we add an option to load variables asynchronously. At first debugger loads only names of the variables, and after that it sends additional commands and evaluates values of the variables in a separate threads. --- .../_pydev_bundle/pydev_console_utils.py | 46 +++- .../pydev/_pydevd_bundle/pydevd_comm.py | 82 +++++- .../pydev/_pydevd_bundle/pydevd_constants.py | 5 + .../pydevd_process_net_command.py | 15 +- .../pydev/_pydevd_bundle/pydevd_resolver.py | 4 +- .../pydev/_pydevd_bundle/pydevd_xml.py | 97 ++++--- python/helpers/pydev/pydevconsole.py | 1 + .../python/debugger/PyDebugValue.java | 258 +++++++++++++----- .../python/debugger/PyFrameAccessor.java | 33 ++- .../python/debugger/PyFullValueEvaluator.java | 22 +- .../debugger/PyLoadingValueEvaluator.java | 19 ++ .../debugger/pydev/AbstractCommand.java | 3 +- .../debugger/pydev/ChangeVariableCommand.java | 6 +- .../pydev/ClientModeMultiProcessDebugger.java | 7 + .../debugger/pydev/EvaluateCommand.java | 5 +- .../debugger/pydev/GetArrayCommand.java | 3 +- .../debugger/pydev/GetCompletionsCommand.java | 2 +- .../debugger/pydev/GetDescriptionCommand.java | 3 +- .../debugger/pydev/GetFrameCommand.java | 9 +- .../debugger/pydev/GetVariableCommand.java | 7 +- .../debugger/pydev/LoadFullValueCommand.java | 72 +++++ .../debugger/pydev/LoadSourceCommand.java | 2 +- .../debugger/pydev/MultiProcessDebugger.java | 7 + .../debugger/pydev/ProcessDebugger.java | 5 + .../python/debugger/pydev/RemoteDebugger.java | 7 + .../pydev/SetNextStatementCommand.java | 2 +- .../python/debugger/pydev/VersionCommand.java | 3 +- .../console/PydevConsoleCommunication.java | 58 +++- .../console/PydevConsoleRunnerImpl.java | 13 + .../python/debugger/PyDebugProcess.java | 104 ++++--- .../python/debugger/PyDebugRunner.java | 4 +- .../python/debugger/PyDebuggerEvaluator.java | 8 +- .../python/debugger/PyStackFrame.java | 4 + .../debugger/PyVariableViewSettings.java | 92 +++++++ .../containerview/PyDataViewerPanel.java | 40 +-- .../debugger/settings/PyDebuggerSettings.java | 9 + python/testData/debug/test_async_eval.py | 15 + .../env/python/PythonDebuggerTest.java | 24 ++ .../env/python/debug/PyBaseDebuggerTask.java | 52 ++++ 39 files changed, 918 insertions(+), 230 deletions(-) create mode 100644 python/pydevSrc/com/jetbrains/python/debugger/PyLoadingValueEvaluator.java create mode 100644 python/pydevSrc/com/jetbrains/python/debugger/pydev/LoadFullValueCommand.java create mode 100644 python/src/com/jetbrains/python/debugger/PyVariableViewSettings.java create mode 100644 python/testData/debug/test_async_eval.py diff --git a/python/helpers/pydev/_pydev_bundle/pydev_console_utils.py b/python/helpers/pydev/_pydev_bundle/pydev_console_utils.py index 3f6d0ca47592..189dd4ce768e 100644 --- a/python/helpers/pydev/_pydev_bundle/pydev_console_utils.py +++ b/python/helpers/pydev/_pydev_bundle/pydev_console_utils.py @@ -9,6 +9,13 @@ from _pydevd_bundle import pydevd_xml from _pydevd_bundle.pydevd_constants import IS_JYTHON, dict_iter_items from _pydevd_bundle.pydevd_utils import to_string +try: + import cStringIO as StringIO #may not always be available @UnusedImport +except: + try: + import StringIO #@Reimport + except: + import io as StringIO # ======================================================================================================================= # Null @@ -450,15 +457,17 @@ class BaseInterpreterInterface: return True def getFrame(self): + xml = StringIO.StringIO() hidden_ns = self.get_ipython_hidden_vars_dict() - xml = "" - xml += pydevd_xml.frame_vars_to_xml(self.get_namespace(), hidden_ns) - xml += "" + xml.write("") + xml.write(pydevd_xml.frame_vars_to_xml(self.get_namespace(), hidden_ns)) + xml.write("") - return xml + return xml.getvalue() def getVariable(self, attributes): - xml = "" + xml = StringIO.StringIO() + xml.write("") valDict = pydevd_vars.resolve_var(self.get_namespace(), attributes) if valDict is None: valDict = {} @@ -466,11 +475,13 @@ class BaseInterpreterInterface: keys = valDict.keys() for k in keys: - xml += pydevd_vars.var_to_xml(valDict[k], to_string(k)) + val = valDict[k] + evaluate_full_value = pydevd_xml.should_evaluate_full_value(val) + xml.write(pydevd_vars.var_to_xml(val, k, evaluate_full_value=evaluate_full_value)) - xml += "" + xml.write("") - return xml + return xml.getvalue() def getArray(self, attr, roffset, coffset, rows, cols, format): name = attr.split("\t")[-1] @@ -478,14 +489,21 @@ class BaseInterpreterInterface: return pydevd_vars.table_like_struct_to_xml(array, name, roffset, coffset, rows, cols, format) def evaluate(self, expression): - xml = "" + xml = StringIO.StringIO() + xml.write("") result = pydevd_vars.eval_in_context(expression, self.get_namespace(), self.get_namespace()) + xml.write(pydevd_vars.var_to_xml(result, expression)) + xml.write("") + return xml.getvalue() - xml += pydevd_vars.var_to_xml(result, expression) - - xml += "" - - return xml + def loadFullValue(self, expressions): + xml = StringIO.StringIO() + xml.write("") + for expression in expressions: + result = pydevd_vars.eval_in_context(expression, self.get_namespace(), self.get_namespace()) + xml.write(pydevd_vars.var_to_xml(result, expression, evaluate_full_value=True)) + xml.write("") + return xml.getvalue() def changeVariable(self, attr, value): def do_change_variable(): diff --git a/python/helpers/pydev/_pydevd_bundle/pydevd_comm.py b/python/helpers/pydev/_pydevd_bundle/pydevd_comm.py index 9e4ba13eb9bb..2e8dc5b3557c 100644 --- a/python/helpers/pydev/_pydevd_bundle/pydevd_comm.py +++ b/python/helpers/pydev/_pydevd_bundle/pydevd_comm.py @@ -89,6 +89,13 @@ from _pydevd_bundle.pydevd_tracing import get_exception_traceback_str from _pydevd_bundle import pydevd_console from _pydev_bundle.pydev_monkey import disable_trace_thread_modules, enable_trace_thread_modules +try: + import cStringIO as StringIO #may not always be available @UnusedImport +except: + try: + import StringIO #@Reimport + except: + import io as StringIO CMD_RUN = 101 @@ -148,6 +155,7 @@ CMD_GET_DESCRIPTION = 148 CMD_PROCESS_CREATED = 149 CMD_SHOW_CYTHON_WARNING = 150 +CMD_LOAD_FULL_VALUE = 151 CMD_VERSION = 501 CMD_RETURN = 502 @@ -207,6 +215,7 @@ ID_TO_MEANING = { '149': 'CMD_PROCESS_CREATED', '150': 'CMD_SHOW_CYTHON_WARNING', + '151': 'CMD_LOAD_FULL_VALUE', '501': 'CMD_VERSION', '502': 'CMD_RETURN', @@ -844,6 +853,12 @@ class NetCommandFactory: except: return self.make_error_message(0, get_exception_traceback_str()) + def make_load_full_value_message(self, seq, payload): + try: + return NetCommand(CMD_LOAD_FULL_VALUE, seq, payload) + except Exception: + return self.make_error_message(seq, get_exception_traceback_str()) + def make_exit_message(self): try: net = NetCommand(CMD_EXIT, 0, '') @@ -1009,7 +1024,8 @@ class InternalGetVariable(InternalThreadCommand): def do_it(self, dbg): """ Converts request into python variable """ try: - xml = "" + xml = StringIO.StringIO() + xml.write("") _typeName, valDict = pydevd_vars.resolve_compound_variable(self.thread_id, self.frame_id, self.scope, self.attributes) if valDict is None: valDict = {} @@ -1025,10 +1041,13 @@ class InternalGetVariable(InternalThreadCommand): keys = sorted(keys, cmp=compare_object_attrs) #Jython 2.1 does not have it (and all must be compared as strings). for k in keys: - xml += pydevd_xml.var_to_xml(valDict[k], to_string(k)) + val = valDict[k] + evaluate_full_value = pydevd_xml.should_evaluate_full_value(val) + xml.write(pydevd_xml.var_to_xml(val, k, evaluate_full_value=evaluate_full_value)) - xml += "" - cmd = dbg.cmd_factory.make_get_variable_message(self.sequence, xml) + xml.write("") + cmd = dbg.cmd_factory.make_get_variable_message(self.sequence, xml.getvalue()) + xml.close() dbg.writer.add_command(cmd) except Exception: cmd = dbg.cmd_factory.make_error_message(self.sequence, "Error resolving variables " + get_exception_traceback_str()) @@ -1442,6 +1461,61 @@ class InternalConsoleExec(InternalThreadCommand): sys.stdout.flush() +#======================================================================================================================= +# InternalLoadFullValue +#======================================================================================================================= +class InternalLoadFullValue(InternalThreadCommand): + """ changes the value of a variable """ + def __init__(self, seq, thread_id, frame_id, vars): + self.sequence = seq + self.thread_id = thread_id + self.frame_id = frame_id + self.vars = vars + + def do_it(self, dbg): + """ Converts request into python variable """ + try: + var_objects = [] + for variable in self.vars: + variable = variable.strip() + if len(variable) > 0: + if '\t' in variable: # there are attributes beyond scope + scope, attrs = variable.split('\t', 1) + name = attrs[0] + else: + scope, attrs = (variable, None) + name = scope + + var_obj = pydevd_vars.getVariable(self.thread_id, self.frame_id, scope, attrs) + var_objects.append((var_obj, name)) + + t = GetValueAsyncThread(dbg, self.sequence, var_objects) + t.start() + except: + exc = get_exception_traceback_str() + sys.stderr.write('%s\n' % (exc,)) + cmd = dbg.cmd_factory.make_error_message(self.sequence, "Error evaluating variable %s " % exc) + dbg.writer.add_command(cmd) + + +class GetValueAsyncThread(PyDBDaemonThread): + def __init__(self, py_db, seq, var_objects): + PyDBDaemonThread.__init__(self) + self.py_db = py_db + self.seq = seq + self.var_objs = var_objects + + def _on_run(self): + xml = StringIO.StringIO() + xml.write("") + for (var_obj, name) in self.var_objs: + xml.write(pydevd_xml.var_to_xml(var_obj, name, evaluate_full_value=True)) + xml.write("") + cmd = self.py_db.cmd_factory.make_load_full_value_message(self.seq, xml.getvalue()) + xml.close() + self.py_db.writer.add_command(cmd) + + #======================================================================================================================= # pydevd_find_thread_by_id #======================================================================================================================= diff --git a/python/helpers/pydev/_pydevd_bundle/pydevd_constants.py b/python/helpers/pydev/_pydevd_bundle/pydevd_constants.py index 07e122a411d5..9a6337d1d143 100644 --- a/python/helpers/pydev/_pydevd_bundle/pydevd_constants.py +++ b/python/helpers/pydev/_pydevd_bundle/pydevd_constants.py @@ -110,6 +110,11 @@ USE_LIB_COPY = SUPPORT_GEVENT and \ INTERACTIVE_MODE_AVAILABLE = sys.platform in ('darwin', 'win32') or os.getenv('DISPLAY') is not None IS_PYCHARM = True +LOAD_VALUES_ASYNC = os.getenv('PYDEVD_LOAD_VALUES_ASYNC', 'False') == 'True' +DEFAULT_VALUE = "__pydevd_value_async" +NEXT_VALUE_SEPARATOR = "__pydev_val__" +BUILTINS_MODULE_NAME = '__builtin__' if IS_PY2 else 'builtins' + def protect_libraries_from_patching(): """ diff --git a/python/helpers/pydev/_pydevd_bundle/pydevd_process_net_command.py b/python/helpers/pydev/_pydevd_bundle/pydevd_process_net_command.py index f326086adaed..640978b7f9c2 100644 --- a/python/helpers/pydev/_pydevd_bundle/pydevd_process_net_command.py +++ b/python/helpers/pydev/_pydevd_bundle/pydevd_process_net_command.py @@ -17,9 +17,10 @@ from _pydevd_bundle.pydevd_comm import CMD_RUN, CMD_VERSION, CMD_LIST_THREADS, C CMD_REMOVE_EXCEPTION_BREAK, CMD_LOAD_SOURCE, CMD_ADD_DJANGO_EXCEPTION_BREAK, CMD_REMOVE_DJANGO_EXCEPTION_BREAK, \ CMD_EVALUATE_CONSOLE_EXPRESSION, InternalEvaluateConsoleExpression, InternalConsoleGetCompletions, \ CMD_RUN_CUSTOM_OPERATION, InternalRunCustomOperation, CMD_IGNORE_THROWN_EXCEPTION_AT, CMD_ENABLE_DONT_TRACE, \ - CMD_SHOW_RETURN_VALUES, ID_TO_MEANING, CMD_GET_DESCRIPTION, InternalGetDescription + CMD_SHOW_RETURN_VALUES, ID_TO_MEANING, CMD_GET_DESCRIPTION, InternalGetDescription, InternalLoadFullValue, \ + CMD_LOAD_FULL_VALUE from _pydevd_bundle.pydevd_constants import get_thread_id, IS_PY3K, DebugInfoHolder, dict_contains, dict_keys, \ - STATE_RUN + STATE_RUN, NEXT_VALUE_SEPARATOR def process_net_command(py_db, cmd_id, seq, text): @@ -220,6 +221,16 @@ def process_net_command(py_db, cmd_id, seq, text): except: traceback.print_exc() + elif cmd_id == CMD_LOAD_FULL_VALUE: + try: + thread_id, frame_id, scopeattrs = text.split('\t', 2) + vars = scopeattrs.split(NEXT_VALUE_SEPARATOR) + + int_cmd = InternalLoadFullValue(seq, thread_id, frame_id, vars) + py_db.post_internal_command(int_cmd, thread_id) + except: + traceback.print_exc() + elif cmd_id == CMD_GET_COMPLETIONS: # we received some command to get a variable # the text is: thread_id\tframe_id\tactivation token diff --git a/python/helpers/pydev/_pydevd_bundle/pydevd_resolver.py b/python/helpers/pydev/_pydevd_bundle/pydevd_resolver.py index dfe3855b0bd2..886e50d806a9 100644 --- a/python/helpers/pydev/_pydevd_bundle/pydevd_resolver.py +++ b/python/helpers/pydev/_pydevd_bundle/pydevd_resolver.py @@ -349,8 +349,8 @@ class SetResolver: d = {} i = 0 for item in var: - i+= 1 - d[id(item)] = item + i += 1 + d[str(id(item))] = item if i > MAX_ITEMS_TO_HANDLE: d[TOO_LARGE_ATTR] = TOO_LARGE_MSG diff --git a/python/helpers/pydev/_pydevd_bundle/pydevd_xml.py b/python/helpers/pydev/_pydevd_bundle/pydevd_xml.py index e6a08b417be5..638e685a7e3f 100644 --- a/python/helpers/pydev/_pydevd_bundle/pydevd_xml.py +++ b/python/helpers/pydev/_pydevd_bundle/pydevd_xml.py @@ -3,7 +3,8 @@ import traceback from _pydevd_bundle import pydevd_resolver import sys from _pydevd_bundle.pydevd_constants import dict_contains, dict_iter_items, dict_keys, IS_PY3K, \ - MAXIMUM_VARIABLE_REPRESENTATION_SIZE, RETURN_VALUES_DICT + IS_PY2, BUILTINS_MODULE_NAME, MAXIMUM_VARIABLE_REPRESENTATION_SIZE, RETURN_VALUES_DICT, LOAD_VALUES_ASYNC, \ + DEFAULT_VALUE from _pydev_bundle.pydev_imports import quote @@ -162,14 +163,22 @@ def get_type(o): return (type_object, type_name, pydevd_resolver.defaultResolver) -def return_values_from_dict_to_xml(return_dict): +def is_builtin(x): + return getattr(x, '__module__', None) == BUILTINS_MODULE_NAME + + +def should_evaluate_full_value(val): + return not LOAD_VALUES_ASYNC or (is_builtin(type(val)) and not isinstance(val, (list, tuple, dict))) + + +def return_values_from_dict_to_xml(return_dict, eval_full_val=True): res = "" for name, val in dict_iter_items(return_dict): - res += var_to_xml(val, name, additionalInXml=' isRetVal="True"') + res += var_to_xml(val, name, additionalInXml=' isRetVal="True"', evaluate_full_value=eval_full_val) return res -def frame_vars_to_xml(frame_f_locals, hidden_ns=None): +def frame_vars_to_xml(frame_f_locals, hidden_ns=None, dbg=None, thread_id=None, frame_id=None): """ dumps frame variables to XML """ @@ -184,13 +193,15 @@ def frame_vars_to_xml(frame_f_locals, hidden_ns=None): for k in keys: try: v = frame_f_locals[k] + eval_full_val = should_evaluate_full_value(v) + if k == RETURN_VALUES_DICT: - xml += return_values_from_dict_to_xml(v) + xml += return_values_from_dict_to_xml(v, eval_full_val=eval_full_val) else: if hidden_ns is not None and dict_contains(hidden_ns, k): - xml += var_to_xml(v, str(k), additionalInXml=' isIPythonHidden="True"') + xml += var_to_xml(v, str(k), additionalInXml=' isIPythonHidden="True"', evaluate_full_value=eval_full_val) else: - xml += var_to_xml(v, str(k)) + xml += var_to_xml(v, str(k), evaluate_full_value=eval_full_val) except Exception: traceback.print_exc() pydev_log.error("Unexpected error, recovered safely.\n") @@ -198,7 +209,7 @@ def frame_vars_to_xml(frame_f_locals, hidden_ns=None): return xml -def var_to_xml(val, name, doTrim=True, additionalInXml='', return_value=False, ipython_hidden=False): +def var_to_xml(val, name, doTrim=True, additionalInXml='', evaluate_full_value=True): """ single variable or dictionary to xml representation """ is_exception_on_eval = isinstance(val, ExceptionOnEvaluate) @@ -210,43 +221,45 @@ def var_to_xml(val, name, doTrim=True, additionalInXml='', return_value=False, i _type, typeName, resolver = get_type(v) type_qualifier = getattr(_type, "__module__", "") - do_not_call_value_str = resolver is not None and resolver.use_value_repr_instead_of_str - try: - if hasattr(v, '__class__'): - if v.__class__ == frame_type: - value = pydevd_resolver.frameResolver.get_frame_name(v) - - elif v.__class__ in (list, tuple): - if len(v) > 300: - value = '%s: %s' % (str(v.__class__), '' % (len(v),)) - else: - value = '%s: %s' % (str(v.__class__), v) - else: - try: - cName = str(v.__class__) - if cName.find('.') != -1: - cName = cName.split('.')[-1] - - elif cName.find("'") != -1: #does not have '.' (could be something like ) - cName = cName[cName.index("'") + 1:] - - if cName.endswith("'>"): - cName = cName[:-2] - except: - cName = str(v.__class__) - - if do_not_call_value_str: - value = '%s: %r' % (cName, v) - else: - value = '%s: %s' % (cName, v) - else: - value = str(v) - except: + if not evaluate_full_value: + value = DEFAULT_VALUE + else: try: - value = repr(v) + if hasattr(v, '__class__'): + if v.__class__ == frame_type: + value = pydevd_resolver.frameResolver.get_frame_name(v) + + elif v.__class__ in (list, tuple, dict): + if len(v) > 300: + value = '%s: %s' % (str(v.__class__), '' % (len(v),)) + else: + value = '%s: %s' % (str(v.__class__), v) + else: + try: + cName = str(v.__class__) + if cName.find('.') != -1: + cName = cName.split('.')[-1] + + elif cName.find("'") != -1: #does not have '.' (could be something like ) + cName = cName[cName.index("'") + 1:] + + if cName.endswith("'>"): + cName = cName[:-2] + except: + cName = str(v.__class__) + + if resolver is not None and resolver.use_value_repr_instead_of_str: + value = '%s: %r' % (cName, v) + else: + value = '%s: %s' % (cName, v) + else: + value = str(v) except: - value = 'Unable to get repr for %s' % v.__class__ + try: + value = repr(v) + except: + value = 'Unable to get repr for %s' % v.__class__ try: name = quote(name, '/>_= ') #TODO: Fix PY-5834 without using quote diff --git a/python/helpers/pydev/pydevconsole.py b/python/helpers/pydev/pydevconsole.py index b50f3d4e3b6e..5c94eea9681a 100644 --- a/python/helpers/pydev/pydevconsole.py +++ b/python/helpers/pydev/pydevconsole.py @@ -315,6 +315,7 @@ def start_console_server(host, port, interpreter): server.register_function(interpreter.getArray) server.register_function(interpreter.evaluate) server.register_function(interpreter.ShowConsole) + server.register_function(interpreter.loadFullValue) # Functions for GUI main loop integration server.register_function(interpreter.enableGui) diff --git a/python/pydevSrc/com/jetbrains/python/debugger/PyDebugValue.java b/python/pydevSrc/com/jetbrains/python/debugger/PyDebugValue.java index 546da2bd248d..98b18f992fb6 100644 --- a/python/pydevSrc/com/jetbrains/python/debugger/PyDebugValue.java +++ b/python/pydevSrc/com/jetbrains/python/debugger/PyDebugValue.java @@ -6,11 +6,14 @@ import com.intellij.icons.AllIcons; import com.intellij.openapi.application.ApplicationManager; import com.intellij.openapi.diagnostic.Logger; import com.intellij.xdebugger.frame.*; +import com.jetbrains.python.debugger.pydev.PyDebugCallback; import com.jetbrains.python.debugger.pydev.PyVariableLocator; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import javax.swing.*; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -25,31 +28,45 @@ public class PyDebugValue extends XNamedValue { public static final int MAX_VALUE = 256; public static final String RETURN_VALUES_PREFIX = "__pydevd_ret_val_dict"; + public static final String DEFAULT_VALUE_ASYNC = "__pydevd_value_async"; - private String myTempName = null; - private final String myType; - private final String myTypeQualifier; - private final String myValue; + private @Nullable String myTempName = null; + private final @Nullable String myType; + private final @Nullable String myTypeQualifier; + private @Nullable String myValue; private final boolean myContainer; private final boolean myIsReturnedVal; private final boolean myIsIPythonHidden; - private final PyDebugValue myParent; - private String myId = null; - - private final PyFrameAccessor myFrameAccessor; - - private PyVariableLocator myVariableLocator; - + private @Nullable PyDebugValue myParent; + private @Nullable String myId = null; + private boolean myLoadValueAsync; + private @NotNull PyFrameAccessor myFrameAccessor; + private @Nullable PyVariableLocator myVariableLocator; + private volatile @Nullable XValueNode myLastNode = null; private final boolean myErrorOnEval; - public PyDebugValue(@NotNull final String name, final String type, String typeQualifier, final String value, final boolean container, - boolean isReturnedVal, boolean isIPythonHidden, boolean errorOnEval, final PyFrameAccessor frameAccessor) { + public PyDebugValue(@NotNull final String name, + @Nullable final String type, + @Nullable String typeQualifier, + @Nullable final String value, + final boolean container, + boolean isReturnedVal, + boolean isIPythonHidden, + boolean errorOnEval, + @NotNull final PyFrameAccessor frameAccessor) { this(name, type, typeQualifier, value, container, isReturnedVal, isIPythonHidden, errorOnEval, null, frameAccessor); } - public PyDebugValue(@NotNull final String name, final String type, String typeQualifier, final String value, final boolean container, - boolean isReturnedVal, boolean isIPythonHidden, boolean errorOnEval, final PyDebugValue parent, - final PyFrameAccessor frameAccessor) { + public PyDebugValue(@NotNull final String name, + @Nullable final String type, + @Nullable String typeQualifier, + @Nullable final String value, + final boolean container, + boolean isReturnedVal, + boolean isIPythonHidden, + boolean errorOnEval, + @Nullable final PyDebugValue parent, + @NotNull final PyFrameAccessor frameAccessor) { super(name); myType = type; myTypeQualifier = Strings.isNullOrEmpty(typeQualifier) ? null : typeQualifier; @@ -60,20 +77,43 @@ public class PyDebugValue extends XNamedValue { myErrorOnEval = errorOnEval; myParent = parent; myFrameAccessor = frameAccessor; + myLoadValueAsync = false; + if (DEFAULT_VALUE_ASYNC.equals(myValue)) { + myLoadValueAsync = true; + setValue(" "); + } } + public PyDebugValue(@NotNull PyDebugValue value, @NotNull String newName) { + this(newName, value.getType(), value.getTypeQualifier(), value.getValue(), value.isContainer(), value.isReturnedVal(), + value.isIPythonHidden(), value.isErrorOnEval(), value.getParent(), value.getFrameAccessor()); + setLoadValueAsync(value.isLoadValueAsync()); + setTempName(value.getTempName()); + } + + public PyDebugValue(@NotNull PyDebugValue value) { + this(value, value.getName()); + } + + @Nullable public String getTempName() { return myTempName != null ? myTempName : myName; } - public void setTempName(String tempName) { + public void setTempName(@Nullable String tempName) { myTempName = tempName; } + @Nullable public String getType() { return myType; } + public void setValue(@Nullable String newValue) { + myValue = newValue; + } + + @Nullable public String getValue() { return myValue; } @@ -94,27 +134,42 @@ public class PyDebugValue extends XNamedValue { return myErrorOnEval; } - public PyDebugValue setParent(@Nullable PyDebugValue parent) { - return new PyDebugValue(myName, myType, myTypeQualifier, myValue, myContainer, myIsReturnedVal, myIsIPythonHidden, myErrorOnEval, - parent, myFrameAccessor); - } - + @Nullable public PyDebugValue getParent() { return myParent; } + public void setParent(@Nullable PyDebugValue parent) { + myParent = parent; + } + + @Nullable public PyDebugValue getTopParent() { return myParent == null ? this : myParent.getTopParent(); } + public boolean isLoadValueAsync() { + return myLoadValueAsync; + } + + public void setLoadValueAsync(boolean loadValueAsync) { + myLoadValueAsync = loadValueAsync; + } + + @Nullable + public XValueNode getLastNode() { + return myLastNode; + } + + @NotNull @Override public String getEvaluationExpression() { StringBuilder stringBuilder = new StringBuilder(); buildExpression(stringBuilder); - return stringBuilder.toString(); + return wrapWithPrefix(stringBuilder.toString()); } - void buildExpression(StringBuilder result) { + void buildExpression(@NotNull StringBuilder result) { if (myParent == null) { result.append(getTempName()); } @@ -139,10 +194,12 @@ public class PyDebugValue extends XNamedValue { } } + @NotNull public String getFullName() { return wrapWithPrefix(getName()); } + @NotNull private static String removeId(@NotNull String name) { if (name.indexOf('(') != -1) { name = name.substring(0, name.indexOf('(')).trim(); @@ -151,6 +208,7 @@ public class PyDebugValue extends XNamedValue { return name; } + @NotNull private static String removeLeadingZeros(@NotNull String name) { //bugs.python.org/issue15254: "0" prefix for octal while (name.length() > 1 && name.startsWith("0")) { @@ -159,25 +217,12 @@ public class PyDebugValue extends XNamedValue { return name; } - private static boolean isLen(String name) { + private static boolean isLen(@NotNull String name) { return "__len__".equals(name); } - private static boolean isCollection(@NotNull PyDebugValue parent) { - String type = parent.getType(); - return type.equals("dict") || type.equals("list"); - } - - private static String getChildNamePresentation(@NotNull PyDebugValue parent, @NotNull String childName) { - if (isCollection(parent)) { - return "[".concat(removeId(childName)).concat("]"); - } - else { - return ".".concat(childName); - } - } - - private String wrapWithPrefix(String name) { + @NotNull + private String wrapWithPrefix(@NotNull String name) { if (isReturnedVal()) { // return values are saved in dictionary on Python side, so the variable's name should be transformed return RETURN_VALUES_PREFIX + "[\"" + name + "\"]"; @@ -187,18 +232,6 @@ public class PyDebugValue extends XNamedValue { } } - private String getFullTreeName() { - String result = ""; - String curNodeName = myName; - PyDebugValue parent = myParent; - while (parent != null) { - result = getChildNamePresentation(parent, curNodeName).concat(result); - curNodeName = parent.getName(); - parent = parent.getParent(); - } - return wrapWithPrefix(curNodeName.concat(result)); - } - @Override public void computePresentation(@NotNull XValueNode node, @NotNull XValuePlace place) { String value = PyTypeHandler.format(this); @@ -209,13 +242,94 @@ public class PyDebugValue extends XNamedValue { node.setPresentation(getValueIcon(), myType, value, myContainer); } - private void setFullValueEvaluator(XValueNode node, String value) { - String treeName = getFullTreeName(); + public void updateNodeValueAfterLoading(@NotNull XValueNode node, @NotNull String value, @NotNull String linkText) { + node.setPresentation(getValueIcon(), myType, value, myContainer); + if (value.length() >= MAX_VALUE) { + node.setFullValueEvaluator(new PyFullValueEvaluator(myFrameAccessor, getEvaluationExpression())); + } + else { + node.setFullValueEvaluator(new XFullValueEvaluator() { + @Override + public void startEvaluation(@NotNull XFullValueEvaluationCallback callback) { + callback.evaluated(value); + } + + @Override + public String getLinkText() { + return linkText; + } + + @Override + public boolean isShowValuePopup() { + return false; + } + }); + } + } + + @NotNull + public PyDebugCallback createDebugValueCallback() { + return new PyDebugCallback() { + @Override + public void ok(String value) { + myLoadValueAsync = false; + myValue = value; + XValueNode node = myLastNode; + if (node != null && !node.isObsolete()) { + updateNodeValueAfterLoading(node, value, ""); + } + } + + @Override + public void error(PyDebuggerException exception) { + LOG.error(exception.getMessage()); + } + }; + } + + public boolean isNumericContainer() { + return EVALUATOR_POSTFIXES.get(myType) != null; + } + + @NotNull + public static List> getAsyncValuesFromChildren(@NotNull XValueChildrenList childrenList) { + List> variables = new ArrayList<>(); + for (int i = 0; i < childrenList.size(); i++) { + XValue value = childrenList.getValue(i); + if (value instanceof PyDebugValue) { + PyDebugValue debugValue = (PyDebugValue)value; + if (debugValue.isLoadValueAsync() && !debugValue.isNumericContainer()) { + variables.add(new PyFrameAccessor.PyAsyncValue<>(debugValue, debugValue.createDebugValueCallback())); + } + } + } + return variables; + } + + public static void getAsyncValues(@NotNull PyFrameAccessor frameAccessor, @NotNull XValueChildrenList childrenList) { + List> variables = getAsyncValuesFromChildren(childrenList); + int cores = Runtime.getRuntime().availableProcessors(); + int chunkSize = Math.max(1, variables.size() / cores); + int left = 0; + int right = Math.min(chunkSize, variables.size()); + while (left < variables.size()) { + frameAccessor.loadAsyncVariablesValues(variables.subList(left, right)); + left = right; + right = Math.min(right + chunkSize, variables.size()); + } + } + + private void setFullValueEvaluator(@NotNull XValueNode node, @NotNull String value) { + String treeName = getEvaluationExpression(); String postfix = EVALUATOR_POSTFIXES.get(myType); if (postfix == null) { if (value.length() >= MAX_VALUE) { node.setFullValueEvaluator(new PyFullValueEvaluator(myFrameAccessor, treeName)); } + if (myLoadValueAsync) { + node.setFullValueEvaluator(new PyLoadingValueEvaluator("... Loading Value", myFrameAccessor, treeName)); + myLastNode = node; + } return; } String linkText = "...View as " + postfix; @@ -226,12 +340,11 @@ public class PyDebugValue extends XNamedValue { public void computeChildren(@NotNull final XCompositeNode node) { if (node.isObsolete()) return; ApplicationManager.getApplication().executeOnPooledThread(() -> { - if (myFrameAccessor == null) return; - try { final XValueChildrenList values = myFrameAccessor.loadVariable(this); if (!node.isObsolete()) { node.addChildren(values, true); + getAsyncValues(myFrameAccessor, values); } } catch (PyDebuggerException e) { @@ -243,6 +356,7 @@ public class PyDebugValue extends XNamedValue { }); } + @NotNull @Override public XValueModifier getModifier() { return new PyValueModifier(myFrameAccessor, this); @@ -260,13 +374,6 @@ public class PyDebugValue extends XNamedValue { } } - public PyDebugValue setName(String newName) { - PyDebugValue value = new PyDebugValue(newName, myType, myTypeQualifier, myValue, myContainer, myIsReturnedVal, myIsIPythonHidden, - myErrorOnEval, myParent, myFrameAccessor); - value.setTempName(myTempName); - return value; - } - @Nullable @Override public XReferrersProvider getReferrersProvider() { @@ -282,23 +389,30 @@ public class PyDebugValue extends XNamedValue { } } + @NotNull public PyFrameAccessor getFrameAccessor() { return myFrameAccessor; } + public void setFrameAccessor(@NotNull PyFrameAccessor frameAccessor) { + myFrameAccessor = frameAccessor; + } + + @Nullable public PyVariableLocator getVariableLocator() { return myVariableLocator; } - public void setVariableLocator(PyVariableLocator variableLocator) { + public void setVariableLocator(@Nullable PyVariableLocator variableLocator) { myVariableLocator = variableLocator; } + @Nullable public String getId() { return myId; } - public void setId(String id) { + public void setId(@Nullable String id) { myId = id; } @@ -312,8 +426,7 @@ public class PyDebugValue extends XNamedValue { if (myParent == null) { navigatable.setSourcePosition(myFrameAccessor.getSourcePositionForName(myName, null)); } - else - { + else { navigatable.setSourcePosition(myFrameAccessor.getSourcePositionForName(myName, myParent.getDeclaringType())); } } @@ -324,32 +437,33 @@ public class PyDebugValue extends XNamedValue { } private static final Pattern IS_TYPE_DECLARATION = Pattern.compile("<(?:class|type)\\s*'(?.*?)'>"); + @Override public void computeTypeSourcePosition(@NotNull XNavigatable navigatable) { - String lookupType = getDeclaringType(); navigatable.setSourcePosition(myFrameAccessor.getSourcePositionForType(lookupType)); } + @Nullable protected final String getDeclaringType() { String lookupType = getQualifiedType(); - if (!Strings.isNullOrEmpty(myValue)) - { + if (!Strings.isNullOrEmpty(myValue)) { Matcher matcher = IS_TYPE_DECLARATION.matcher(myValue); - if (matcher.matches()) - { + if (matcher.matches()) { lookupType = matcher.group("TYPE"); } } return lookupType; } + @Nullable public String getQualifiedType() { if (Strings.isNullOrEmpty(myType)) return null; return (myTypeQualifier == null) ? myType : (myTypeQualifier + "." + myType); } + @Nullable public String getTypeQualifier() { return myTypeQualifier; } diff --git a/python/pydevSrc/com/jetbrains/python/debugger/PyFrameAccessor.java b/python/pydevSrc/com/jetbrains/python/debugger/PyFrameAccessor.java index b75e2d374eff..afae380d7b92 100644 --- a/python/pydevSrc/com/jetbrains/python/debugger/PyFrameAccessor.java +++ b/python/pydevSrc/com/jetbrains/python/debugger/PyFrameAccessor.java @@ -2,9 +2,12 @@ package com.jetbrains.python.debugger; import com.intellij.xdebugger.XSourcePosition; import com.intellij.xdebugger.frame.XValueChildrenList; +import com.jetbrains.python.debugger.pydev.PyDebugCallback; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import java.util.List; + /** * Facade to access python variables frame * @@ -26,12 +29,38 @@ public interface PyFrameAccessor { ArrayChunk getArrayItems(PyDebugValue var, int rowOffset, int colOffset, int rows, int cols, String format) throws PyDebuggerException; @Nullable - XSourcePosition getSourcePositionForName(String name, String parentType); + XSourcePosition getSourcePositionForName(@Nullable String name, @Nullable String parentType); @Nullable XSourcePosition getSourcePositionForType(String type); - default void showNumericContainer(PyDebugValue value) {} + default void showNumericContainer(@NotNull PyDebugValue value) {} default void addFrameListener(@NotNull PyFrameListener listener) {} + + default void loadAsyncVariablesValues(@NotNull final List> pyAsyncValues) {} + + default boolean isCurrentFrameCached() { + return false; + } + + class PyAsyncValue { + private final @NotNull PyDebugValue myDebugValue; + private final @NotNull PyDebugCallback myCallback; + + public PyAsyncValue(@NotNull PyDebugValue debugValue, @NotNull PyDebugCallback callback) { + myDebugValue = debugValue; + myCallback = callback; + } + + @NotNull + public PyDebugValue getDebugValue() { + return myDebugValue; + } + + @NotNull + public PyDebugCallback getCallback() { + return myCallback; + } + } } diff --git a/python/pydevSrc/com/jetbrains/python/debugger/PyFullValueEvaluator.java b/python/pydevSrc/com/jetbrains/python/debugger/PyFullValueEvaluator.java index fc3cbd77b55a..a1ab5b616427 100644 --- a/python/pydevSrc/com/jetbrains/python/debugger/PyFullValueEvaluator.java +++ b/python/pydevSrc/com/jetbrains/python/debugger/PyFullValueEvaluator.java @@ -1,5 +1,6 @@ package com.jetbrains.python.debugger; +import com.intellij.openapi.application.ApplicationManager; import com.intellij.xdebugger.frame.XFullValueEvaluator; import org.jetbrains.annotations.NotNull; @@ -36,14 +37,19 @@ public class PyFullValueEvaluator extends XFullValueEvaluator { return; } - try { - final PyDebugValue value = myDebugProcess.evaluate(expression, false, false); - callback.evaluated(value.getValue()); - showCustomPopup(myDebugProcess, value); - } - catch (PyDebuggerException e) { - callback.errorOccurred(e.getTracebackError()); - } + ApplicationManager.getApplication().executeOnPooledThread(() -> { + try { + final PyDebugValue value = myDebugProcess.evaluate(expression, false, false); + if (value.getValue() == null) { + throw new PyDebuggerException("Failed to Load Value"); + } + callback.evaluated(value.getValue()); + ApplicationManager.getApplication().invokeLater(() -> showCustomPopup(myDebugProcess, value)); + } + catch (PyDebuggerException e) { + callback.errorOccurred(e.getTracebackError()); + } + }); } protected void showCustomPopup(PyFrameAccessor debugProcess, PyDebugValue debugValue) { diff --git a/python/pydevSrc/com/jetbrains/python/debugger/PyLoadingValueEvaluator.java b/python/pydevSrc/com/jetbrains/python/debugger/PyLoadingValueEvaluator.java new file mode 100644 index 000000000000..a80db51ca79f --- /dev/null +++ b/python/pydevSrc/com/jetbrains/python/debugger/PyLoadingValueEvaluator.java @@ -0,0 +1,19 @@ +package com.jetbrains.python.debugger; + +import org.jetbrains.annotations.NotNull; + +public class PyLoadingValueEvaluator extends PyFullValueEvaluator { + protected PyLoadingValueEvaluator(@NotNull String linkText, @NotNull PyFrameAccessor debugProcess, @NotNull String expression) { + super(linkText, debugProcess, expression); + } + + @Override + public void startEvaluation(@NotNull XFullValueEvaluationCallback callback) { + callback.evaluated("... Loading value"); + } + + @Override + public boolean isShowValuePopup() { + return false; + } +} diff --git a/python/pydevSrc/com/jetbrains/python/debugger/pydev/AbstractCommand.java b/python/pydevSrc/com/jetbrains/python/debugger/pydev/AbstractCommand.java index 884398894109..0ff32f25ff05 100644 --- a/python/pydevSrc/com/jetbrains/python/debugger/pydev/AbstractCommand.java +++ b/python/pydevSrc/com/jetbrains/python/debugger/pydev/AbstractCommand.java @@ -57,6 +57,7 @@ public abstract class AbstractCommand { public static final int PROCESS_CREATED = 149; public static final int SHOW_CYTHON_WARNING = 150; + public static final int LOAD_FULL_VALUE = 151; public static final int ERROR = 901; @@ -188,7 +189,7 @@ public abstract class AbstractCommand { } - protected void processResponse(final ProtocolFrame response) throws PyDebuggerException { + protected void processResponse(@NotNull final ProtocolFrame response) throws PyDebuggerException { if (response.getCommand() >= 900 && response.getCommand() < 1000) { throw new PyDebuggerException(response.getPayload()); } diff --git a/python/pydevSrc/com/jetbrains/python/debugger/pydev/ChangeVariableCommand.java b/python/pydevSrc/com/jetbrains/python/debugger/pydev/ChangeVariableCommand.java index c9f53babdff0..d959267dec73 100644 --- a/python/pydevSrc/com/jetbrains/python/debugger/pydev/ChangeVariableCommand.java +++ b/python/pydevSrc/com/jetbrains/python/debugger/pydev/ChangeVariableCommand.java @@ -4,6 +4,7 @@ package com.jetbrains.python.debugger.pydev; import com.jetbrains.python.debugger.IPyDebugProcess; import com.jetbrains.python.debugger.PyDebugValue; import com.jetbrains.python.debugger.PyDebuggerException; +import org.jetbrains.annotations.NotNull; public class ChangeVariableCommand extends AbstractFrameCommand { @@ -32,9 +33,10 @@ public class ChangeVariableCommand extends AbstractFrameCommand { return true; } - protected void processResponse(final ProtocolFrame response) throws PyDebuggerException { + protected void processResponse(@NotNull final ProtocolFrame response) throws PyDebuggerException { super.processResponse(response); - myNewValue = ProtocolParser.parseValue(response.getPayload(), myDebugProcess).setName(myVariableName); + PyDebugValue returnedValue = ProtocolParser.parseValue(response.getPayload(), myDebugProcess); + myNewValue = new PyDebugValue(returnedValue, myVariableName); } public PyDebugValue getNewValue() { diff --git a/python/pydevSrc/com/jetbrains/python/debugger/pydev/ClientModeMultiProcessDebugger.java b/python/pydevSrc/com/jetbrains/python/debugger/pydev/ClientModeMultiProcessDebugger.java index b8139653edf3..05acb5440416 100644 --- a/python/pydevSrc/com/jetbrains/python/debugger/pydev/ClientModeMultiProcessDebugger.java +++ b/python/pydevSrc/com/jetbrains/python/debugger/pydev/ClientModeMultiProcessDebugger.java @@ -187,6 +187,13 @@ public class ClientModeMultiProcessDebugger implements ProcessDebugger { return debugger(threadId).changeVariable(threadId, frameId, var, value); } + @Override + public void loadFullVariableValues(@NotNull String threadId, + @NotNull String frameId, + @NotNull List> vars) throws PyDebuggerException { + debugger(threadId).loadFullVariableValues(threadId, frameId, vars); + } + @Override public String loadSource(String path) { return myMainDebugger.loadSource(path); diff --git a/python/pydevSrc/com/jetbrains/python/debugger/pydev/EvaluateCommand.java b/python/pydevSrc/com/jetbrains/python/debugger/pydev/EvaluateCommand.java index 373c02a28c05..8895d665a580 100644 --- a/python/pydevSrc/com/jetbrains/python/debugger/pydev/EvaluateCommand.java +++ b/python/pydevSrc/com/jetbrains/python/debugger/pydev/EvaluateCommand.java @@ -3,6 +3,7 @@ package com.jetbrains.python.debugger.pydev; import com.jetbrains.python.debugger.IPyDebugProcess; import com.jetbrains.python.debugger.PyDebugValue; import com.jetbrains.python.debugger.PyDebuggerException; +import org.jetbrains.annotations.NotNull; public class EvaluateCommand extends AbstractFrameCommand { @@ -37,10 +38,10 @@ public class EvaluateCommand extends AbstractFrameCommand { } @Override - protected void processResponse(final ProtocolFrame response) throws PyDebuggerException { + protected void processResponse(@NotNull final ProtocolFrame response) throws PyDebuggerException { super.processResponse(response); final PyDebugValue value = ProtocolParser.parseValue(response.getPayload(), myDebugProcess); - myValue = value.setName((myExecute ? "" : myExpression)); + myValue = new PyDebugValue(value, myExecute ? "" : myExpression); if (!myTempName.isEmpty()) { myValue.setTempName(myTempName); } diff --git a/python/pydevSrc/com/jetbrains/python/debugger/pydev/GetArrayCommand.java b/python/pydevSrc/com/jetbrains/python/debugger/pydev/GetArrayCommand.java index da19bf2381a6..1487d1ad32a5 100644 --- a/python/pydevSrc/com/jetbrains/python/debugger/pydev/GetArrayCommand.java +++ b/python/pydevSrc/com/jetbrains/python/debugger/pydev/GetArrayCommand.java @@ -3,6 +3,7 @@ package com.jetbrains.python.debugger.pydev; import com.jetbrains.python.debugger.ArrayChunk; import com.jetbrains.python.debugger.PyDebugValue; import com.jetbrains.python.debugger.PyDebuggerException; +import org.jetbrains.annotations.NotNull; /** * @author amarch @@ -51,7 +52,7 @@ public class GetArrayCommand extends GetFrameCommand { } @Override - protected void processResponse(final ProtocolFrame response) throws PyDebuggerException { + protected void processResponse(@NotNull final ProtocolFrame response) throws PyDebuggerException { if (response.getCommand() >= 900 && response.getCommand() < 1000) { throw new PyDebuggerException(response.getPayload()); } diff --git a/python/pydevSrc/com/jetbrains/python/debugger/pydev/GetCompletionsCommand.java b/python/pydevSrc/com/jetbrains/python/debugger/pydev/GetCompletionsCommand.java index e7e56653670c..35d3bbf86f40 100644 --- a/python/pydevSrc/com/jetbrains/python/debugger/pydev/GetCompletionsCommand.java +++ b/python/pydevSrc/com/jetbrains/python/debugger/pydev/GetCompletionsCommand.java @@ -31,7 +31,7 @@ public class GetCompletionsCommand extends AbstractFrameCommand { } @Override - protected void processResponse(ProtocolFrame response) throws PyDebuggerException { + protected void processResponse(@NotNull ProtocolFrame response) throws PyDebuggerException { super.processResponse(response); try { myCompletions = PydevXmlUtils.xmlToCompletions(response.getPayload(), myActionToken); diff --git a/python/pydevSrc/com/jetbrains/python/debugger/pydev/GetDescriptionCommand.java b/python/pydevSrc/com/jetbrains/python/debugger/pydev/GetDescriptionCommand.java index ca216c7eb02e..07b53bd4d09f 100644 --- a/python/pydevSrc/com/jetbrains/python/debugger/pydev/GetDescriptionCommand.java +++ b/python/pydevSrc/com/jetbrains/python/debugger/pydev/GetDescriptionCommand.java @@ -2,6 +2,7 @@ package com.jetbrains.python.debugger.pydev; import com.jetbrains.python.debugger.PyDebugValue; import com.jetbrains.python.debugger.PyDebuggerException; +import org.jetbrains.annotations.NotNull; /** * @author traff @@ -23,7 +24,7 @@ public class GetDescriptionCommand extends AbstractFrameCommand { } @Override - protected void processResponse(ProtocolFrame response) throws PyDebuggerException { + protected void processResponse(@NotNull ProtocolFrame response) throws PyDebuggerException { super.processResponse(response); try { PyDebugValue pyDebugValue = ProtocolParser.parseValue(response.getPayload(), getDebugger().getDebugProcess()); diff --git a/python/pydevSrc/com/jetbrains/python/debugger/pydev/GetFrameCommand.java b/python/pydevSrc/com/jetbrains/python/debugger/pydev/GetFrameCommand.java index e9ac6b810f01..a520f42e24ca 100644 --- a/python/pydevSrc/com/jetbrains/python/debugger/pydev/GetFrameCommand.java +++ b/python/pydevSrc/com/jetbrains/python/debugger/pydev/GetFrameCommand.java @@ -4,6 +4,7 @@ import com.intellij.xdebugger.frame.XValueChildrenList; import com.jetbrains.python.debugger.IPyDebugProcess; import com.jetbrains.python.debugger.PyDebugValue; import com.jetbrains.python.debugger.PyDebuggerException; +import org.jetbrains.annotations.NotNull; import java.util.List; @@ -34,7 +35,7 @@ public class GetFrameCommand extends AbstractFrameCommand { } @Override - protected void processResponse(final ProtocolFrame response) throws PyDebuggerException { + protected void processResponse(@NotNull final ProtocolFrame response) throws PyDebuggerException { super.processResponse(response); final List values = ProtocolParser.parseValues(response.getPayload(), myDebugProcess); myFrameVariables = new XValueChildrenList(values.size()); @@ -47,8 +48,10 @@ public class GetFrameCommand extends AbstractFrameCommand { } protected PyDebugValue extend(final PyDebugValue value) { - return new PyDebugValue(value.getName(), value.getType(), value.getTypeQualifier(), value.getValue(), value.isContainer(), - value.isReturnedVal(), value.isIPythonHidden(), value.isErrorOnEval(), null, myDebugProcess); + PyDebugValue debugValue = new PyDebugValue(value); + debugValue.setParent(null); + debugValue.setFrameAccessor(myDebugProcess); + return debugValue; } public XValueChildrenList getVariables() { diff --git a/python/pydevSrc/com/jetbrains/python/debugger/pydev/GetVariableCommand.java b/python/pydevSrc/com/jetbrains/python/debugger/pydev/GetVariableCommand.java index 6acf73d8dd0a..506e52438732 100644 --- a/python/pydevSrc/com/jetbrains/python/debugger/pydev/GetVariableCommand.java +++ b/python/pydevSrc/com/jetbrains/python/debugger/pydev/GetVariableCommand.java @@ -50,8 +50,9 @@ public class GetVariableCommand extends GetFrameCommand { @Override protected PyDebugValue extend(final PyDebugValue value) { - return new PyDebugValue(value.getName(), value.getType(), value.getTypeQualifier(), value.getValue(), value.isContainer(), - value.isReturnedVal(), value.isIPythonHidden(), value.isErrorOnEval(), myParent, - myDebugProcess); + PyDebugValue debugValue = new PyDebugValue(value); + debugValue.setParent(myParent); + debugValue.setFrameAccessor(myDebugProcess); + return debugValue; } } diff --git a/python/pydevSrc/com/jetbrains/python/debugger/pydev/LoadFullValueCommand.java b/python/pydevSrc/com/jetbrains/python/debugger/pydev/LoadFullValueCommand.java new file mode 100644 index 000000000000..eef864d0c64d --- /dev/null +++ b/python/pydevSrc/com/jetbrains/python/debugger/pydev/LoadFullValueCommand.java @@ -0,0 +1,72 @@ +package com.jetbrains.python.debugger.pydev; + +import com.jetbrains.python.debugger.IPyDebugProcess; +import com.jetbrains.python.debugger.PyDebugValue; +import com.jetbrains.python.debugger.PyDebuggerException; +import com.jetbrains.python.debugger.PyFrameAccessor; +import org.jetbrains.annotations.NotNull; + +import java.util.List; + + +public class LoadFullValueCommand extends AbstractFrameCommand { + private static String NEXT_VALUE_SEPARATOR = "__pydev_val__"; + private final @NotNull IPyDebugProcess myDebugProcess; + private final @NotNull List> myVars; + + public LoadFullValueCommand(final @NotNull RemoteDebugger debugger, + final @NotNull String threadId, + final @NotNull String frameId, + final @NotNull List> vars) { + super(debugger, LOAD_FULL_VALUE, threadId, frameId); + myDebugProcess = debugger.getDebugProcess(); + myVars = vars; + } + + @Override + public boolean isResponseExpected() { + return true; + } + + @Override + protected void processResponse(@NotNull ProtocolFrame response) throws PyDebuggerException { + super.processResponse(response); + try { + List debugValues = ProtocolParser.parseValues(response.getPayload(), myDebugProcess); + for (int i = 0; i < myVars.size(); ++i) { + PyDebugValue resultValue = debugValues.get(i); + myVars.get(i).getCallback().ok(resultValue.getValue()); + } + } + catch (Exception e) { + for (PyFrameAccessor.PyAsyncValue vars : myVars) { + vars.getCallback().error(new PyDebuggerException(response.getPayload())); + } + } + } + + @NotNull + private String buildPayloadForVar(@NotNull PyDebugValue var) { + StringBuilder sb = new StringBuilder(); + String varName = GetVariableCommand.composeName(var); + if (var.getVariableLocator() != null) { + sb.append(var.getVariableLocator().getThreadId()).append(var.getVariableLocator().getPyDBLocation()); + } + else if (varName.contains(GetVariableCommand.BY_ID)) { + sb.append(getThreadId()).append(varName); + } + else { + sb.append(varName); + } + return sb.toString(); + } + + @Override + protected void buildPayload(Payload payload) { + super.buildPayload(payload); + for (PyFrameAccessor.PyAsyncValue var : myVars) { + PyDebugValue debugValue = var.getDebugValue(); + payload.add("FRAME").add(buildPayloadForVar(debugValue)).add(NEXT_VALUE_SEPARATOR); + } + } +} diff --git a/python/pydevSrc/com/jetbrains/python/debugger/pydev/LoadSourceCommand.java b/python/pydevSrc/com/jetbrains/python/debugger/pydev/LoadSourceCommand.java index cdafeae93a52..32581214cda2 100644 --- a/python/pydevSrc/com/jetbrains/python/debugger/pydev/LoadSourceCommand.java +++ b/python/pydevSrc/com/jetbrains/python/debugger/pydev/LoadSourceCommand.java @@ -22,7 +22,7 @@ public class LoadSourceCommand extends AbstractCommand { } @Override - protected void processResponse(final ProtocolFrame response) throws PyDebuggerException { + protected void processResponse(@NotNull final ProtocolFrame response) throws PyDebuggerException { super.processResponse(response); myContent = ProtocolParser.parseSourceContent(response.getPayload()); } diff --git a/python/pydevSrc/com/jetbrains/python/debugger/pydev/MultiProcessDebugger.java b/python/pydevSrc/com/jetbrains/python/debugger/pydev/MultiProcessDebugger.java index 79e17ee5983b..d4644f66a0a1 100644 --- a/python/pydevSrc/com/jetbrains/python/debugger/pydev/MultiProcessDebugger.java +++ b/python/pydevSrc/com/jetbrains/python/debugger/pydev/MultiProcessDebugger.java @@ -224,6 +224,13 @@ public class MultiProcessDebugger implements ProcessDebugger { return debugger(threadId).changeVariable(threadId, frameId, var, value); } + @Override + public void loadFullVariableValues(@NotNull String threadId, + @NotNull String frameId, + @NotNull List> vars) throws PyDebuggerException { + debugger(threadId).loadFullVariableValues(threadId, frameId, vars); + } + @Override public String loadSource(String path) { return myMainDebugger.loadSource(path); diff --git a/python/pydevSrc/com/jetbrains/python/debugger/pydev/ProcessDebugger.java b/python/pydevSrc/com/jetbrains/python/debugger/pydev/ProcessDebugger.java index f21f99c9b667..05518561208e 100644 --- a/python/pydevSrc/com/jetbrains/python/debugger/pydev/ProcessDebugger.java +++ b/python/pydevSrc/com/jetbrains/python/debugger/pydev/ProcessDebugger.java @@ -50,6 +50,11 @@ public interface ProcessDebugger { PyDebugValue changeVariable(String threadId, String frameId, PyDebugValue var, String value) throws PyDebuggerException; + void loadFullVariableValues(@NotNull String threadId, + @NotNull String frameId, + @NotNull List> vars) + throws PyDebuggerException; + @Nullable String loadSource(String path); diff --git a/python/pydevSrc/com/jetbrains/python/debugger/pydev/RemoteDebugger.java b/python/pydevSrc/com/jetbrains/python/debugger/pydev/RemoteDebugger.java index 3edc37ac5a52..bdbb27b99405 100644 --- a/python/pydevSrc/com/jetbrains/python/debugger/pydev/RemoteDebugger.java +++ b/python/pydevSrc/com/jetbrains/python/debugger/pydev/RemoteDebugger.java @@ -205,6 +205,13 @@ public class RemoteDebugger implements ProcessDebugger { return command.getNewValue(); } + public void loadFullVariableValues(@NotNull String threadId, + @NotNull String frameId, + @NotNull List> vars) throws PyDebuggerException { + final LoadFullValueCommand command = new LoadFullValueCommand(this, threadId, frameId, vars); + command.execute(); + } + @Override @Nullable public String loadSource(String path) { diff --git a/python/pydevSrc/com/jetbrains/python/debugger/pydev/SetNextStatementCommand.java b/python/pydevSrc/com/jetbrains/python/debugger/pydev/SetNextStatementCommand.java index 82d59cba8f24..0339c5b538f2 100644 --- a/python/pydevSrc/com/jetbrains/python/debugger/pydev/SetNextStatementCommand.java +++ b/python/pydevSrc/com/jetbrains/python/debugger/pydev/SetNextStatementCommand.java @@ -28,7 +28,7 @@ public class SetNextStatementCommand extends AbstractThreadCommand { } @Override - protected void processResponse(ProtocolFrame response) throws PyDebuggerException { + protected void processResponse(@NotNull ProtocolFrame response) throws PyDebuggerException { super.processResponse(response); try { Pair result = ProtocolParser.parseSetNextStatementCommand(response.getPayload()); diff --git a/python/pydevSrc/com/jetbrains/python/debugger/pydev/VersionCommand.java b/python/pydevSrc/com/jetbrains/python/debugger/pydev/VersionCommand.java index a458954374af..a5fe3e5bd50a 100644 --- a/python/pydevSrc/com/jetbrains/python/debugger/pydev/VersionCommand.java +++ b/python/pydevSrc/com/jetbrains/python/debugger/pydev/VersionCommand.java @@ -1,6 +1,7 @@ package com.jetbrains.python.debugger.pydev; import com.jetbrains.python.debugger.PyDebuggerException; +import org.jetbrains.annotations.NotNull; public class VersionCommand extends AbstractCommand { @@ -25,7 +26,7 @@ public class VersionCommand extends AbstractCommand { } @Override - protected void processResponse(final ProtocolFrame response) throws PyDebuggerException { + protected void processResponse(@NotNull final ProtocolFrame response) throws PyDebuggerException { super.processResponse(response); myRemoteVersion = response.getPayload(); } diff --git a/python/src/com/jetbrains/python/console/PydevConsoleCommunication.java b/python/src/com/jetbrains/python/console/PydevConsoleCommunication.java index ca7a1072a0f8..fafc9e7479cf 100644 --- a/python/src/com/jetbrains/python/console/PydevConsoleCommunication.java +++ b/python/src/com/jetbrains/python/console/PydevConsoleCommunication.java @@ -32,6 +32,7 @@ import com.intellij.util.WaitFor; import com.intellij.util.containers.ContainerUtil; import com.intellij.xdebugger.XSourcePosition; import com.intellij.xdebugger.frame.XValueChildrenList; +import com.intellij.xdebugger.frame.XValueNode; import com.jetbrains.python.console.parsing.PythonConsoleData; import com.jetbrains.python.console.pydev.*; import com.jetbrains.python.debugger.*; @@ -67,6 +68,7 @@ public class PydevConsoleCommunication extends AbstractConsoleCommunication impl private static final String CLOSE = "close"; private static final String EVALUATE = "evaluate"; private static final String GET_ARRAY = "getArray"; + private static final String LOAD_FULL_VALUE = "loadFullValue"; private static final String PYDEVD_EXTRA_ENVS = "PYDEVD_EXTRA_ENVS"; /** @@ -519,11 +521,63 @@ public class PydevConsoleCommunication extends AbstractConsoleCommunication impl return new XValueChildrenList(); } + @Override + public void loadAsyncVariablesValues(@NotNull List> pyAsyncValues) { + ApplicationManager.getApplication().executeOnPooledThread(() -> { + if (myClient != null) { + try { + List evaluationExpressions = new ArrayList<>(); + for (PyAsyncValue asyncValue : pyAsyncValues) { + evaluationExpressions.add(asyncValue.getDebugValue().getEvaluationExpression()); + } + Object ret = myClient.execute(LOAD_FULL_VALUE, new Object[]{evaluationExpressions.toArray()}); + + if (ret instanceof String) { + List debugValues = ProtocolParser.parseValues((String)ret, this); + for (int i = 0; i < pyAsyncValues.size(); ++i) { + pyAsyncValues.get(i).getCallback().ok(debugValues.get(i).getValue()); + } + } + else { + checkError(ret); + } + } + catch (PyDebuggerException e) { + if (myWebServer != null) { + LOG.error(e); + } + } + catch (XmlRpcException e) { + for (PyAsyncValue asyncValue : pyAsyncValues) { + PyDebugValue value = asyncValue.getDebugValue(); + XValueNode node = value.getLastNode(); + if (node != null && !node.isObsolete()) { + if (e.getMessage().startsWith("Timeout") || e.getMessage().startsWith("Console already exited")) { + value.updateNodeValueAfterLoading(node, " ", "Timeout Exceeded"); + } + else { + LOG.error(e); + } + } + } + } + } + }); + } + private XValueChildrenList parseVars(String ret, PyDebugValue parent) throws PyDebuggerException { final List values = ProtocolParser.parseValues(ret, this); XValueChildrenList list = new XValueChildrenList(values.size()); for (PyDebugValue v : values) { - list.add(v.getName(), parent != null ? v.setParent(parent) : v); + PyDebugValue value; + if (parent != null) { + value = new PyDebugValue(v); + value.setParent(parent); + } + else { + value = v; + } + list.add(v.getName(), value); } return list; } @@ -689,7 +743,7 @@ public class PydevConsoleCommunication extends AbstractConsoleCommunication impl } @Override - public void showNumericContainer(PyDebugValue value) { + public void showNumericContainer(@NotNull PyDebugValue value) { PyViewNumericContainerAction.showNumericViewer(myProject, value); } diff --git a/python/src/com/jetbrains/python/console/PydevConsoleRunnerImpl.java b/python/src/com/jetbrains/python/console/PydevConsoleRunnerImpl.java index 053a755ae90c..cadd8b1d6db5 100644 --- a/python/src/com/jetbrains/python/console/PydevConsoleRunnerImpl.java +++ b/python/src/com/jetbrains/python/console/PydevConsoleRunnerImpl.java @@ -94,6 +94,8 @@ import com.jetbrains.python.console.actions.ShowVarsAction; import com.jetbrains.python.console.pydev.ConsoleCommunicationListener; import com.jetbrains.python.debugger.PyDebugRunner; import com.jetbrains.python.debugger.PySourcePosition; +import com.jetbrains.python.debugger.PyVariableViewSettings; +import com.jetbrains.python.debugger.settings.PyDebuggerSettings; import com.jetbrains.python.remote.PyRemotePathMapper; import com.jetbrains.python.remote.PyRemoteProcessHandlerBase; import com.jetbrains.python.remote.PyRemoteSdkAdditionalDataBase; @@ -223,6 +225,13 @@ public class PydevConsoleRunnerImpl implements PydevConsoleRunner { toolbarActions.add(new ConnectDebuggerAction()); + DefaultActionGroup settings = new DefaultActionGroup("Settings", true); + settings.getTemplatePresentation().setIcon(AllIcons.General.SecondaryGroup); + settings.add(new PyVariableViewSettings.SimplifiedView(null)); + settings.add(new PyVariableViewSettings.AsyncView()); + + toolbarActions.add(settings); + toolbarActions.add(new NewConsoleAction()); return actions; @@ -1111,6 +1120,10 @@ public class PydevConsoleRunnerImpl implements PydevConsoleRunner { mySdk = sdk; myEnvironmentVariables = envs; myEnvironmentVariables.putAll(consoleSettings.getEnvs()); + PyDebuggerSettings debuggerSettings = PyDebuggerSettings.getInstance(); + if (debuggerSettings.isLoadValuesAsync()) { + myEnvironmentVariables.put(PyVariableViewSettings.PYDEVD_LOAD_VALUES_ASYNC, "True"); + } } @Override diff --git a/python/src/com/jetbrains/python/debugger/PyDebugProcess.java b/python/src/com/jetbrains/python/debugger/PyDebugProcess.java index 79210a7ee233..7a2b8be1713d 100644 --- a/python/src/com/jetbrains/python/debugger/PyDebugProcess.java +++ b/python/src/com/jetbrains/python/debugger/PyDebugProcess.java @@ -57,10 +57,7 @@ import com.intellij.util.ui.UIUtil; import com.intellij.xdebugger.*; import com.intellij.xdebugger.breakpoints.*; import com.intellij.xdebugger.evaluation.XDebuggerEditorsProvider; -import com.intellij.xdebugger.frame.XExecutionStack; -import com.intellij.xdebugger.frame.XStackFrame; -import com.intellij.xdebugger.frame.XSuspendContext; -import com.intellij.xdebugger.frame.XValueChildrenList; +import com.intellij.xdebugger.frame.*; import com.intellij.xdebugger.stepping.XSmartStepIntoHandler; import com.jetbrains.python.PythonFileType; import com.jetbrains.python.console.PythonConsoleView; @@ -105,7 +102,8 @@ public class PyDebugProcess extends XDebugProcess implements IPyDebugProcess, Pr new ConcurrentHashMap<>(); private final List mySuspendedThreads = Collections.synchronizedList(Lists.newArrayList()); - private final Map myStackFrameCache = Maps.newHashMap(); + private final Map myStackFrameCache = Maps.newConcurrentMap(); + private final Object myFrameCacheObject = new Object(); private final Map myNewVariableValue = Maps.newHashMap(); private boolean myDownloadSources = false; @@ -451,7 +449,8 @@ public class PyDebugProcess extends XDebugProcess implements IPyDebugProcess, Pr @NotNull DefaultActionGroup settings) { super.registerAdditionalActions(leftToolbar, topToolbar, settings); settings.add(new WatchReturnValuesAction(this)); - settings.add(new SimplifiedView(this)); + settings.add(new PyVariableViewSettings.SimplifiedView(this)); + settings.add(new PyVariableViewSettings.AsyncView()); } private static class WatchReturnValuesAction extends ToggleAction { @@ -491,38 +490,6 @@ public class PyDebugProcess extends XDebugProcess implements IPyDebugProcess, Pr } } - private static class SimplifiedView extends ToggleAction { - private volatile boolean mySimplifiedView; - private final PyDebugProcess myProcess; - private final String myText; - - public SimplifiedView(@NotNull PyDebugProcess debugProcess) { - super("", "Disables watching classes, functions and modules objects", null); - mySimplifiedView = PyDebuggerSettings.getInstance().isSimplifiedView(); - myProcess = debugProcess; - myText = "Simplified Variables View"; - } - - @Override - public void update(@NotNull final AnActionEvent e) { - super.update(e); - final Presentation presentation = e.getPresentation(); - presentation.setEnabled(true); - presentation.setText(myText); - } - - @Override - public boolean isSelected(AnActionEvent e) { - return mySimplifiedView; - } - - @Override - public void setSelected(AnActionEvent e, boolean hide) { - mySimplifiedView = hide; - PyDebuggerSettings.getInstance().setSimplifiedView(hide); - myProcess.getSession().rebuildViews(); - } - } public void setShowReturnValues(boolean showReturnValues) { myDebugger.setShowReturnValues(showReturnValues); @@ -732,18 +699,67 @@ public class PyDebugProcess extends XDebugProcess implements IPyDebugProcess, Pr } } + @Override + public boolean isCurrentFrameCached() { + try { + synchronized (myFrameCacheObject) { + final PyStackFrame frame = currentFrame(); + return myStackFrameCache.containsKey(frame.getThreadFrameId()); + } + } + catch (PyDebuggerException e) { + LOG.warn(e); + } + return false; + } + @Override @Nullable public XValueChildrenList loadFrame() throws PyDebuggerException { final PyStackFrame frame = currentFrame(); - //do not reload frame every time it is needed, because due to bug in pdb, reloading frame clears all variable changes - if (!myStackFrameCache.containsKey(frame.getThreadFrameId())) { - XValueChildrenList values = myDebugger.loadFrame(frame.getThreadId(), frame.getFrameId()); - myStackFrameCache.put(frame.getThreadFrameId(), values); + synchronized (myFrameCacheObject) { + //do not reload frame every time it is needed, because due to bug in pdb, reloading frame clears all variable changes + if (!myStackFrameCache.containsKey(frame.getThreadFrameId())) { + XValueChildrenList values = myDebugger.loadFrame(frame.getThreadId(), frame.getFrameId()); + myStackFrameCache.put(frame.getThreadFrameId(), values); + } } return applyNewValue(myStackFrameCache.get(frame.getThreadFrameId()), frame.getThreadFrameId()); } + public void loadAsyncVariablesValues(@NotNull final List> pyAsyncValues) { + ApplicationManager.getApplication().executeOnPooledThread(() -> { + try { + if (isConnected()) { + final PyStackFrame frame = currentFrame(); + XSuspendContext context = getSession().getSuspendContext(); + String threadId = threadIdBeforeResumeOrStep(context); + for (PyThreadInfo suspendedThread : mySuspendedThreads) { + if (threadId == null || threadId.equals(suspendedThread.getId())) { + myDebugger.loadFullVariableValues(frame.getThreadId(), frame.getFrameId(), pyAsyncValues); + break; + } + } + } + } + catch (PyDebuggerException e) { + if (!isConnected()) return; + for (PyAsyncValue asyncValue: pyAsyncValues) { + PyDebugValue value = asyncValue.getDebugValue(); + XValueNode node = value.getLastNode(); + if (node != null && !node.isObsolete()) { + if (e.getMessage().startsWith("Timeout")) { + value.updateNodeValueAfterLoading(node, " ", "Timeout Exceeded"); + } + else { + LOG.error(e); + } + } + } + } + }); + } + private XValueChildrenList applyNewValue(XValueChildrenList pyDebugValues, String threadFrameId) { if (myNewVariableValue.containsKey(threadFrameId)) { PyDebugValue newValue = myNewVariableValue.get(threadFrameId); @@ -767,7 +783,7 @@ public class PyDebugProcess extends XDebugProcess implements IPyDebugProcess, Pr @Override public XValueChildrenList loadVariable(final PyDebugValue var) throws PyDebuggerException { final PyStackFrame frame = currentFrame(); - PyDebugValue debugValue = var.setName(var.getFullName()); + PyDebugValue debugValue = new PyDebugValue(var, var.getFullName()); return myDebugger.loadVariable(frame.getThreadId(), frame.getFrameId(), debugValue); } @@ -1183,7 +1199,7 @@ public class PyDebugProcess extends XDebugProcess implements IPyDebugProcess, Pr } @Override - public void showNumericContainer(PyDebugValue value) { + public void showNumericContainer(@NotNull PyDebugValue value) { PyViewNumericContainerAction.showNumericViewer(getProject(), value); } diff --git a/python/src/com/jetbrains/python/debugger/PyDebugRunner.java b/python/src/com/jetbrains/python/debugger/PyDebugRunner.java index 05960f3929d9..84c7aff0c955 100644 --- a/python/src/com/jetbrains/python/debugger/PyDebugRunner.java +++ b/python/src/com/jetbrains/python/debugger/PyDebugRunner.java @@ -305,7 +305,6 @@ public class PyDebugRunner extends GenericProgramRunner { configureDebugParameters(project, debugParams, pyState, cmd); - configureDebugEnvironment(project, cmd.getEnvironment()); configureDebugConnectionParameters(debugParams, serverLocalPort); @@ -323,6 +322,9 @@ public class PyDebugRunner extends GenericProgramRunner { if (debuggerSettings.isLibrariesFilterEnabled()) { environment.put(PYDEVD_FILTER_LIBRARIES, "True"); } + if (debuggerSettings.isLoadValuesAsync()) { + environment.put(PyVariableViewSettings.PYDEVD_LOAD_VALUES_ASYNC, "True"); + } PydevConsoleRunnerFactory.putIPythonEnvFlag(project, environment); diff --git a/python/src/com/jetbrains/python/debugger/PyDebuggerEvaluator.java b/python/src/com/jetbrains/python/debugger/PyDebuggerEvaluator.java index 989dee3fa1a9..9ed48665f827 100644 --- a/python/src/com/jetbrains/python/debugger/PyDebuggerEvaluator.java +++ b/python/src/com/jetbrains/python/debugger/PyDebuggerEvaluator.java @@ -28,8 +28,6 @@ import org.jetbrains.annotations.Nullable; public class PyDebuggerEvaluator extends XDebuggerEvaluator { - private static final PyDebugValue NONE = new PyDebugValue("", "NoneType", null, "None", false, false, false, false, null, null); - private Project myProject; private final PyFrameAccessor myDebugProcess; @@ -43,11 +41,15 @@ public class PyDebuggerEvaluator extends XDebuggerEvaluator { doEvaluate(expression, callback, true); } + private PyDebugValue getNone() { + return new PyDebugValue("", "NoneType", null, "None", false, false, false, false, null, myDebugProcess); + } + private void doEvaluate(final String expr, final XEvaluationCallback callback, final boolean doTrunc) { ApplicationManager.getApplication().executeOnPooledThread(() -> { String expression = expr.trim(); if (expression.isEmpty()) { - callback.evaluated(NONE); + callback.evaluated(getNone()); return; } diff --git a/python/src/com/jetbrains/python/debugger/PyStackFrame.java b/python/src/com/jetbrains/python/debugger/PyStackFrame.java index f0aa6906e552..2c3149b12b1e 100644 --- a/python/src/com/jetbrains/python/debugger/PyStackFrame.java +++ b/python/src/com/jetbrains/python/debugger/PyStackFrame.java @@ -147,10 +147,14 @@ public class PyStackFrame extends XStackFrame { if (node.isObsolete() || !isVariablesViewVisible()) return; ApplicationManager.getApplication().executeOnPooledThread(() -> { try { + boolean cached = myDebugProcess.isCurrentFrameCached(); XValueChildrenList values = myDebugProcess.loadFrame(); if (!node.isObsolete()) { addChildren(node, values); } + if (values != null && !cached) { + PyDebugValue.getAsyncValues(myDebugProcess, values); + } } catch (PyDebuggerException e) { if (!node.isObsolete()) { diff --git a/python/src/com/jetbrains/python/debugger/PyVariableViewSettings.java b/python/src/com/jetbrains/python/debugger/PyVariableViewSettings.java new file mode 100644 index 000000000000..96c3d250cca2 --- /dev/null +++ b/python/src/com/jetbrains/python/debugger/PyVariableViewSettings.java @@ -0,0 +1,92 @@ +/* + * Copyright 2000-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.jetbrains.python.debugger; + +import com.intellij.openapi.actionSystem.AnActionEvent; +import com.intellij.openapi.actionSystem.Presentation; +import com.intellij.openapi.actionSystem.ToggleAction; +import com.jetbrains.python.debugger.settings.PyDebuggerSettings; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +public class PyVariableViewSettings { + public static final String PYDEVD_LOAD_VALUES_ASYNC = "PYDEVD_LOAD_VALUES_ASYNC"; + + public static class SimplifiedView extends ToggleAction { + private final PyDebugProcess myProcess; + private final String myText; + private volatile boolean mySimplifiedView; + + public SimplifiedView(@Nullable PyDebugProcess debugProcess) { + super("", "Disables watching classes, functions and modules objects", null); + mySimplifiedView = PyDebuggerSettings.getInstance().isSimplifiedView(); + myProcess = debugProcess; + myText = "Simplified Variables View"; + } + + @Override + public void update(@NotNull final AnActionEvent e) { + super.update(e); + final Presentation presentation = e.getPresentation(); + presentation.setEnabled(true); + presentation.setText(myText); + } + + @Override + public boolean isSelected(AnActionEvent e) { + return mySimplifiedView; + } + + @Override + public void setSelected(AnActionEvent e, boolean hide) { + mySimplifiedView = hide; + PyDebuggerSettings.getInstance().setSimplifiedView(hide); + if (myProcess != null) { + myProcess.getSession().rebuildViews(); + } + } + } + + public static class AsyncView extends ToggleAction { + private final String myText; + private volatile boolean myLazyVariablesEvaluation; + + public AsyncView() { + super("", "Load variable values asynchronously", null); + myLazyVariablesEvaluation = PyDebuggerSettings.getInstance().isLoadValuesAsync(); + myText = "Load Values Asynchronously"; + } + + @Override + public void update(@NotNull final AnActionEvent e) { + super.update(e); + final Presentation presentation = e.getPresentation(); + presentation.setEnabled(true); + presentation.setText(myText); + } + + @Override + public boolean isSelected(AnActionEvent e) { + return myLazyVariablesEvaluation; + } + + @Override + public void setSelected(AnActionEvent e, boolean hide) { + myLazyVariablesEvaluation = hide; + PyDebuggerSettings.getInstance().setLoadValuesAsync(hide); + } + } +} diff --git a/python/src/com/jetbrains/python/debugger/containerview/PyDataViewerPanel.java b/python/src/com/jetbrains/python/debugger/containerview/PyDataViewerPanel.java index 3d4c187e1961..30d29b05f6cb 100644 --- a/python/src/com/jetbrains/python/debugger/containerview/PyDataViewerPanel.java +++ b/python/src/com/jetbrains/python/debugger/containerview/PyDataViewerPanel.java @@ -19,6 +19,7 @@ import com.intellij.codeInsight.completion.CompletionResultSet; import com.intellij.codeInsight.completion.PrioritizedLookupElement; import com.intellij.codeInsight.lookup.LookupElementBuilder; import com.intellij.ide.util.PropertiesComponent; +import com.intellij.openapi.application.ApplicationManager; import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.editor.EditorFactory; import com.intellij.openapi.editor.ex.EditorEx; @@ -81,7 +82,7 @@ public class PyDataViewerPanel extends JPanel { } private void setupChangeListener() { - myFrameAccessor.addFrameListener(() -> UIUtil.invokeLaterIfNeeded(() -> updateModel())); + myFrameAccessor.addFrameListener(() -> ApplicationManager.getApplication().executeOnPooledThread(() -> updateModel())); } private void updateModel() { @@ -91,9 +92,11 @@ public class PyDataViewerPanel extends JPanel { } model.invalidateCache(); updateDebugValue(model); - if (isShowing()) { - model.fireTableDataChanged(); - } + ApplicationManager.getApplication().invokeLater(() -> { + if (isShowing()) { + model.fireTableDataChanged(); + } + }); } private void updateDebugValue(@NotNull AsyncArrayTableModel model) { @@ -153,12 +156,13 @@ public class PyDataViewerPanel extends JPanel { } public void apply(String name) { - myErrorLabel.setVisible(false); - PyDebugValue debugValue = getDebugValue(name); - if (debugValue == null) { - return; - } - apply(debugValue); + ApplicationManager.getApplication().executeOnPooledThread(() -> { + PyDebugValue debugValue = getDebugValue(name); + if (debugValue == null) { + return; + } + ApplicationManager.getApplication().invokeLater(() -> apply(debugValue)); + }); } public void apply(@NotNull PyDebugValue debugValue) { @@ -169,13 +173,15 @@ public class PyDataViewerPanel extends JPanel { setError(type + " is not supported"); return; } - try { - ArrayChunk arrayChunk = debugValue.getFrameAccessor().getArrayItems(debugValue, 0, 0, -1, -1, getFormat()); - updateUI(arrayChunk, debugValue, strategy); - } - catch (PyDebuggerException e) { - LOG.error(e); - } + ApplicationManager.getApplication().executeOnPooledThread(() -> { + try { + ArrayChunk arrayChunk = debugValue.getFrameAccessor().getArrayItems(debugValue, 0, 0, -1, -1, getFormat()); + ApplicationManager.getApplication().invokeLater(() -> updateUI(arrayChunk, debugValue, strategy)); + } + catch (PyDebuggerException e) { + LOG.error(e); + } + }); } public void resize(boolean autoResize) { diff --git a/python/src/com/jetbrains/python/debugger/settings/PyDebuggerSettings.java b/python/src/com/jetbrains/python/debugger/settings/PyDebuggerSettings.java index cb7e8646c32d..e3b1020ed991 100644 --- a/python/src/com/jetbrains/python/debugger/settings/PyDebuggerSettings.java +++ b/python/src/com/jetbrains/python/debugger/settings/PyDebuggerSettings.java @@ -40,6 +40,7 @@ public class PyDebuggerSettings extends XDebuggerSettings im public static final String FILTERS_DIVIDER = ";"; private boolean myWatchReturnValues = false; private boolean mySimplifiedView = true; + private boolean myLoadValuesAsync = true; public PyDebuggerSettings() { super("python"); @@ -62,6 +63,14 @@ public class PyDebuggerSettings extends XDebuggerSettings im mySimplifiedView = simplifiedView; } + public boolean isLoadValuesAsync() { + return myLoadValuesAsync; + } + + public void setLoadValuesAsync(boolean loadValuesAsync) { + myLoadValuesAsync = loadValuesAsync; + } + public static PyDebuggerSettings getInstance() { return getInstance(PyDebuggerSettings.class); } diff --git a/python/testData/debug/test_async_eval.py b/python/testData/debug/test_async_eval.py new file mode 100644 index 000000000000..04a7d4aca99b --- /dev/null +++ b/python/testData/debug/test_async_eval.py @@ -0,0 +1,15 @@ +import time + + +class Foo(object): + def __init__(self, name): + self.name = name + + def __repr__(self): + time.sleep(1) + return self.name + + +f = Foo("foo") +l = [Foo("list"), Foo("list")] +a = 1 \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/env/python/PythonDebuggerTest.java b/python/testSrc/com/jetbrains/env/python/PythonDebuggerTest.java index 769e123163eb..38afc99bb67b 100644 --- a/python/testSrc/com/jetbrains/env/python/PythonDebuggerTest.java +++ b/python/testSrc/com/jetbrains/env/python/PythonDebuggerTest.java @@ -1326,6 +1326,30 @@ public class PythonDebuggerTest extends PyEnvTestCase { }); } + @Test + public void testLoadValuesAsync() { + runPythonTest(new PyDebuggerTask("/debug", "test_async_eval.py") { + @Override + public void before() { + toggleBreakpoint(getFilePath(getScriptName()), 14); + } + + @Override + public void testing() throws Exception { + waitForPause(); + List frameVariables = loadFrame(); + assertTrue(findDebugValueByName(frameVariables, "f").isLoadValueAsync()); + String result = computeValueAsync(frameVariables, "f"); + assertEquals("foo", result); + + List listChildren = loadChildren(frameVariables, "l"); + assertTrue(findDebugValueByName(frameVariables, "l").isLoadValueAsync()); + result = computeValueAsync(listChildren, "0"); + assertEquals("list", result); + } + }); + } + //TODO: That doesn't work now: case from test_continuation.py and test_continuation2.py are treated differently by interpreter // (first line is executed in first case and last line in second) diff --git a/python/testSrc/com/jetbrains/env/python/debug/PyBaseDebuggerTask.java b/python/testSrc/com/jetbrains/env/python/debug/PyBaseDebuggerTask.java index 2257b97a7130..1eed80048592 100644 --- a/python/testSrc/com/jetbrains/env/python/debug/PyBaseDebuggerTask.java +++ b/python/testSrc/com/jetbrains/env/python/debug/PyBaseDebuggerTask.java @@ -45,6 +45,8 @@ import org.junit.Assert; import javax.swing.*; import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.List; import java.util.Set; import java.util.concurrent.Semaphore; @@ -165,6 +167,56 @@ public abstract class PyBaseDebuggerTask extends PyExecutionFixtureTestTask { return result.first; } + protected List loadChildren(List debugValues, String name) throws PyDebuggerException { + PyDebugValue var = findDebugValueByName(debugValues, name); + return convertToList(myDebugProcess.loadVariable(var)); + } + + protected List loadFrame() throws PyDebuggerException { + return convertToList(myDebugProcess.loadFrame()); + } + + protected String computeValueAsync(List debugValues, String name) throws PyDebuggerException { + final PyDebugValue debugValue = findDebugValueByName(debugValues, name); + assert debugValue != null; + Semaphore variableSemaphore = new Semaphore(0); + ArrayList> valuesForEvaluation = new ArrayList<>(); + valuesForEvaluation.add(new PyFrameAccessor.PyAsyncValue<>(debugValue, new PyDebugCallback() { + @Override + public void ok(String value) { + debugValue.setValue(value); + variableSemaphore.release(); + } + + @Override + public void error(PyDebuggerException exception) { + variableSemaphore.release(); + } + })); + myDebugProcess.loadAsyncVariablesValues(valuesForEvaluation); + XDebuggerTestUtil.waitFor(variableSemaphore, NORMAL_TIMEOUT); + return debugValue.getValue(); + } + + public static List convertToList(XValueChildrenList childrenList) { + List values = new ArrayList<>(); + for (int i = 0; i < childrenList.size(); i++) { + PyDebugValue value = (PyDebugValue)childrenList.getValue(i); + values.add(value); + } + return values; + } + + @Nullable + public static PyDebugValue findDebugValueByName(@NotNull List debugValues, @NotNull String name) { + for (PyDebugValue val : debugValues) { + if (val.getName().equals(name)) { + return val; + } + } + return null; + } + @NotNull protected String output() { if (mySession != null && mySession.getConsoleView() != null) {