diff --git a/python/helpers/pydev/_pydev_bundle/pydev_console_utils.py b/python/helpers/pydev/_pydev_bundle/pydev_console_utils.py index 838d6f6d956a..5670381c0228 100644 --- a/python/helpers/pydev/_pydev_bundle/pydev_console_utils.py +++ b/python/helpers/pydev/_pydev_bundle/pydev_console_utils.py @@ -6,8 +6,7 @@ from _pydev_bundle._pydev_calltip_util import get_description from _pydev_imps._pydev_saved_modules import thread from _pydevd_bundle import pydevd_vars from _pydevd_bundle import pydevd_xml -from _pydevd_bundle.pydevd_constants import IS_JYTHON, dict_iter_items -from _pydevd_bundle.pydevd_utils import to_string +from _pydevd_bundle.pydevd_constants import IS_JYTHON, dict_iter_items, NEXT_VALUE_SEPARATOR try: import cStringIO as StringIO #may not always be available @UnusedImport @@ -484,14 +483,13 @@ class BaseInterpreterInterface: def getVariable(self, attributes): xml = StringIO.StringIO() xml.write("") - valDict = pydevd_vars.resolve_var(self.get_namespace(), attributes) - if valDict is None: - valDict = {} - - keys = valDict.keys() + val_dict = pydevd_vars.resolve_compound_var_object_fields(self.get_namespace(), attributes) + if val_dict is None: + val_dict = {} + keys = val_dict.keys() for k in keys: - val = valDict[k] + val = val_dict[k] evaluate_full_value = pydevd_xml.should_evaluate_full_value(val) xml.write(pydevd_vars.var_to_xml(val, k, evaluate_full_value=evaluate_full_value)) @@ -512,14 +510,34 @@ class BaseInterpreterInterface: xml.write("") return xml.getvalue() - def loadFullValue(self, expressions): - xml = StringIO.StringIO() - xml.write("") - for expression in expressions: - result = pydevd_vars.eval_in_context(expression, self.get_namespace(), self.get_namespace()) - xml.write(pydevd_vars.var_to_xml(result, expression, evaluate_full_value=True)) - xml.write("") - return xml.getvalue() + def loadFullValue(self, seq, scope_attrs): + """ + Evaluate full value for async Console variables in a separate thread and send results to IDE side + :param seq: id of command + :param scope_attrs: a sequence of variables with their attributes separated by NEXT_VALUE_SEPARATOR + (i.e.: obj\tattr1\tattr2NEXT_VALUE_SEPARATORobj2\attr1\tattr2) + :return: + """ + frame_variables = self.get_namespace() + var_objects = [] + vars = scope_attrs.split(NEXT_VALUE_SEPARATOR) + for var_attrs in vars: + if '\t' in var_attrs: + name, attrs = var_attrs.split('\t', 1) + + else: + name = var_attrs + attrs = None + if name in frame_variables.keys(): + var_object = pydevd_vars.resolve_var_object(frame_variables[name], attrs) + var_objects.append((var_object, name)) + else: + var_object = pydevd_vars.eval_in_context(name, frame_variables, frame_variables) + var_objects.append((var_object, name)) + + from _pydevd_bundle.pydevd_comm import GetValueAsyncThreadConsole + t = GetValueAsyncThreadConsole(self.get_server(), seq, var_objects) + t.start() def changeVariable(self, attr, value): def do_change_variable(): diff --git a/python/helpers/pydev/_pydevd_bundle/pydevd_comm.py b/python/helpers/pydev/_pydevd_bundle/pydevd_comm.py index 10892505e397..1b499f45e67e 100644 --- a/python/helpers/pydev/_pydevd_bundle/pydevd_comm.py +++ b/python/helpers/pydev/_pydevd_bundle/pydevd_comm.py @@ -1026,7 +1026,7 @@ class InternalGetVariable(InternalThreadCommand): try: xml = StringIO.StringIO() xml.write("") - _typeName, val_dict = pydevd_vars.resolve_compound_variable(self.thread_id, self.frame_id, self.scope, self.attributes) + _typeName, val_dict = pydevd_vars.resolve_compound_variable_fields(self.thread_id, self.frame_id, self.scope, self.attributes) if val_dict is None: val_dict = {} @@ -1482,11 +1482,10 @@ class InternalLoadFullValue(InternalThreadCommand): else: scope, attrs = (variable, None) name = scope - var_obj = pydevd_vars.getVariable(self.thread_id, self.frame_id, scope, attrs) var_objects.append((var_obj, name)) - t = GetValueAsyncThread(dbg, self.thread_id, self.frame_id, self.sequence, var_objects) + t = GetValueAsyncThreadDebug(dbg, self.sequence, var_objects) t.start() except: exc = get_exception_traceback_str() @@ -1495,16 +1494,20 @@ class InternalLoadFullValue(InternalThreadCommand): dbg.writer.add_command(cmd) -class GetValueAsyncThread(PyDBDaemonThread): - def __init__(self, py_db, thread_id, frame_id, seq, var_objects): +class AbstractGetValueAsyncThread(PyDBDaemonThread): + """ + Abstract class for a thread, which evaluates values for async variables + """ + def __init__(self, frame_accessor, seq, var_objects): PyDBDaemonThread.__init__(self) - self.py_db = py_db - self.thread_id = thread_id - self.frame_id = frame_id + self.frame_accessor = frame_accessor self.seq = seq self.var_objs = var_objects self.cancel_event = threading.Event() + def send_result(self, xml): + raise NotImplementedError() + def _on_run(self): start = time.time() xml = StringIO.StringIO() @@ -1515,9 +1518,29 @@ class GetValueAsyncThread(PyDBDaemonThread): break xml.write(pydevd_xml.var_to_xml(var_obj, name, evaluate_full_value=True)) xml.write("") - cmd = self.py_db.cmd_factory.make_load_full_value_message(self.seq, xml.getvalue()) + self.send_result(xml) xml.close() - self.py_db.writer.add_command(cmd) + + +class GetValueAsyncThreadDebug(AbstractGetValueAsyncThread): + """ + A thread for evaluation async values, which returns result for debugger + Create message and send it via writer thread + """ + def send_result(self, xml): + if self.frame_accessor is not None: + cmd = self.frame_accessor.cmd_factory.make_load_full_value_message(self.seq, xml.getvalue()) + self.frame_accessor.writer.add_command(cmd) + + +class GetValueAsyncThreadConsole(AbstractGetValueAsyncThread): + """ + A thread for evaluation async values, which returns result for Console + Send result directly to Console's server + """ + def send_result(self, xml): + if self.frame_accessor is not None: + self.frame_accessor.ReturnFullValue(self.seq, xml.getvalue()) #======================================================================================================================= diff --git a/python/helpers/pydev/_pydevd_bundle/pydevd_vars.py b/python/helpers/pydev/_pydevd_bundle/pydevd_vars.py index 27b3b727338c..51c6949621b1 100644 --- a/python/helpers/pydev/_pydevd_bundle/pydevd_vars.py +++ b/python/helpers/pydev/_pydevd_bundle/pydevd_vars.py @@ -229,8 +229,17 @@ def getVariable(thread_id, frame_id, scope, attrs): return var -def resolve_compound_variable(thread_id, frame_id, scope, attrs): - """ returns the value of the compound variable as a dictionary""" +def resolve_compound_variable_fields(thread_id, frame_id, scope, attrs): + """ + Resolve compound variable in debugger scopes by its name and attributes + + :param thread_id: id of the variable's thread + :param frame_id: id of the variable's frame + :param scope: can be BY_ID, EXPRESSION, GLOBAL, LOCAL, FRAME + :param attrs: after reaching the proper scope, we have to get the attributes until we find + the proper location (i.e.: obj\tattr1\tattr2) + :return: a dictionary of variables's fields + """ var = getVariable(thread_id, frame_id, scope, attrs) @@ -243,12 +252,36 @@ def resolve_compound_variable(thread_id, frame_id, scope, attrs): traceback.print_exc() -def resolve_var(var, attrs): - attrList = attrs.split('\t') +def resolve_var_object(var, attrs): + """ + Resolve variable's attribute - for k in attrList: + :param var: an object of variable + :param attrs: a sequence of variable's attributes separated by \t (i.e.: obj\tattr1\tattr2) + :return: a value of resolved variable's attribute + """ + if attrs is not None: + attr_list = attrs.split('\t') + else: + attr_list = [] + for k in attr_list: type, _typeName, resolver = get_type(var) + var = resolver.resolve(var, k) + return var + +def resolve_compound_var_object_fields(var, attrs): + """ + Resolve compound variable by its object and attributes + + :param var: an object of variable + :param attrs: a sequence of variable's attributes separated by \t (i.e.: obj\tattr1\tattr2) + :return: a dictionary of variables's fields + """ + attr_list = attrs.split('\t') + + for k in attr_list: + type, _typeName, resolver = get_type(var) var = resolver.resolve(var, k) try: diff --git a/python/pydevSrc/com/jetbrains/python/debugger/pydev/GetVariableCommand.java b/python/pydevSrc/com/jetbrains/python/debugger/pydev/GetVariableCommand.java index 506e52438732..d543e32aefea 100644 --- a/python/pydevSrc/com/jetbrains/python/debugger/pydev/GetVariableCommand.java +++ b/python/pydevSrc/com/jetbrains/python/debugger/pydev/GetVariableCommand.java @@ -1,6 +1,7 @@ package com.jetbrains.python.debugger.pydev; import com.jetbrains.python.debugger.PyDebugValue; +import org.jetbrains.annotations.NotNull; public class GetVariableCommand extends GetFrameCommand { @@ -15,6 +16,13 @@ public class GetVariableCommand extends GetFrameCommand { myParent = var; } + /** + * Return a full path in a variables tree from the top-level parent to the debug value + * + * @param var a debug variable + * @return A string of attributes in a path separated by \t + */ + @NotNull public static String composeName(final PyDebugValue var) { final StringBuilder sb = new StringBuilder(); PyDebugValue p = var; @@ -26,7 +34,10 @@ public class GetVariableCommand extends GetFrameCommand { sb.insert(0, BY_ID).insert(0, '\t').insert(0, p.getId()); break; } else { - sb.insert(0, p.getTempName().replaceAll("\t", TAB_CHAR)); + final String tempName = p.getTempName(); + if (tempName != null) { + sb.insert(0, tempName.replaceAll("\t", TAB_CHAR)); + } } p = p.getParent(); } diff --git a/python/pydevSrc/com/jetbrains/python/debugger/pydev/LoadFullValueCommand.java b/python/pydevSrc/com/jetbrains/python/debugger/pydev/LoadFullValueCommand.java index 4ecb6c71c237..645439536f5f 100644 --- a/python/pydevSrc/com/jetbrains/python/debugger/pydev/LoadFullValueCommand.java +++ b/python/pydevSrc/com/jetbrains/python/debugger/pydev/LoadFullValueCommand.java @@ -10,7 +10,7 @@ import java.util.List; public class LoadFullValueCommand extends AbstractFrameCommand { - private static final String NEXT_VALUE_SEPARATOR = "__pydev_val__"; + public static final String NEXT_VALUE_SEPARATOR = "__pydev_val__"; private final @NotNull IPyDebugProcess myDebugProcess; private final @NotNull List> myVars; diff --git a/python/src/com/jetbrains/python/console/PydevConsoleCommunication.java b/python/src/com/jetbrains/python/console/PydevConsoleCommunication.java index ec3bc416f1ee..36fbbc1c64ea 100644 --- a/python/src/com/jetbrains/python/console/PydevConsoleCommunication.java +++ b/python/src/com/jetbrains/python/console/PydevConsoleCommunication.java @@ -38,6 +38,7 @@ import com.jetbrains.python.console.pydev.*; import com.jetbrains.python.debugger.*; import com.jetbrains.python.debugger.containerview.PyViewNumericContainerAction; import com.jetbrains.python.debugger.pydev.GetVariableCommand; +import com.jetbrains.python.debugger.pydev.LoadFullValueCommand; import com.jetbrains.python.debugger.pydev.ProtocolParser; import org.apache.xmlrpc.WebServer; import org.apache.xmlrpc.XmlRpcException; @@ -48,6 +49,7 @@ import org.jetbrains.annotations.Nullable; import java.net.MalformedURLException; import java.util.*; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Future; /** @@ -107,6 +109,9 @@ public class PydevConsoleCommunication extends AbstractConsoleCommunication impl private PythonDebugConsoleCommunication myDebugCommunication; private boolean myNeedsMore = false; + private int myFullValueSeq = 0; + private final Map>> myCallbackHashMap = new ConcurrentHashMap<>(); + private @Nullable PythonConsoleView myConsoleView; private final List myFrameListeners = ContainerUtil.createLockFreeCopyOnWriteList(); @@ -178,6 +183,7 @@ public class PydevConsoleCommunication extends AbstractConsoleCommunication impl public synchronized void close() { sendCloseMessageToScript(); PyDebugValueExecutionService.getInstance(myProject).sessionStopped(this); + myCallbackHashMap.clear(); if (myWebServer != null) { myWebServer.shutdown(); @@ -195,6 +201,7 @@ public class PydevConsoleCommunication extends AbstractConsoleCommunication impl public synchronized Future closeAsync() { sendCloseMessageToScript(); PyDebugValueExecutionService.getInstance(myProject).sessionStopped(this); + myCallbackHashMap.clear(); if (myWebServer != null) { Future shutdownFuture = myWebServer.shutdownAsync(); @@ -238,11 +245,31 @@ public class PydevConsoleCommunication extends AbstractConsoleCommunication impl } return ""; } + else if ("ReturnFullValue".equals(method)) { + processFullValueResponse((Integer)params.get(0), (String)params.get(1)); + return ""; + } else { throw new UnsupportedOperationException(); } } + private void processFullValueResponse(int seq, String response) { + final List> values = myCallbackHashMap.remove(seq); + try { + List debugValues = ProtocolParser.parseValues(response, this); + for (int i = 0; i < debugValues.size(); ++i) { + PyDebugValue resultValue = debugValues.get(i); + values.get(i).getCallback().ok(resultValue.getValue()); + } + } + catch (Exception e) { + for (PyFrameAccessor.PyAsyncValue vars : values) { + vars.getCallback().error(new PyDebuggerException(response)); + } + } + } + private Object execNotifyAboutMagic(Vector params) { List commands = (List)params.get(0); boolean isAutoMagic = (Boolean)params.get(1); @@ -257,10 +284,7 @@ public class PydevConsoleCommunication extends AbstractConsoleCommunication impl } private Object execIPythonEditor(Vector params) { - String path = (String)params.get(0); - int line = Integer.parseInt((String)params.get(1)); - final VirtualFile file = StringUtil.isEmpty(path) ? null : LocalFileSystem.getInstance().findFileByPath(path); if (file != null) { ApplicationManager.getApplication().invokeLater(() -> { @@ -469,40 +493,41 @@ public class PydevConsoleCommunication extends AbstractConsoleCommunication impl } }.queue(); - - //busy loop waiting for the answer (or having the console die). - ProgressManager.getInstance().runProcessWithProgressSynchronously(() -> { - final ProgressIndicator progressIndicator = ProgressManager.getInstance().getProgressIndicator(); - progressIndicator.setText("Waiting for REPL response with " + (int)(TIMEOUT / 10e8) + "s timeout"); - progressIndicator.setIndeterminate(false); - final long startTime = System.nanoTime(); - while (nextResponse == null) { - if (progressIndicator.isCanceled()) { - LOG.debug("Canceled"); - nextResponse = new InterpreterResponse(false, false); - } - - final long time = System.nanoTime() - startTime; - progressIndicator.setFraction(((double)time) / TIMEOUT); - if (time > TIMEOUT) { - LOG.debug("Timeout exceeded"); - nextResponse = new InterpreterResponse(false, false); - } - synchronized (lock2) { - try { - lock2.wait(20); + ProgressManager.getInstance().run(new Task.Backgroundable(myProject, "Waiting for REPL Response") { + @Override + public void run(@NotNull ProgressIndicator indicator) { + final ProgressIndicator progressIndicator = ProgressManager.getInstance().getProgressIndicator(); + progressIndicator.setText("Waiting for REPL response with " + (int)(TIMEOUT / 10e8) + "s timeout"); + progressIndicator.setIndeterminate(false); + final long startTime = System.nanoTime(); + while (nextResponse == null) { + if (progressIndicator.isCanceled()) { + LOG.debug("Canceled"); + nextResponse = new InterpreterResponse(false, false); } - catch (InterruptedException e) { - LOG.error(e); + + final long time = System.nanoTime() - startTime; + progressIndicator.setFraction(((double)time) / TIMEOUT); + if (time > TIMEOUT) { + LOG.debug("Timeout exceeded"); + nextResponse = new InterpreterResponse(false, false); + } + synchronized (lock2) { + try { + lock2.wait(20); + } + catch (InterruptedException e) { + LOG.error(e); + } } } + if (nextResponse.more) { + myNeedsMore = true; + notifyCommandExecuted(true); + } + onResponseReceived.fun(nextResponse); } - if (nextResponse.more) { - myNeedsMore = true; - notifyCommandExecuted(true); - } - onResponseReceived.fun(nextResponse); - }, "Waiting for REPL response", true, myProject); + }); } } @@ -564,6 +589,11 @@ public class PydevConsoleCommunication extends AbstractConsoleCommunication impl return new XValueChildrenList(); } + public synchronized int getNextFullValueSeq() { + myFullValueSeq++; + return myFullValueSeq; + } + @Override public void loadAsyncVariablesValues(@NotNull List> pyAsyncValues) { PyDebugValueExecutionService.getInstance(myProject).submitTask(this, () -> { @@ -571,9 +601,12 @@ public class PydevConsoleCommunication extends AbstractConsoleCommunication impl try { List evaluationExpressions = new ArrayList<>(); for (PyAsyncValue asyncValue : pyAsyncValues) { - evaluationExpressions.add(asyncValue.getDebugValue().getEvaluationExpression()); + evaluationExpressions.add(GetVariableCommand.composeName(asyncValue.getDebugValue())); } - Object ret = myClient.execute(LOAD_FULL_VALUE, new Object[]{evaluationExpressions.toArray()}); + final int seq = getNextFullValueSeq(); + myCallbackHashMap.put(seq, pyAsyncValues); + Object ret = myClient + .execute(LOAD_FULL_VALUE, new Object[]{seq, String.join(LoadFullValueCommand.NEXT_VALUE_SEPARATOR, evaluationExpressions)}); if (ret instanceof String) { List debugValues = ProtocolParser.parseValues((String)ret, this);