PY-57290 Debugger asyncio improvements

Patch field '_ready' from asyncio event loop for internal coroutine.
Patch `asyncio.new_event_loop()`
Add wrapper 'PyDevCoro' for internal coroutine.
Patch `asyncio.Task.__init__`, `asyncio.ensure_future` and `asyncio.call_soon`.
Add tests for `asyncio.gather` function.
Patch `asyncio.new_event_loop()`.

IJ-CR-99158

GitOrigin-RevId: 881087716a84c8ebb5331de4e2829bf38f2090e2
This commit is contained in:
Egor Eliseev
2023-03-22 16:10:02 +00:00
committed by intellij-monorepo-bot
parent 95d8f4d808
commit 2ecdf794a3
6 changed files with 300 additions and 78 deletions
@@ -4,6 +4,7 @@ from _pydevd_bundle.pydevd_constants import IS_ASYNCIO_DEBUGGER_ENV, IS_ASYNCIO_
from _pydevd_bundle.pydevd_exec2 import Exec
from _pydev_bundle.pydev_log import warn
eval_async_expression_in_context = None
eval_async_expression = None
exec_async_code = None
@@ -11,7 +12,7 @@ asyncio_command_compiler = None
if IS_ASYNCIO_DEBUGGER_ENV or IS_ASYNCIO_REPL:
from _pydevd_bundle import pydevd_save_locals
from _pydevd_asyncio_util.pydevd_nest_asyncio import apply
from _pydevd_asyncio_util.pydevd_nest_asyncio import apply, PyDevCoro
from codeop import CommandCompiler
import ast, types, inspect, asyncio
@@ -114,9 +115,9 @@ if IS_ASYNCIO_DEBUGGER_ENV or IS_ASYNCIO_REPL:
try:
if inspect.iscoroutine(result) and MODULE in str(result):
loop = asyncio.get_event_loop()
result = loop.run_until_complete(result)
result = loop.run_until_complete(PyDevCoro(result))
except:
warn('Failed to run coroutine %s'%str(result))
warn('Failed to run coroutine %s' % str(result))
finally:
return result
@@ -26,28 +26,77 @@
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from _pydevd_bundle.pydevd_constants import IS_PY3K
apply = None
from _pydevd_bundle.pydevd_constants import IS_ASYNCIO_DEBUGGER_ENV
if IS_PY3K:
apply = None
PyDevCoro = None
if IS_ASYNCIO_DEBUGGER_ENV:
import asyncio
import asyncio.events as events
import os
import sys
import threading
import contextvars
import inspect
import itertools
from contextlib import contextmanager, suppress
from heapq import heappop
from _pydevd_bundle.pydevd_constants import IS_PY3K
from heapq import heappop, heappush
from _pydev_bundle.pydev_log import warn
_task_name_counter = itertools.count(1).__next__
class _PyDevCoro:
""" Internal coroutine wrapper """
def __init__(self, coroutine):
self.pydevd_coro = coroutine
class _PydevdAsyncioUtils:
@staticmethod
def get_event_loop(is_from_event=False):
try:
if is_from_event:
loop = events.get_event_loop_policy().get_event_loop()
else:
loop = asyncio.get_event_loop()
except:
loop = asyncio.new_event_loop()
return loop
@staticmethod
def try_to_get_internal_coro(coroutine):
if isinstance(coroutine, _PyDevCoro):
return True, coroutine.pydevd_coro
if isinstance(coroutine, asyncio.Task):
if hasattr(coroutine, '_is_internal') and coroutine._is_internal:
return True, coroutine
return False, coroutine
@staticmethod
def try_to_get_internal_callback(task):
if isinstance(task, asyncio.Task):
callback = task._Task__step
if hasattr(task, '_is_internal') and task._is_internal:
return True, callback
return False, callback
return False, task
def _apply(loop=None):
"""Patch asyncio to make its event loop reentrant."""
_patch_asyncio()
_patch_task()
_patch_tornado()
""" Patch asyncio to make its event loop reentrant. """
try:
_patch_asyncio()
_patch_task()
_patch_tornado()
loop = loop or asyncio.get_event_loop()
_patch_loop(loop)
loop = loop or asyncio.get_event_loop()
loop._compute_internal_coro = False
_patch_loop(loop)
except:
warn("Failed to patch asyncio library")
def _patch_asyncio():
@@ -55,13 +104,16 @@ if IS_PY3K:
Patch asyncio module to use pure Python tasks and futures,
use module level _current_tasks, all_tasks and patch run method.
"""
def new_event_loop():
loop = asyncio.get_event_loop_policy().new_event_loop()
asyncio.set_event_loop(loop)
loop._compute_internal_coro = False
_patch_loop(loop)
return loop
def run(main, debug=False):
try:
loop = asyncio.get_event_loop()
except RuntimeError:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
_patch_loop(loop)
loop = _PydevdAsyncioUtils.get_event_loop()
loop.set_debug(debug)
task = asyncio.ensure_future(main)
try:
@@ -75,34 +127,59 @@ if IS_PY3K:
def _get_event_loop(stacklevel=3):
loop = events._get_running_loop()
if loop is None:
try:
loop = events.get_event_loop_policy().get_event_loop()
except:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
_patch_loop(loop)
loop = _PydevdAsyncioUtils.get_event_loop(is_from_event=True)
return loop
if hasattr(asyncio, '_nest_patched'):
def ensure_future(coro_or_future, loop=None):
is_internal_coro, target_coroutine = _PydevdAsyncioUtils.try_to_get_internal_coro(coro_or_future)
if asyncio.futures.isfuture(target_coroutine):
if loop is not None and loop is not asyncio.futures._get_loop(target_coroutine):
raise ValueError('The future belongs to a different loop than '
'the one specified as the loop argument')
return target_coroutine
if not asyncio.coroutines.iscoroutine(target_coroutine):
if inspect.isawaitable(target_coroutine):
target_coroutine = asyncio.tasks._wrap_awaitable(target_coroutine)
else:
raise TypeError('An asyncio.Future, a coroutine or an awaitable '
'is required')
if is_internal_coro:
coro_or_future.pydevd_coro = target_coroutine
else:
coro_or_future = target_coroutine
if loop is None:
loop = _PydevdAsyncioUtils.get_event_loop(is_from_event=True)
return loop.create_task(coro_or_future)
if hasattr(asyncio, '_pydevd_nest_patched'):
return
if sys.version_info >= (3, 6, 0):
asyncio.Task = asyncio.tasks._CTask = asyncio.tasks.Task = \
asyncio.tasks._PyTask
asyncio.Future = asyncio.futures._CFuture = asyncio.futures.Future = \
asyncio.futures._PyFuture
if sys.version_info < (3, 7, 0):
asyncio.tasks._current_tasks = asyncio.tasks.Task._current_tasks
asyncio.all_tasks = asyncio.tasks.Task.all_tasks
asyncio.Task = asyncio.tasks._CTask = asyncio.tasks.Task = asyncio.tasks._PyTask
asyncio.Future = asyncio.futures._CFuture = asyncio.futures.Future = asyncio.futures._PyFuture
if sys.version_info >= (3, 9, 0):
events._get_event_loop = events.get_event_loop = \
asyncio.get_event_loop = _get_event_loop
events._get_event_loop = events.get_event_loop = asyncio.get_event_loop = _get_event_loop
asyncio.run = run
asyncio.ensure_future = ensure_future
asyncio.new_event_loop = new_event_loop
asyncio._nest_patched = True
asyncio._pydevd_nest_patched = True
def _patch_loop(loop):
"""Patch loop to make it reentrant."""
def _get_internal_coroutines(loop):
delta = loop._ready.copy()
for handle in loop._original_ready:
delta.remove(handle)
return delta
def run_forever(self):
with manage_run(self), manage_asyncgens(self):
while True:
@@ -113,6 +190,11 @@ if IS_PY3K:
def run_until_complete(self, future):
with manage_run(self):
is_internal_coro, _ = _PydevdAsyncioUtils.try_to_get_internal_coro(future)
if is_internal_coro:
self._original_ready = self._ready.copy()
self._compute_internal_coro = True
f = asyncio.ensure_future(future, loop=self)
if f is not future:
f._log_destroy_pending = False
@@ -121,8 +203,11 @@ if IS_PY3K:
if self._stopping:
break
if not f.done():
raise RuntimeError(
'Event loop stopped before Future completed.')
raise RuntimeError('Event loop stopped before Future completed.')
if is_internal_coro:
self._compute_internal_coro = False
return f.result()
def _run_once(self):
@@ -148,14 +233,55 @@ if IS_PY3K:
handle = heappop(scheduled)
ready.append(handle)
for _ in range(len(ready)):
if not ready:
break
handle = ready.popleft()
if not handle._cancelled:
handle._run()
if self._compute_internal_coro:
internal_coroutines = _get_internal_coroutines(self)
for elem in internal_coroutines:
try:
ready.remove(elem)
if not elem._cancelled:
elem._run()
except:
pass
else:
for _ in range(len(ready)):
if not ready or self._compute_internal_coro:
break
handle = ready.popleft()
if not handle._cancelled:
handle._run()
handle = None
def call_at(self, when, callback, *args, context=None):
if when is None:
raise TypeError("when cannot be None")
self._check_closed()
_, target_callback = _PydevdAsyncioUtils.try_to_get_internal_callback(callback)
if self._debug:
self._check_thread()
self._check_callback(target_callback, 'call_at')
timer = events.TimerHandle(when, target_callback, args, self, context)
if timer._source_traceback:
del timer._source_traceback[-1]
heappush(self._scheduled, timer)
timer._scheduled = True
return timer
def call_soon(self, callback, *args, context=None):
self._check_closed()
_, target_callback = _PydevdAsyncioUtils.try_to_get_internal_callback(callback)
if self._debug:
self._check_thread()
self._check_callback(target_callback, 'call_soon')
handle = events.Handle(target_callback, args, self, context)
if handle._source_traceback:
del handle._source_traceback[-1]
self._ready.append(handle)
return handle
@contextmanager
def manage_run(self):
"""Set up the loop for running."""
@@ -205,7 +331,7 @@ if IS_PY3K:
"""Do not throw exception if loop is already running."""
pass
if hasattr(loop, '_nest_patched'):
if hasattr(loop, '_pydevd_nest_patched'):
return
if not isinstance(loop, asyncio.BaseEventLoop):
raise ValueError('Can\'t patch loop of type %s' % type(loop))
@@ -213,14 +339,16 @@ if IS_PY3K:
cls.run_forever = run_forever
cls.run_until_complete = run_until_complete
cls._run_once = _run_once
cls.call_soon = call_soon
cls.call_at = call_at
cls._check_running = _check_running
cls._check_runnung = _check_running # typo in Python 3.7 source
cls._num_runs_pending = 0
cls._is_proactorloop = (
os.name == 'nt' and issubclass(cls, asyncio.ProactorEventLoop))
if sys.version_info < (3, 7, 0):
cls._set_coroutine_origin_tracking = cls._set_coroutine_wrapper
cls._nest_patched = True
cls._pydevd_nest_patched = True
def _patch_task():
@@ -236,27 +364,52 @@ if IS_PY3K:
else:
curr_tasks[task._loop] = curr_task
def task_new_init(self, coro, loop=None, name=None, context=None):
asyncio.futures.Future.__init__(self, loop=loop)
if self._source_traceback:
del self._source_traceback[-1]
self._is_internal, target_coroutine = _PydevdAsyncioUtils.try_to_get_internal_coro(coro)
if not asyncio.coroutines.iscoroutine(target_coroutine):
self._log_destroy_pending = False
raise TypeError('a coroutine was expected, got %s' % target_coroutine)
if name is None:
self._name = 'Task-%s' %_task_name_counter()
else:
self._name = str(name)
self._num_cancels_requested = 0
self._must_cancel = False
self._fut_waiter = None
self._coro = target_coroutine
if context is None:
self._context = contextvars.copy_context()
else:
self._context = context
self._loop.call_soon(self, context=self._context)
asyncio.tasks._register_task(self)
Task = asyncio.Task
if hasattr(Task, '_nest_patched'):
if hasattr(Task, '_pydevd_nest_patched'):
return
if sys.version_info >= (3, 7, 0):
def enter_task(loop, task):
curr_tasks[loop] = task
Task.__init__ = task_new_init
def leave_task(loop, task):
curr_tasks.pop(loop, None)
def enter_task(loop, task):
curr_tasks[loop] = task
def leave_task(loop, task):
curr_tasks.pop(loop, None)
asyncio.tasks._enter_task = enter_task
asyncio.tasks._leave_task = leave_task
curr_tasks = asyncio.tasks._current_tasks
step_orig = Task._Task__step
Task._Task__step = step
asyncio.tasks._enter_task = enter_task
asyncio.tasks._leave_task = leave_task
curr_tasks = asyncio.tasks._current_tasks
step_orig = Task._Task__step
Task._Task__step = step
else:
curr_tasks = Task._current_tasks
step_orig = Task._step
Task._step = step
Task._nest_patched = True
Task._pydevd_nest_patched = True
def _patch_tornado():
@@ -270,4 +423,6 @@ if IS_PY3K:
if asyncio.Future not in tc.FUTURES:
tc.FUTURES += (asyncio.Future,)
apply = _apply
PyDevCoro = _PyDevCoro
@@ -18,7 +18,8 @@ from _pydevd_bundle import pydevd_vars, pydevd_save_locals
from _pydevd_bundle.pydevd_console_pytest import enable_pytest_output
from _pydevd_bundle.pydevd_constants import IS_ASYNCIO_DEBUGGER_ENV
from _pydevd_asyncio_util.pydevd_asyncio_utils import asyncio_command_compiler, exec_async_code
from _pydevd_asyncio_util.pydevd_nest_asyncio import apply
if IS_ASYNCIO_DEBUGGER_ENV:
from _pydevd_asyncio_util.pydevd_nest_asyncio import apply
try:
import __builtin__
+5 -1
View File
@@ -28,7 +28,7 @@ from collections import defaultdict
from _pydevd_bundle.pydevd_constants import IS_JYTH_LESS25, IS_PYCHARM, get_thread_id, get_current_thread_id, \
dict_keys, dict_iter_items, DebugInfoHolder, PYTHON_SUSPEND, STATE_SUSPEND, STATE_RUN, get_frame, xrange, \
clear_cached_thread_id, INTERACTIVE_MODE_AVAILABLE, SHOW_DEBUG_INFO_ENV, IS_PY34_OR_GREATER, IS_PY36_OR_GREATER, \
IS_PY2, NULL, NO_FTRACE, dummy_excepthook, IS_CPYTHON, GOTO_HAS_RESPONSE
IS_PY2, NULL, NO_FTRACE, dummy_excepthook, IS_CPYTHON, GOTO_HAS_RESPONSE, IS_ASYNCIO_DEBUGGER_ENV
from _pydev_bundle import fix_getpass
from _pydev_bundle import pydev_imports, pydev_log
from _pydev_bundle._pydev_filesystem_encoding import getfilesystemencoding
@@ -1486,6 +1486,10 @@ class PyDB(object):
if set_trace:
self.enable_tracing()
if IS_ASYNCIO_DEBUGGER_ENV:
from _pydevd_asyncio_util.pydevd_nest_asyncio import apply
apply()
return self._exec(is_module, entry_point_fn, module_name, file, globals, locals)
def _exec(self, is_module, entry_point_fn, module_name, file, globals, locals):
@@ -0,0 +1,26 @@
import asyncio
async def foo(y):
return y + 1
async def factorial(name, number):
f = 1
for i in range(2, number + 1):
print(f"Task {name}: Compute factorial({number}), currently i={i}...")
await asyncio.sleep(1)
f *= i
print(f"Task {name}: factorial({number}) = {f}")
return f
async def main():
L = await asyncio.gather(
factorial("A", 2),
factorial("B", 3),
factorial("C", 4),
)
print(L)
asyncio.run(main())
@@ -18,13 +18,16 @@ public class PythonDebuggerAsyncioTest extends PyEnvTestCase {
private enum TestCase {
CONSOLE,
EVALUATE,
BREAKPOINT
BREAKPOINT,
GATHER
}
private static class AsyncioPyDebuggerTask extends PyDebuggerTask {
private static final String RELATIVE_PATH = "/debug";
private static final String SCRIPT_NAME = "test_asyncio_debugger.py";
private static final String SIMPLE_SCRIPT_NAME = "test_asyncio_debugger.py";
private static final String GATHER_SCRIPT_NAME = "test_asyncio_gather_debugger.py";
private static final String AWAIT_FOO = "await foo(1)";
@@ -34,8 +37,6 @@ public class PythonDebuggerAsyncioTest extends PyEnvTestCase {
private static final String GET_EVENT_LOOP = "loop = asyncio.get_event_loop()";
private static final String CLOSE_EVENT_LOOP = "loop.close()";
private static final String RUN_UNTIL_COMPLETE = "asyncio.get_event_loop().run_until_complete(foo(1))";
private final TestCase myTestCase;
@@ -60,7 +61,6 @@ public class PythonDebuggerAsyncioTest extends PyEnvTestCase {
consoleExec(GET_EVENT_LOOP);
consoleExec(RUN_FOO_WITH_LOOP);
waitForOutput("2");
consoleExec(CLOSE_EVENT_LOOP);
}
protected void testEvaluate() {
@@ -78,6 +78,17 @@ public class PythonDebuggerAsyncioTest extends PyEnvTestCase {
waitForTerminate();
}
protected void testGather() throws Exception {
for (int i = 0; i < 2; i++) {
if (i != 0) {
waitForPause();
}
testConsole();
testEvaluate();
resume();
}
}
@Override
public void testing() throws Exception {
waitForPause();
@@ -85,6 +96,7 @@ public class PythonDebuggerAsyncioTest extends PyEnvTestCase {
case CONSOLE -> testConsole();
case EVALUATE -> testEvaluate();
case BREAKPOINT -> testBreakpoints();
case GATHER -> testGather();
}
}
@@ -99,6 +111,11 @@ public class PythonDebuggerAsyncioTest extends PyEnvTestCase {
XDebuggerTestUtil.setBreakpointCondition(getProject(), 9, "await foo(1) != 2");
XDebuggerTestUtil.setBreakpointLogExpression(getProject(), 8, "await foo(1)");
}
case GATHER -> {
setWaitForTermination(false);
toggleBreakpoint(9);
XDebuggerTestUtil.setBreakpointCondition(getProject(), 9, "await foo(1) == 2");
}
}
setWaitForTermination(false);
}
@@ -112,55 +129,73 @@ public class PythonDebuggerAsyncioTest extends PyEnvTestCase {
@EnvTestTagsRequired(tags = "python3.8")
@Test
public void testAsyncioConsole38() {
runPythonTest(new AsyncioPyDebuggerTask(TestCase.CONSOLE, AsyncioPyDebuggerTask.SCRIPT_NAME));
runPythonTest(new AsyncioPyDebuggerTask(TestCase.CONSOLE, AsyncioPyDebuggerTask.SIMPLE_SCRIPT_NAME));
}
@EnvTestTagsRequired(tags = "python3.9")
@Test
public void testAsyncioConsole39() {
runPythonTest(new AsyncioPyDebuggerTask(TestCase.CONSOLE, AsyncioPyDebuggerTask.SCRIPT_NAME));
runPythonTest(new AsyncioPyDebuggerTask(TestCase.CONSOLE, AsyncioPyDebuggerTask.SIMPLE_SCRIPT_NAME));
}
@EnvTestTagsRequired(tags = "python3.10")
@Test
public void testAsyncioConsole310() {
runPythonTest(new AsyncioPyDebuggerTask(TestCase.CONSOLE, AsyncioPyDebuggerTask.SCRIPT_NAME));
runPythonTest(new AsyncioPyDebuggerTask(TestCase.CONSOLE, AsyncioPyDebuggerTask.SIMPLE_SCRIPT_NAME));
}
@EnvTestTagsRequired(tags = "python3.8")
@Test
public void testAsyncioEvaluate38() {
runPythonTest(new AsyncioPyDebuggerTask(TestCase.EVALUATE, AsyncioPyDebuggerTask.SCRIPT_NAME));
runPythonTest(new AsyncioPyDebuggerTask(TestCase.EVALUATE, AsyncioPyDebuggerTask.SIMPLE_SCRIPT_NAME));
}
@EnvTestTagsRequired(tags = "python3.9")
@Test
public void testAsyncioEvaluate39() {
runPythonTest(new AsyncioPyDebuggerTask(TestCase.EVALUATE, AsyncioPyDebuggerTask.SCRIPT_NAME));
runPythonTest(new AsyncioPyDebuggerTask(TestCase.EVALUATE, AsyncioPyDebuggerTask.SIMPLE_SCRIPT_NAME));
}
@EnvTestTagsRequired(tags = "python3.10")
@Test
public void testAsyncioEvaluate310() {
runPythonTest(new AsyncioPyDebuggerTask(TestCase.EVALUATE, AsyncioPyDebuggerTask.SCRIPT_NAME));
runPythonTest(new AsyncioPyDebuggerTask(TestCase.EVALUATE, AsyncioPyDebuggerTask.SIMPLE_SCRIPT_NAME));
}
@EnvTestTagsRequired(tags = "python3.8")
@Test
public void testAsyncioBreakpoint38() {
runPythonTest(new AsyncioPyDebuggerTask(TestCase.BREAKPOINT, AsyncioPyDebuggerTask.SCRIPT_NAME));
runPythonTest(new AsyncioPyDebuggerTask(TestCase.BREAKPOINT, AsyncioPyDebuggerTask.SIMPLE_SCRIPT_NAME));
}
@EnvTestTagsRequired(tags = "python3.9")
@Test
public void testAsyncioBreakpoint39() {
runPythonTest(new AsyncioPyDebuggerTask(TestCase.BREAKPOINT, AsyncioPyDebuggerTask.SCRIPT_NAME));
runPythonTest(new AsyncioPyDebuggerTask(TestCase.BREAKPOINT, AsyncioPyDebuggerTask.SIMPLE_SCRIPT_NAME));
}
@EnvTestTagsRequired(tags = "python3.10")
@Test
public void testAsyncioBreakpoint310() {
runPythonTest(new AsyncioPyDebuggerTask(TestCase.BREAKPOINT, AsyncioPyDebuggerTask.SCRIPT_NAME));
runPythonTest(new AsyncioPyDebuggerTask(TestCase.BREAKPOINT, AsyncioPyDebuggerTask.SIMPLE_SCRIPT_NAME));
}
@EnvTestTagsRequired(tags = "python3.8")
@Test
public void testAsyncioGather38() {
runPythonTest(new AsyncioPyDebuggerTask(TestCase.GATHER, AsyncioPyDebuggerTask.GATHER_SCRIPT_NAME));
}
@EnvTestTagsRequired(tags = "python3.9")
@Test
public void testAsyncioGather39() {
runPythonTest(new AsyncioPyDebuggerTask(TestCase.GATHER, AsyncioPyDebuggerTask.GATHER_SCRIPT_NAME));
}
@EnvTestTagsRequired(tags = "python3.10")
@Test
public void testAsyncioGather310() {
runPythonTest(new AsyncioPyDebuggerTask(TestCase.GATHER, AsyncioPyDebuggerTask.GATHER_SCRIPT_NAME));
}
}