From ba365df91771465fe2a1cf8d6d52c55c58e6418d Mon Sep 17 00:00:00 2001 From: Andrey Lisin Date: Thu, 7 Nov 2019 15:52:53 +0300 Subject: [PATCH] PY-38941 Support debugging of processes created with `posix_spawn` (cherry picked from commit e55c2168595e22e80a8f95c2e3812cf7f687c31c) GitOrigin-RevId: 4db4aaaa9ce7198cfb5177bac262f9f72b92cdee --- .../pydev/_pydev_bundle/pydev_monkey.py | 24 ++- .../pydev/_pydevd_bundle/pydevd_constants.py | 2 + python/testData/debug/test4.py | 1 - python/testData/debug/test_posix_spawn.py | 7 + .../PythonDebuggerMultiprocessingTest.java | 195 ++++++++++++++++++ .../env/python/debug/PythonDebuggerTest.java | 161 --------------- 6 files changed, 227 insertions(+), 163 deletions(-) create mode 100644 python/testData/debug/test_posix_spawn.py create mode 100644 python/testSrc/com/jetbrains/env/python/debug/PythonDebuggerMultiprocessingTest.java diff --git a/python/helpers/pydev/_pydev_bundle/pydev_monkey.py b/python/helpers/pydev/_pydev_bundle/pydev_monkey.py index ad4d54018213..d083004ab83b 100644 --- a/python/helpers/pydev/_pydev_bundle/pydev_monkey.py +++ b/python/helpers/pydev/_pydev_bundle/pydev_monkey.py @@ -3,7 +3,8 @@ import os import sys import traceback from _pydev_imps._pydev_saved_modules import threading -from _pydevd_bundle.pydevd_constants import get_global_debugger, IS_WINDOWS, IS_MACOS, IS_JYTHON, IS_PY36_OR_LESSER, get_current_thread_id +from _pydevd_bundle.pydevd_constants import get_global_debugger, IS_WINDOWS, IS_MACOS, IS_JYTHON, IS_PY36_OR_LESSER, IS_PY38_OR_GREATER, \ + get_current_thread_id from _pydev_bundle import pydev_log try: @@ -497,6 +498,19 @@ def create_spawnve(original_name): return new_spawnve +def create_posix_spawn(original_name): + """ + os.posix_spawn(path, argv, env, *, file_actions=None, ... (6 more)) + os.posix_spawnp(path, argv, env, *, file_actions=None, ... (6 more)) + """ + def new_posix_spawn(path, argv, env, **kwargs): + import os + argv = patch_args(argv) + send_process_created_message() + return getattr(os, original_name)(path, argv, env, **kwargs) + return new_posix_spawn + + def create_fork_exec(original_name): """ _posixsubprocess.fork_exec(args, executable_list, close_fds, ... (13 more)) @@ -656,6 +670,10 @@ def patch_new_process_functions(): monkey_patch_os('spawnvp', create_spawnv) monkey_patch_os('spawnvpe', create_spawnve) + if IS_PY38_OR_GREATER and not IS_WINDOWS: + monkey_patch_os('posix_spawn', create_posix_spawn) + monkey_patch_os('posix_spawnp', create_posix_spawn) + if not IS_JYTHON: if not IS_WINDOWS: monkey_patch_os('fork', create_fork) @@ -691,6 +709,10 @@ def patch_new_process_functions_with_warning(): monkey_patch_os('spawnvp', create_warn_multiproc) monkey_patch_os('spawnvpe', create_warn_multiproc) + if IS_PY38_OR_GREATER and not IS_WINDOWS: + monkey_patch_os('posix_spawn', create_warn_multiproc) + monkey_patch_os('posix_spawnp', create_warn_multiproc) + if not IS_JYTHON: if not IS_WINDOWS: monkey_patch_os('fork', create_warn_multiproc) diff --git a/python/helpers/pydev/_pydevd_bundle/pydevd_constants.py b/python/helpers/pydev/_pydevd_bundle/pydevd_constants.py index d6eda3ce6043..973e0e525f19 100644 --- a/python/helpers/pydev/_pydevd_bundle/pydevd_constants.py +++ b/python/helpers/pydev/_pydevd_bundle/pydevd_constants.py @@ -108,6 +108,7 @@ IS_PY34_OR_GREATER = False IS_PY36_OR_GREATER = False IS_PY37_OR_GREATER = False IS_PY36_OR_LESSER = False +IS_PY38_OR_GREATER = False IS_PY2 = True IS_PY27 = False IS_PY24 = False @@ -119,6 +120,7 @@ try: IS_PY36_OR_GREATER = sys.version_info >= (3, 6) IS_PY37_OR_GREATER = sys.version_info >= (3, 7) IS_PY36_OR_LESSER = sys.version_info[:2] <= (3, 6) + IS_PY38_OR_GREATER = sys.version_info >= (3, 8) elif sys.version_info[0] == 2 and sys.version_info[1] == 7: IS_PY27 = True elif sys.version_info[0] == 2 and sys.version_info[1] == 4: diff --git a/python/testData/debug/test4.py b/python/testData/debug/test4.py index b8b7e91501f2..9d45d0228ace 100644 --- a/python/testData/debug/test4.py +++ b/python/testData/debug/test4.py @@ -2,4 +2,3 @@ xval = 0 xvalue1 = 1 xvalue2 = 2 print(xvalue1 + xvalue2) - diff --git a/python/testData/debug/test_posix_spawn.py b/python/testData/debug/test_posix_spawn.py new file mode 100644 index 000000000000..a9adf287883f --- /dev/null +++ b/python/testData/debug/test_posix_spawn.py @@ -0,0 +1,7 @@ +import os +import sys + +pid = os.posix_spawn(sys.executable, [sys.executable, "test2.py"], os.environ) + +pid, status = os.waitpid(pid, 0) +print(pid, status) diff --git a/python/testSrc/com/jetbrains/env/python/debug/PythonDebuggerMultiprocessingTest.java b/python/testSrc/com/jetbrains/env/python/debug/PythonDebuggerMultiprocessingTest.java new file mode 100644 index 000000000000..e83bb5cb11b6 --- /dev/null +++ b/python/testSrc/com/jetbrains/env/python/debug/PythonDebuggerMultiprocessingTest.java @@ -0,0 +1,195 @@ +// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.jetbrains.env.python.debug; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import com.intellij.openapi.util.SystemInfo; +import com.jetbrains.env.PyEnvTestCase; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.Assume; +import org.junit.Test; + +import java.util.Set; + +import static org.junit.Assert.assertFalse; + +public class PythonDebuggerMultiprocessingTest extends PyEnvTestCase { + + private static class PyDebuggerMultiprocessTask extends PyDebuggerTask { + + public PyDebuggerMultiprocessTask(@Nullable String relativeTestDataPath, String scriptName) { + super(relativeTestDataPath, scriptName); + } + + @Override + protected void init() { + setMultiprocessDebug(true); + } + } + + @Test + public void testMultiprocess() { + runPythonTest(new PyDebuggerMultiprocessTask("/debug", "test_multiprocess.py") { + @Override + public void before() { + toggleBreakpoint(getScriptName(), 9); + } + + @Override + public void testing() throws Exception { + waitForPause(); + + eval("i").hasValue("'Result:OK'"); + + resume(); + + waitForOutput("Result:OK"); + } + + @NotNull + @Override + public Set getTags() { + return Sets.newHashSet("python3"); + } + }); + } + + @Test + public void testMultiprocessingSubprocess() { + runPythonTest(new PyDebuggerMultiprocessTask("/debug", "test_multiprocess_args.py") { + @Override + public void before() { + toggleBreakpoint(getFilePath("test_remote.py"), 2); + setWaitForTermination(false); + } + + @Override + public void testing() throws Exception { + waitForPause(); + eval("sys.argv[1]").hasValue("'subprocess'"); + eval("sys.argv[2]").hasValue("'etc etc'"); + + resume(); + } + + @NotNull + @Override + public Set getTags() { + return ImmutableSet.of("-iron", "-jython"); //can't run on iron and jython + } + }); + } + + @Test + public void testMultiprocessPool() { + runPythonTest(new PyDebuggerMultiprocessTask("/debug", "test_multiprocess_pool.py") { + @Override + public void testing() throws Exception { + waitForOutput("Done"); + assertFalse(output().contains("KeyboardInterrupt")); + } + + @NotNull + @Override + public Set getTags() { + return ImmutableSet.of("-iron"); + } + }); + } + + @Test + public void testPythonSubprocessWithCParameter() { + runPythonTest(new PyDebuggerMultiprocessTask("/debug", "test_python_subprocess_with_c_parameter.py") { + @Override + public void before() { + toggleBreakpoint(getFilePath("test_python_subprocess_another_helper.py"), 2); + } + + @Override + public void testing() throws Exception { + waitForPause(); + eval("x").hasValue("42"); + resume(); + waitForOutput("Hello!"); + } + + @NotNull + @Override + public Set getTags() { + return ImmutableSet.of("-iron", "-jython"); + } + }); + } + + @Test + public void testMultiprocessProcess() { + runPythonTest(new PyDebuggerMultiprocessTask("/debug", "test_multiprocess_process.py") { + @Override + public void before() { + toggleBreakpoint(getFilePath("test_multiprocess_process.py"), 5); + setWaitForTermination(false); + } + + @Override + public void testing() throws Exception { + waitForPause(); + eval("name").hasValue("'subprocess'"); + resume(); + } + + @NotNull + @Override + public Set getTags() { + return ImmutableSet.of("-iron", "-jython"); //can't run on iron and jython + } + }); + } + + @Test + public void testSubprocess() { + runPythonTest(new PyDebuggerTask("/debug", "test_subprocess.py") { + @Override + public void before() throws Exception { + toggleBreakpoint(getFilePath(getScriptName()), 8); + } + + @Override + public void testing() throws Exception { + waitForPause(); + resume(); + waitForTerminate(); + outputContains("The subprocess finished with the return code 0."); + } + }); + } + + @Test + public void testPosixSpawn() { + runPythonTest(new PyDebuggerMultiprocessTask("/debug", "test_posix_spawn.py") { + @Override + public void before() { + toggleBreakpoint(getFilePath(getScriptName()), 6); + toggleBreakpoint(getFilePath("test2.py"), 7); + setWaitForTermination(false); + } + + @Override + public void testing() throws Exception { + Assume.assumeFalse("Windows doesn't support `posix_spawn`", SystemInfo.isWindows); + waitForPause(); + eval("z").hasValue("2"); + resume(); + waitForPause(); + resume(); + waitForOutput("Process finished with exit code 0"); + } + + @NotNull + @Override + public Set getTags() { + return ImmutableSet.of("python3.8"); + } + }); + } +} diff --git a/python/testSrc/com/jetbrains/env/python/debug/PythonDebuggerTest.java b/python/testSrc/com/jetbrains/env/python/debug/PythonDebuggerTest.java index 2336990b7a26..7b0716ff46ab 100644 --- a/python/testSrc/com/jetbrains/env/python/debug/PythonDebuggerTest.java +++ b/python/testSrc/com/jetbrains/env/python/debug/PythonDebuggerTest.java @@ -696,149 +696,6 @@ public class PythonDebuggerTest extends PyEnvTestCase { }); } - @Test - public void testMultiprocess() { - runPythonTest(new PyDebuggerTask("/debug", "test_multiprocess.py") { - @Override - protected void init() { - setMultiprocessDebug(true); - } - - @Override - public void before() { - toggleBreakpoint(getScriptName(), 9); - } - - @Override - public void testing() throws Exception { - waitForPause(); - - eval("i").hasValue("'Result:OK'"); - - resume(); - - waitForOutput("Result:OK"); - } - - @NotNull - @Override - public Set getTags() { - return Sets.newHashSet("python3"); - } - }); - } - - @Test - public void testMultiprocessingSubprocess() { - runPythonTest(new PyDebuggerTask("/debug", "test_multiprocess_args.py") { - @Override - protected void init() { - setMultiprocessDebug(true); - } - - @Override - public void before() { - toggleBreakpoint(getFilePath("test_remote.py"), 2); - setWaitForTermination(false); - } - - @Override - public void testing() throws Exception { - waitForPause(); - eval("sys.argv[1]").hasValue("'subprocess'"); - eval("sys.argv[2]").hasValue("'etc etc'"); - - resume(); - } - - @NotNull - @Override - public Set getTags() { - return ImmutableSet.of("-iron", "-jython"); //can't run on iron and jython - } - }); - } - - @Test - public void testMultiprocessPool() { - runPythonTest(new PyDebuggerTask("/debug", "test_multiprocess_pool.py") { - @Override - protected void init() { - setMultiprocessDebug(true); - } - - @Override - public void testing() throws Exception { - waitForOutput("Done"); - assertFalse(output().contains("KeyboardInterrupt")); - } - - @NotNull - @Override - public Set getTags() { - return ImmutableSet.of("-iron"); - } - }); - } - - @Test - public void testPythonSubprocessWithCParameter() { - runPythonTest(new PyDebuggerTask("/debug", "test_python_subprocess_with_c_parameter.py") { - @Override - protected void init() { - setMultiprocessDebug(true); - } - - @Override - public void before() { - toggleBreakpoint(getFilePath("test_python_subprocess_another_helper.py"), 2); - } - - @Override - public void testing() throws Exception { - waitForPause(); - eval("x").hasValue("42"); - resume(); - waitForOutput("Hello!"); - } - - @NotNull - @Override - public Set getTags() { - return ImmutableSet.of("-iron", "-jython"); - } - }); - } - - @Test - public void testMultiprocessProcess() { - runPythonTest(new PyDebuggerTask("/debug", "test_multiprocess_process.py") { - @Override - protected void init() { - setMultiprocessDebug(true); - } - - @Override - public void before() { - toggleBreakpoint(getFilePath("test_multiprocess_process.py"), 5); - setWaitForTermination(false); - } - - @Override - public void testing() throws Exception { - waitForPause(); - eval("name").hasValue("'subprocess'"); - resume(); - } - - @NotNull - @Override - public Set getTags() { - return ImmutableSet.of("-iron", "-jython"); //can't run on iron and jython - } - }); - } - @Test public void testStepOverYieldFrom() { runPythonTest(new PyDebuggerTask("/debug", "test_step_over_yield.py") { @@ -1881,24 +1738,6 @@ public class PythonDebuggerTest extends PyEnvTestCase { }); } - @Test - public void testSubprocess() { - runPythonTest(new PyDebuggerTask("/debug", "test_subprocess.py") { - @Override - public void before() throws Exception { - toggleBreakpoint(getFilePath(getScriptName()), 8); - } - - @Override - public void testing() throws Exception { - waitForPause(); - resume(); - waitForTerminate(); - outputContains("The subprocess finished with the return code 0."); - } - }); - } - @Test public void testCodeEvaluationWithGeneratorExpression() { runPythonTest(new PyDebuggerTaskTagAware("/debug", "test_code_eval_with_generator_expr.py") {