Fix Attach to process with Python 3 on Windows (taken from PyDev) (PY-17986)

python/helpers/pydev/pydevd_attach_to_process/add_code_to_python_process.py

@@ -280,8 +280,8 @@ def run_python_code_windows(pid, python_code, connect_debugger_tracing=False, sh
 
     if is_64 != is_python_64bit():
         raise RuntimeError("The architecture of the Python used to connect doesn't match the architecture of the target.\n"
-                           "Target 64 bits: %s\n"
-                           "Current Python 64 bits: %s" % (is_64, is_python_64bit()))
+        "Target 64 bits: %s\n"
+        "Current Python 64 bits: %s" % (is_64, is_python_64bit()))
 
     print('Connecting to %s bits target' % (bits,))
     assert resolve_label(process, compat.b('PyGILState_Ensure'))
@@ -366,16 +366,16 @@ def run_python_code_windows(pid, python_code, connect_debugger_tracing=False, sh
 
 
     # Uncomment to see the disassembled version of what we just did...
-    #     with open('f.asm', 'wb') as stream:
-    #         stream.write(code)
-    #
-    #     exe = r'x:\nasm\nasm-2.07-win32\nasm-2.07\ndisasm.exe'
-    #     if is_64:
-    #         arch = '64'
-    #     else:
-    #         arch = '32'
-    #
-    #     subprocess.call((exe + ' -b %s f.asm' % arch).split())
+#     with open('f.asm', 'wb') as stream:
+#         stream.write(code)
+#
+#     exe = r'x:\nasm\nasm-2.07-win32\nasm-2.07\ndisasm.exe'
+#     if is_64:
+#         arch = '64'
+#     else:
+#         arch = '32'
+#
+#     subprocess.call((exe + ' -b %s f.asm' % arch).split())
 
     print('Injecting code to target process')
     thread, _thread_address = process.inject_code(code, 0)
@@ -428,11 +428,11 @@ def run_python_code_linux(pid, python_code, connect_debugger_tracing=False, show
         '--nw',  # no gui interface
         '--nh',  # no ~/.gdbinit
         '--nx',  # no .gdbinit
-        #         '--quiet',  # no version number on startup
+#         '--quiet',  # no version number on startup
         '--pid',
         str(pid),
         '--batch',
-        #         '--batch-silent',
+#         '--batch-silent',
     ]
 
     cmd.extend(["--eval-command='set scheduler-locking off'"])  # If on we'll deadlock.
@@ -449,7 +449,7 @@ def run_python_code_linux(pid, python_code, connect_debugger_tracing=False, show
     if connect_debugger_tracing:
         cmd.extend([
             "--command='%s'" % (gdb_threads_settrace_file,),
-            ])
+        ])
 
     #print ' '.join(cmd)
 
@@ -527,8 +527,8 @@ def run_python_code_mac(pid, python_code, connect_debugger_tracing=False, show_d
         "-o 'process attach --pid %d'"%pid,
         "-o 'command script import \"%s\"'" % (lldb_prepare_file,),
         "-o 'load_lib_and_attach \"%s\" %s \"%s\" %s'" % (target_dll,
-                                                          is_debug, python_code, show_debug_info),
-        ])
+            is_debug, python_code, show_debug_info),
+    ])
 
 
     if connect_debugger_tracing:
@@ -556,7 +556,7 @@ def run_python_code_mac(pid, python_code, connect_debugger_tracing=False, show_d
         env=env,
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
-    )
+        )
     print('Running lldb in target process.')
     out, err = p.communicate()
     print('stdout: %s' % (out,))

python/helpers/pydev/pydevd_attach_to_process/dll/attach.cpp

@@ -594,16 +594,6 @@ void IncRef(PyObject* object) {
     object->ob_refcnt++;
 }
 
-// Structure for our shared memory communication, aligned to be identical on 64-bit and 32-bit
-struct MemoryBuffer {
-    int PortNumber;             // offset 0-4
-    __declspec(align(8)) HANDLE AttachStartingEvent;  // offset 8 - 16
-    __declspec(align(8)) HANDLE AttachDoneEvent;  // offset 16 - 24
-    __declspec(align(8)) int ErrorNumber;            // offset 24-28
-    int VersionNumber;          // offset 28-32
-    char DebugId[1];            // null terminated string
-};
-
 
 // Ensures handles are closed when they go out of scope
 class HandleHolder {
@@ -623,8 +613,8 @@ long GetPythonThreadId(PythonVersion version, PyThreadState* curThread) {
         threadId = ((PyThreadState_25_27*)curThread)->thread_id;
     } else if (PyThreadState_30_33::IsFor(version)) {
         threadId = ((PyThreadState_30_33*)curThread)->thread_id;
-    } else if (PyThreadState_34::IsFor(version)) {
-        threadId = ((PyThreadState_34*)curThread)->thread_id;
+    } else if (PyThreadState_34_36::IsFor(version)) {
+        threadId = ((PyThreadState_34_36*)curThread)->thread_id;
     }
     return threadId;
 }
@@ -1090,7 +1080,7 @@ extern "C"
         auto pyFalse = boolFromLong(0);
 
         
-        auto pydevdTracingMod = PyObjectHolder(isDebug, pyImportMod("_pydevd_bundle.pydevd_tracing"));
+        auto pydevdTracingMod = PyObjectHolder(isDebug, pyImportMod("pydevd_tracing"));
         if (*pydevdTracingMod == nullptr) {
             if(showDebugInfo){
                 std::cout << "pydevd_tracing module null! " << std::endl << std::flush;
@@ -1201,8 +1191,8 @@ extern "C"
                             frame = ((PyThreadState_25_27*)curThread)->frame;
                         } else if (PyThreadState_30_33::IsFor(version)) {
                             frame = ((PyThreadState_30_33*)curThread)->frame;
-                        } else if (PyThreadState_34::IsFor(version)) {
-                            frame = ((PyThreadState_34*)curThread)->frame;
+                        } else if (PyThreadState_34_36::IsFor(version)) {
+                            frame = ((PyThreadState_34_36*)curThread)->frame;
                         }else{
                             if(showDebugInfo){
                                 std::cout << "Python version not handled! " << version << std::endl << std::flush;

python/helpers/pydev/pydevd_attach_to_process/dll/compile_dll.bat

@@ -1,9 +1,9 @@
-call "C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat" x86
+call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x86
 cl -DUNICODE -D_UNICODE /EHsc /LD attach.cpp /link /out:attach_x86.dll
 copy attach_x86.dll ..\attach_x86.dll /Y
 
 
 
-call "C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat" x86_amd64
+call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x86_amd64
 cl -DUNICODE -D_UNICODE /EHsc /LD attach.cpp /link /out:attach_amd64.dll
 copy attach_amd64.dll ..\attach_amd64.dll /Y

python/helpers/pydev/pydevd_attach_to_process/dll/python.h

@@ -25,7 +25,9 @@ enum PythonVersion {
     PythonVersion_31 = 0x0301,
     PythonVersion_32 = 0x0302,
     PythonVersion_33 = 0x0303,
-    PythonVersion_34 = 0x0304
+    PythonVersion_34 = 0x0304,
+    PythonVersion_35 = 0x0305,
+    PythonVersion_36 = 0x0306
 };
 
 
@@ -110,8 +112,8 @@ public:
     }
 };
 
-// 3.3-3.4
-class PyCodeObject33_34 : public PyObject {
+// 3.3-3.5
+class PyCodeObject33_35 : public PyObject {
 public:
     int co_argcount;            /* #arguments, except *args */
     int co_kwonlyargcount;      /* #keyword only arguments */
@@ -133,15 +135,46 @@ public:
     void *co_zombieframe;       /* for optimization only (see frameobject.c) */
 
     static bool IsFor(int majorVersion, int minorVersion) {
-        return majorVersion == 3 && (minorVersion >= 3 && minorVersion <= 4);
+        return majorVersion == 3 && (minorVersion >= 3 && minorVersion <= 5);
     }
 
     static bool IsFor(PythonVersion version) {
-        return version >= PythonVersion_33 && version <= PythonVersion_34;
+        return version >= PythonVersion_33 && version <= PythonVersion_35;
     }
 };
 
-// 2.5 - 3.1
+// 3.6
+class PyCodeObject36 : public PyObject {
+public:
+    int co_argcount;            /* #arguments, except *args */
+    int co_kwonlyargcount;      /* #keyword only arguments */
+    int co_nlocals;             /* #local variables */
+    int co_stacksize;           /* #entries needed for evaluation stack */
+    int co_flags;               /* CO_..., see below */
+    int co_firstlineno;         /* first source line number */
+    PyObject *co_code;          /* instruction opcodes */
+    PyObject *co_consts;        /* list (constants used) */
+    PyObject *co_names;         /* list of strings (names used) */
+    PyObject *co_varnames;      /* tuple of strings (local variable names) */
+    PyObject *co_freevars;      /* tuple of strings (free variable names) */
+    PyObject *co_cellvars;      /* tuple of strings (cell variable names) */
+    /* The rest doesn't count for hash or comparisons */
+    unsigned char *co_cell2arg; /* Maps cell vars which are arguments. */
+    PyObject *co_filename;      /* unicode (where it was loaded from) */
+    PyObject *co_name;          /* unicode (name, for reference) */
+    PyObject *co_lnotab;        /* string (encoding addr<->lineno mapping) */
+    void *co_zombieframe;       /* for optimization only (see frameobject.c) */
+
+    static bool IsFor(int majorVersion, int minorVersion) {
+        return majorVersion == 3 && minorVersion >= 6;
+    }
+
+    static bool IsFor(PythonVersion version) {
+        return version >= PythonVersion_36;
+    }
+};
+
+// 2.5 - 3.6
 class PyFunctionObject : public PyObject {
 public:
     PyObject *func_code;    /* A code object */
@@ -172,7 +205,7 @@ typedef struct {
     long hash;          /* Hash value; -1 if not set */
 } PyUnicodeObject;
 
-// 2.4 - 3.4 compatible
+// 2.4 - 3.6 compatible
 class PyFrameObject : public PyVarObject {
 public:
     PyFrameObject *f_back;  /* previous frame, or NULL */
@@ -213,7 +246,7 @@ public:
     }
 };
 
-class PyFrameObject34 : public PyFrameObject {
+class PyFrameObject34_36 : public PyFrameObject {
 public:
     /* Borrowed reference to a generator, or NULL */
     PyObject *f_gen;
@@ -228,14 +261,14 @@ public:
     PyObject *f_localsplus[1];    /* locals+stack, dynamically sized */
 
     static bool IsFor(int majorVersion, int minorVersion) {
-        return majorVersion == 3 && minorVersion == 4;
+        return majorVersion == 3 && minorVersion >= 4 && minorVersion <= 6;
     }
 };
 
 
 typedef void (*destructor)(PyObject *);
 
-// 2.4 - 3.4
+// 2.4 - 3.6
 class PyMethodDef {
 public:
     char    *ml_name;    /* The name of the built-in function/method */
@@ -243,7 +276,7 @@ public:
 
 
 // 
-// 2.4 - 3.4, 2.4 has different compat in 64-bit but we don't support any of the released 64-bit platforms (which includes only IA-64)
+// 2.4 - 3.5, 2.4 has different compat in 64-bit but we don't support any of the released 64-bit platforms (which includes only IA-64)
 // While these are compatible there are fields only available on later versions.
 class PyTypeObject : public PyVarObject {
 public:
@@ -256,7 +289,10 @@ public:
     void* tp_print;
     void* tp_getattr;
     void* tp_setattr;
-    void* tp_compare;
+    union {
+        void* tp_compare; /* 2.4 - 3.4 */
+        void* tp_as_async; /* 3.5 - 3.6 */
+    };
     void* tp_repr;
 
     /* Method suites for standard classes */
@@ -325,7 +361,7 @@ public:
     unsigned int tp_version_tag;
 };
 
-// 2.4 - 3.4
+// 2.4 - 3.6
 class PyTupleObject : public PyVarObject {
 public:
     PyObject *ob_item[1];
@@ -336,7 +372,7 @@ public:
      */
 };
 
-// 2.4 - 3.4
+// 2.4 - 3.6
 class PyCFunctionObject : public PyObject {
 public:
     PyMethodDef *m_ml;      /* Description of the C function to call */
@@ -467,7 +503,7 @@ public:
     }
 };
 
-class PyThreadState_34 : public PyThreadState {
+class PyThreadState_34_36 : public PyThreadState {
 public:
     PyThreadState *prev;
     PyThreadState *next;
@@ -507,11 +543,11 @@ public:
 
     /* XXX signal handlers should also be here */
     static bool IsFor(int majorVersion, int minorVersion) {
-        return majorVersion == 3 && minorVersion == 4;
+        return majorVersion == 3 && minorVersion >= 4 && minorVersion <= 6;
     }
 
     static bool IsFor(PythonVersion version) {
-        return version == PythonVersion_34;
+        return version >= PythonVersion_34 && version <= PythonVersion_36;
     }
 };
 
@@ -563,6 +599,8 @@ static PythonVersion GetPythonVersion(HMODULE hMod) {
                 case '2': return PythonVersion_32;
                 case '3': return PythonVersion_33;
                 case '4': return PythonVersion_34;
+                case '5': return PythonVersion_35;
+                case '6': return PythonVersion_36;
                 }
             }
         }
@@ -570,4 +608,4 @@ static PythonVersion GetPythonVersion(HMODULE hMod) {
     return PythonVersion_Unknown;
 }
 
-#endif
+#endif

python/helpers/pydev/pydevd_attach_to_process/linux/attach_linux.c

@@ -271,7 +271,7 @@ int _PYDEVD_ExecWithGILSetSysStrace(bool showDebugInfo, bool isDebug){
         }
         return 13;
     }
-
+    
     PyObjectHolder traceFunc = PyObjectHolder(isDebug, pyGetAttr(globalDbg.ToPython(), "trace_dispatch"));
     CHECK_NULL(traceFunc.ToPython(), "pydevd.GetGlobalDebugger().trace_dispatch returned null!\n", 14);
     

python/helpers/pydev/pydevd_attach_to_process/linux/compile_mac.sh

@@ -1,4 +1,4 @@
-g++ -fPIC -D_REENTRANT -arch x86_64 -I. -c -o attach_linux_x86_64.o attach_linux.c
+g++ -fPIC -D_REENTRANT -arch x86_64 I. -c -o attach_linux_x86_64.o attach_linux.c
 g++ -dynamiclib -arch x86_64 -o attach_x86_64.dylib attach_linux_x86_64.o -lc
 
 

python/helpers/pydev/pydevd_attach_to_process/winappdbg/system.py

@@ -567,7 +567,7 @@ class System (_ProcessContainer):
                 try:
 
                     # Load a specific dbghelp.dll file
-                    debug.system.load_dbghelp("C:\Some folder\dbghelp.dll")
+                    debug.system.load_dbghelp("C:\\Some folder\\dbghelp.dll")
 
                     # Start a new process for debugging
                     debug.execv( argv )