Merge remote-tracking branch 'origin/master'

java/debugger/impl/src/com/intellij/debugger/engine/JavaStackFrame.java

@@ -68,6 +68,8 @@ import java.util.*;
  */
 public class JavaStackFrame extends XStackFrame implements JVMStackFrameInfoProvider {
   private static final Logger LOG = Logger.getInstance(JavaStackFrame.class);
+  public static final DummyMessageValueNode LOCAL_VARIABLES_INFO_UNAVAILABLE_MESSAGE_NODE =
+    new DummyMessageValueNode(MessageDescriptor.LOCAL_VARIABLES_INFO_UNAVAILABLE.getLabel(), XDebuggerUIConstants.INFORMATION_MESSAGE_ICON);
 
   private final DebugProcessImpl myDebugProcess;
   @Nullable private final XSourcePosition myXSourcePosition;
@@ -321,7 +323,7 @@ public class JavaStackFrame extends XStackFrame implements JVMStackFrameInfoProv
     }
     catch (EvaluateException e) {
       if (e.getCause() instanceof AbsentInformationException) {
-        children.add(new DummyMessageValueNode(MessageDescriptor.LOCAL_VARIABLES_INFO_UNAVAILABLE.getLabel(), XDebuggerUIConstants.INFORMATION_MESSAGE_ICON));
+        children.add(LOCAL_VARIABLES_INFO_UNAVAILABLE_MESSAGE_NODE);
         // trying to collect values from variable slots
         try {
           for (Map.Entry<DecompiledLocalVariable, Value> entry : LocalVariablesUtil.fetchValues(getStackFrameProxy(), debugProcess, true).entrySet()) {

java/debugger/impl/src/com/intellij/debugger/memory/utils/StackFrameItem.java

@@ -24,7 +24,6 @@ import com.intellij.debugger.settings.CaptureConfigurable;
 import com.intellij.debugger.settings.DebuggerSettings;
 import com.intellij.debugger.settings.NodeRendererSettings;
 import com.intellij.debugger.ui.breakpoints.StackCapturingLineBreakpoint;
-import com.intellij.debugger.ui.impl.watch.MessageDescriptor;
 import com.intellij.debugger.ui.tree.render.ClassRenderer;
 import com.intellij.icons.AllIcons;
 import com.intellij.openapi.diagnostic.Logger;
@@ -61,9 +60,6 @@ public class StackFrameItem {
   private static final List<XNamedValue> VARS_NOT_CAPTURED = Collections.singletonList(
     JavaStackFrame.createMessageNode(DebuggerBundle.message("message.node.local.variables.not.captured"),
                                      XDebuggerUIConstants.INFORMATION_MESSAGE_ICON));
-  private static final XNamedValue VAR_NO_DEBUG_INFO =
-    JavaStackFrame.createMessageNode(MessageDescriptor.LOCAL_VARIABLES_INFO_UNAVAILABLE.getLabel(),
-                                     XDebuggerUIConstants.INFORMATION_MESSAGE_ICON);
 
   private final Location myLocation;
   private final List<XNamedValue> myVariables;
@@ -125,7 +121,7 @@ public class StackFrameItem {
               }
               catch (EvaluateException e) {
                 if (e.getCause() instanceof AbsentInformationException) {
-                  vars.add(VAR_NO_DEBUG_INFO);
+                  vars.add(JavaStackFrame.LOCAL_VARIABLES_INFO_UNAVAILABLE_MESSAGE_NODE);
                   // only args for frames w/o debug info for now
                   try {
                     for (Map.Entry<DecompiledLocalVariable, Value> entry : LocalVariablesUtil

java/java-tests/testSrc/com/intellij/testFramework/fixtures/MultiModuleJava9ProjectDescriptor.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2016 JetBrains s.r.o.
+ * Copyright 2000-2017 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -46,7 +46,7 @@ object MultiModuleJava9ProjectDescriptor : DefaultLightProjectDescriptor() {
         if (this == MAIN) LightPlatformTestCase.getSourceRoot() else TempFileSystem.getInstance().findFileByPath("/$rootName")!!
   }
 
-  override fun getSdk(): Sdk = IdeaTestUtil.getMockJdk18()
+  override fun getSdk(): Sdk = IdeaTestUtil.getMockJdk9()
 
   override fun setUpProject(project: Project, handler: SetupHandler) {
     super.setUpProject(project, handler)

platform/diff-impl/tests/com/intellij/diff/comparison/WordComparisonUtilTest.kt

@@ -338,6 +338,23 @@ class WordComparisonUtilTest : ComparisonUtilTestBase() {
     }
   }
 
+  fun `test trailing punctuation`() {
+    lines_inner {
+      ("X = { };" - "X = { _ };")
+      ("        " - "     --   ").default()
+      ("        " - "          ").trim()
+      testAll()
+    }
+
+    // TODO
+    lines_inner {
+      ("X = { };_" - "X = { _ };_")
+      ("      -- " - "       ----").default()
+      ("      -- " - "        -- ").trim()
+      testAll()
+    }
+  }
+
   fun `test legacy cases from ByWordTest`() {
     lines_inner {
       ("abc def, 123" - "ab def, 12")

platform/util/src/com/intellij/util/text/SemVer.java

@@ -15,6 +15,7 @@
  */
 package com.intellij.util.text;
 
+import com.intellij.openapi.util.text.StringUtil;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
@@ -87,13 +88,35 @@ public class SemVer implements Comparable<SemVer> {
 
   @Nullable
   public static SemVer parseFromText(@NotNull String text) {
-    return SemVerMatcher.parseFromText(text);
+    int majorEndInd = text.indexOf('.');
+    if (majorEndInd < 0) {
+      return null;
+    }
+    int major = StringUtil.parseInt(text.substring(0, majorEndInd), -1);
+    int minorEndInd = text.indexOf('.', majorEndInd + 1);
+    if (minorEndInd < 0) {
+      return null;
+    }
+    int minor = StringUtil.parseInt(text.substring(majorEndInd + 1, minorEndInd), -1);
+    final String patchStr;
+    int dashInd = text.indexOf('-', minorEndInd + 1);
+    if (dashInd >= 0) {
+      patchStr = text.substring(minorEndInd + 1, dashInd);
+    }
+    else {
+      patchStr = text.substring(minorEndInd + 1);
+    }
+    int patch = StringUtil.parseInt(patchStr, -1);
+    if (major >= 0 && minor >= 0 && patch >= 0) {
+      return new SemVer(text, major, minor, patch);
+    }
+    return null;
   }
 
   @NotNull
   public static SemVer parseFromTextNonNullize(@Nullable final String text) {
     if (text == null) return UNKNOWN;
-    final SemVer ver = SemVerMatcher.parseFromText(text);
+    final SemVer ver = parseFromText(text);
     return ver == null ? UNKNOWN : ver;
   }
 

platform/util/src/com/intellij/util/text/SemVerMatcher.java

@@ -1,61 +0,0 @@
-/*
- * Copyright 2000-2017 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.intellij.util.text;
-
-import com.intellij.openapi.util.text.StringUtil;
-import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
-
-/**
- * @author Irina.Chernushina on 3/9/2017.
- */
-public class SemVerMatcher {
-  @Nullable
-  public static SemVer parseFromText(@NotNull String text) {
-    return parseFromText(text, false);
-  }
-
-  @Nullable
-  public static SemVer parseFromText(@NotNull String text, final boolean allowPartial) {
-    int majorEndInd = text.indexOf('.');
-    if (majorEndInd < 0) {
-      if (!allowPartial) return null;
-      final int major = StringUtil.parseInt(text, -1);
-      return major < 0 ? null : new SemVer(text, major, 0, 0);
-    }
-    int major = StringUtil.parseInt(text.substring(0, majorEndInd), -1);
-    int minorEndInd = text.indexOf('.', majorEndInd + 1);
-    if (minorEndInd < 0) {
-      if (!allowPartial) return null;
-      final int minor = StringUtil.parseInt(text.substring(majorEndInd + 1), -1);
-      return new SemVer(text, major, minor < 0 ? 0 : minor, 0);
-    }
-    int minor = StringUtil.parseInt(text.substring(majorEndInd + 1, minorEndInd), -1);
-    final String patchStr;
-    int dashInd = text.indexOf('-', minorEndInd + 1);
-    if (dashInd >= 0) {
-      patchStr = text.substring(minorEndInd + 1, dashInd);
-    }
-    else {
-      patchStr = text.substring(minorEndInd + 1);
-    }
-    int patch = StringUtil.parseInt(patchStr, -1);
-    if (major >= 0 && minor >= 0 && patch >= 0) {
-      return new SemVer(text, major, minor, patch);
-    }
-    return null;
-  }
-}

plugins/gradle/src/org/jetbrains/plugins/gradle/service/project/GradleProjectResolver.java

@@ -309,7 +309,7 @@ public class GradleProjectResolver implements ExternalSystemProjectResolver<Grad
       throw new IllegalStateException("No modules found for the target project: " + ideaProject);
     }
 
-    Collection<IdeaModule> includedModules = exposeCompositeBuild(allModels, projectDataNode);
+    Collection<IdeaModule> includedModules = exposeCompositeBuild(allModels, resolverCtx, projectDataNode);
     final Map<String /* module id */, Pair<DataNode<ModuleData>, IdeaModule>> moduleMap = ContainerUtilRt.newHashMap();
     final Map<String /* module id */, Pair<DataNode<GradleSourceSetData>, ExternalSourceSet>> sourceSetsMap = ContainerUtil.newHashMap();
     projectDataNode.putUserData(RESOLVED_SOURCE_SETS, sourceSetsMap);
@@ -421,8 +421,12 @@ public class GradleProjectResolver implements ExternalSystemProjectResolver<Grad
 
   @NotNull
   private static Collection<IdeaModule> exposeCompositeBuild(ProjectImportAction.AllModels allModels,
+                                                             DefaultProjectResolverContext resolverCtx,
                                                              DataNode<ProjectData> projectDataNode) {
-    CompositeBuildData compositeBuildData = null;
+    if(resolverCtx.getSettings() != null && !resolverCtx.getSettings().getExecutionWorkspace().getBuildParticipants().isEmpty()) {
+      return Collections.emptyList();
+    }
+    CompositeBuildData compositeBuildData;
     List<IdeaModule> gradleIncludedModules = new SmartList<>();
     List<IdeaProject> includedBuilds = allModels.getIncludedBuilds();
     if (!includedBuilds.isEmpty()) {
@@ -432,9 +436,7 @@ public class GradleProjectResolver implements ExternalSystemProjectResolver<Grad
         if (!project.getModules().isEmpty()) {
           String rootProjectName = project.getName();
           BuildParticipant buildParticipant = new BuildParticipant();
-          for (IdeaModule ideaModule : project.getModules()) {
-            gradleIncludedModules.add(ideaModule);
-          }
+          gradleIncludedModules.addAll(project.getModules());
           GradleProject gradleProject = project.getModules().getAt(0).getGradleProject();
           String projectPath = null;
           do {

plugins/gradle/src/org/jetbrains/plugins/gradle/service/project/data/CompositeBuildDataService.java

@@ -54,7 +54,9 @@ public class CompositeBuildDataService extends AbstractProjectDataService<Compos
       if (projectData != null) {
         GradleProjectSettings projectSettings =
           GradleSettings.getInstance(project).getLinkedProjectSettings(projectData.getLinkedExternalProjectPath());
-        if (projectSettings != null) {
+        if (projectSettings != null &&
+            projectSettings.getCompositeBuild() != null &&
+            projectSettings.getCompositeBuild().getCompositeDefinitionSource() == CompositeDefinitionSource.SCRIPT) {
           projectSettings.setCompositeBuild(null);
         }
       }

python/helpers/pydev/_pydev_bundle/pydev_monkey.py

@@ -21,24 +21,17 @@ def log_error_once(msg):
 
 pydev_src_dir = os.path.dirname(os.path.dirname(__file__))
 
-def _get_pydevd_args():
-    new_args = []
-    for x in sys.original_argv:
-        new_args.append(x)
-        if x == '--file':
-            break
-    return new_args
 
-def _get_python_c_args(host, port, indC, args):
+def _get_python_c_args(host, port, indC, args, setup):
     host_literal = "'" + host + "'" if host is not None else 'None'
     return ("import sys; sys.path.append(r'%s'); import pydevd; "
             "pydevd.settrace(host=%s, port=%s, suspend=False, trace_only_current_thread=False, patch_multiprocessing=True); "
-            "sys.original_argv = %s; %s"
+            "from pydevd import SetupHolder; SetupHolder.setup = %s; %s"
             ) % (
                pydev_src_dir,
                host_literal,
                port,
-               _get_pydevd_args(),
+               setup,
                args[indC + 1])
 
 def _get_host_port():
@@ -105,6 +98,7 @@ def patch_args(args):
         log_debug("Patching args: %s"% str(args))
         args = remove_quotes_from_args(args)
 
+        from pydevd import SetupHolder
         import sys
         new_args = []
         i = 0
@@ -122,7 +116,7 @@ def patch_args(args):
 
                 if port is not None:
                     new_args.extend(args)
-                    new_args[indC + 1] = _get_python_c_args(host, port, indC, args)
+                    new_args[indC + 1] = _get_python_c_args(host, port, indC, args, SetupHolder.setup)
                     return quote_args(new_args)
             else:
                 # Check for Python ZIP Applications and don't patch the args for them.
@@ -155,7 +149,8 @@ def patch_args(args):
         # Original args should be something as:
         # ['X:\\pysrc\\pydevd.py', '--multiprocess', '--print-in-debugger-startup',
         #  '--vm_type', 'python', '--client', '127.0.0.1', '--port', '56352', '--file', 'x:\\snippet1.py']
-        original = sys.original_argv[:]
+        from _pydevd_bundle.pydevd_command_line_handling import setup_to_argv
+        original = setup_to_argv(SetupHolder.setup) + ['--file']
         while i < len(args):
             if args[i] == '-m':
                 # Always insert at pos == 1 (i.e.: pydevd "--module" --multiprocess ...)

python/helpers/pydev/_pydev_bundle/pydev_monkey_qt.py

@@ -5,14 +5,20 @@ def set_trace_in_qt():
     from _pydevd_bundle.pydevd_comm import get_global_debugger
     debugger = get_global_debugger()
     if debugger is not None:
-        pydevd_tracing.SetTrace(debugger.trace_dispatch, debugger.frame_eval_func)
+        pydevd_tracing.SetTrace(debugger.trace_dispatch)
 
 
 _patched_qt = False
-def patch_qt():
+def patch_qt(qt_support_mode):
     '''
     This method patches qt (PySide, PyQt4, PyQt5) so that we have hooks to set the tracing for QThread.
     '''
+    if not qt_support_mode:
+        return
+
+    if qt_support_mode is True or qt_support_mode == 'True':
+        # do not break backward compatibility
+        qt_support_mode = 'auto'
 
     # Avoid patching more than once
     global _patched_qt
@@ -21,38 +27,53 @@ def patch_qt():
 
     _patched_qt = True
 
+    if qt_support_mode == 'auto':
 
-    # Ok, we have an issue here:
-    # PyDev-452: Selecting PyQT API version using sip.setapi fails in debug mode
-    # http://pyqt.sourceforge.net/Docs/PyQt4/incompatible_apis.html
-    # Mostly, if the user uses a different API version (i.e.: v2 instead of v1),
-    # that has to be done before importing PyQt4/5 modules (PySide doesn't have this issue
-    # as it only implements v2).
-
-    patch_qt_on_import = None
-    try:
-        import PySide  # @UnresolvedImport @UnusedImport
-    except:
+        patch_qt_on_import = None
         try:
-            import PyQt4  # @UnresolvedImport @UnusedImport
-            patch_qt_on_import = 'PyQt4'
+            import PySide  # @UnresolvedImport @UnusedImport
+            qt_support_mode = 'pyside'
         except:
             try:
                 import PyQt5  # @UnresolvedImport @UnusedImport
-                patch_qt_on_import = 'PyQt5'
+                qt_support_mode = 'pyqt5'
             except:
-                return
+                try:
+                    import PyQt4  # @UnresolvedImport @UnusedImport
+                    qt_support_mode = 'pyqt4'
+                except:
+                    return
+
+
+    if qt_support_mode == 'pyside':
+        import PySide.QtCore  # @UnresolvedImport
+        _internal_patch_qt(PySide.QtCore)
+
+    elif qt_support_mode == 'pyqt5':
+        import PyQt5.QtCore  # @UnresolvedImport
+        _internal_patch_qt(PyQt5.QtCore)
+
+    elif qt_support_mode == 'pyqt4':
+        # Ok, we have an issue here:
+        # PyDev-452: Selecting PyQT API version using sip.setapi fails in debug mode
+        # http://pyqt.sourceforge.net/Docs/PyQt4/incompatible_apis.html
+        # Mostly, if the user uses a different API version (i.e.: v2 instead of v1),
+        # that has to be done before importing PyQt4 modules (PySide/PyQt5 don't have this issue
+        # as they only implements v2).
+        patch_qt_on_import = 'PyQt4'
+        def get_qt_core_module():
+            import PyQt4.QtCore  # @UnresolvedImport
+            return PyQt4.QtCore
+        _patch_import_to_patch_pyqt_on_import(patch_qt_on_import, get_qt_core_module)
 
-    if patch_qt_on_import:
-        _patch_import_to_patch_pyqt_on_import(patch_qt_on_import)
     else:
-        _internal_patch_qt()
+        raise ValueError('Unexpected qt support mode: %s' % (qt_support_mode,))
 
 
-def _patch_import_to_patch_pyqt_on_import(patch_qt_on_import):
+def _patch_import_to_patch_pyqt_on_import(patch_qt_on_import, get_qt_core_module):
     # I don't like this approach very much as we have to patch __import__, but I like even less
     # asking the user to configure something in the client side...
-    # So, our approach is to patch PyQt4/5 right before the user tries to import it (at which
+    # So, our approach is to patch PyQt4 right before the user tries to import it (at which
     # point he should've set the sip api version properly already anyways).
 
     dotted = patch_qt_on_import + '.'
@@ -67,7 +88,7 @@ def _patch_import_to_patch_pyqt_on_import(patch_qt_on_import):
         if patch_qt_on_import == name or name.startswith(dotted):
             builtins.__import__ = original_import
             cancel_patches_in_sys_module()
-            _internal_patch_qt() # Patch it only when the user would import the qt module
+            _internal_patch_qt(get_qt_core_module()) # Patch it only when the user would import the qt module
         return original_import(name, *args, **kwargs)
 
     try:
@@ -77,47 +98,36 @@ def _patch_import_to_patch_pyqt_on_import(patch_qt_on_import):
     builtins.__import__ = patched_import
 
 
-def _internal_patch_qt():
-    try:
-        from PySide import QtCore # @UnresolvedImport @UnusedImport
-    except:
-        try:
-            from PyQt4 import QtCore # @UnresolvedImport @UnusedImport @Reimport
-        except:
-            try:
-                from PyQt5 import QtCore # @UnresolvedImport @UnusedImport @Reimport
-            except:
-                return
-
+def _internal_patch_qt(QtCore):
     _original_thread_init = QtCore.QThread.__init__
     _original_runnable_init = QtCore.QRunnable.__init__
     _original_QThread = QtCore.QThread
 
+    class StartedSignalWrapper(QtCore.QObject):  # Wrapper for the QThread.started signal
 
-    class FuncWrapper:
-
-        def __init__(self, original):
-            self._original = original
-
-        def __call__(self, *args, **kwargs):
-            set_trace_in_qt()
-            return self._original(*args, **kwargs)
-
-    class StartedSignalWrapper:  # Wrapper for the QThread.started signal
+        try:
+            _signal = QtCore.Signal()  # @UndefinedVariable
+        except:
+            _signal = QtCore.pyqtSignal()  # @UndefinedVariable
 
         def __init__(self, thread, original_started):
+            QtCore.QObject.__init__(self)
             self.thread = thread
             self.original_started = original_started
+            self._signal.connect(self._on_call)
+            self.original_started.connect(self._signal)
 
-        def connect(self, func, *args, **kwargs):
-            return self.original_started.connect(FuncWrapper(func), *args, **kwargs)
+        def connect(self, *args, **kwargs):
+            return self._signal.connect(*args, **kwargs)
 
         def disconnect(self, *args, **kwargs):
-            return self.original_started.disconnect(*args, **kwargs)
+            return self._signal.disconnect(*args, **kwargs)
 
         def emit(self, *args, **kwargs):
-            return self.original_started.emit(*args, **kwargs)
+            return self._signal.emit(*args, **kwargs)
 
+        def _on_call(self, *args, **kwargs):
+            set_trace_in_qt()
 
     class ThreadWrapper(QtCore.QThread):  # Wrapper for QThread
 

python/helpers/pydev/_pydev_runfiles/pydev_runfiles_pytest2.py

@@ -1,39 +1,61 @@
-import pickle, zlib, base64, os
-import py
 from _pydev_runfiles import pydev_runfiles_xml_rpc
+import pickle
+import zlib
+import base64
+import os
+import py
 from pydevd_file_utils import _NormFile
 import pytest
 import sys
 import time
 
 
-#===================================================================================================
+#=========================================================================
 # Load filters with tests we should skip
-#===================================================================================================
+#=========================================================================
 py_test_accept_filter = None
 
+
 def _load_filters():
     global py_test_accept_filter
     if py_test_accept_filter is None:
         py_test_accept_filter = os.environ.get('PYDEV_PYTEST_SKIP')
         if py_test_accept_filter:
-            py_test_accept_filter = pickle.loads(zlib.decompress(base64.b64decode(py_test_accept_filter)))
+            py_test_accept_filter = pickle.loads(
+                zlib.decompress(base64.b64decode(py_test_accept_filter)))
         else:
             py_test_accept_filter = {}
 
 
-def connect_to_server_for_communication_to_xml_rpc_on_xdist():
+def is_in_xdist_node():
     main_pid = os.environ.get('PYDEV_MAIN_PID')
     if main_pid and main_pid != str(os.getpid()):
+        return True
+    return False
+
+
+connected = False
+def connect_to_server_for_communication_to_xml_rpc_on_xdist():
+    global connected
+    if connected:
+        return
+    connected = True
+    if is_in_xdist_node():
         port = os.environ.get('PYDEV_PYTEST_SERVER')
         if not port:
-            sys.stderr.write('Error: no PYDEV_PYTEST_SERVER environment variable defined.\n')
+            sys.stderr.write(
+                'Error: no PYDEV_PYTEST_SERVER environment variable defined.\n')
         else:
             pydev_runfiles_xml_rpc.initialize_server(int(port), daemon=True)
 
+
 PY2 = sys.version_info[0] <= 2
 PY3 = not PY2
 
+#=========================================================================
+# Mocking to get clickable file representations
+#=========================================================================
+
 _mock_code = []
 try:
     from py._code import code  # @UnresolvedImport
@@ -46,14 +68,10 @@ try:
 except ImportError:
     pass
 
-#===================================================================================================
-# Mocking to get clickable file representations
-#===================================================================================================
 def _MockFileRepresentation():
     for code in _mock_code:
         code.ReprFileLocation._original_toterminal = code.ReprFileLocation.toterminal
 
-
         def toterminal(self, tw):
             # filename and lineno output for each entry,
             # using an output format that most editors understand
@@ -65,12 +83,15 @@ def _MockFileRepresentation():
             path = os.path.abspath(self.path)
 
             if PY2:
-                if not isinstance(path, unicode):  # Note: it usually is NOT unicode...
+                # Note: it usually is NOT unicode...
+                if not isinstance(path, unicode):
                     path = path.decode(sys.getfilesystemencoding(), 'replace')
 
-                if not isinstance(msg, unicode):  # Note: it usually is unicode...
+                # Note: it usually is unicode...
+                if not isinstance(msg, unicode):
                     msg = msg.decode('utf-8', 'replace')
-                unicode_line = unicode('File "%s", line %s\n%s') % (path, self.lineno, msg)
+                unicode_line = unicode('File "%s", line %s\n%s') % (
+                    path, self.lineno, msg)
                 tw.line(unicode_line)
             else:
                 tw.line('File "%s", line %s\n%s' % (path, self.lineno, msg))
@@ -80,209 +101,238 @@ def _MockFileRepresentation():
 
 def _UninstallMockFileRepresentation():
     for code in _mock_code:
-        code.ReprFileLocation.toterminal = code.ReprFileLocation._original_toterminal #@UndefinedVariable
+        # @UndefinedVariable
+        code.ReprFileLocation.toterminal = code.ReprFileLocation._original_toterminal
 
+#=========================================================================
+# End mocking to get clickable file representations
+#=========================================================================
 
 class State:
-    numcollected = 0
     start_time = time.time()
+    buf_err = None
+    buf_out = None
 
 
-def pytest_configure(*args, **kwargs):
+def start_redirect():
+    if State.buf_out is not None:
+        return
+    from _pydevd_bundle import pydevd_io
+    State.buf_err = pydevd_io.start_redirect(keep_original_redirection=True, std='stderr')
+    State.buf_out = pydevd_io.start_redirect(keep_original_redirection=True, std='stdout')
+
+
+def get_curr_output():
+    return State.buf_out.getvalue(), State.buf_err.getvalue()
+
+
+def pytest_configure():
     _MockFileRepresentation()
 
 
-def pytest_collectreport(report):
-
-    i = 0
-    for x in report.result:
-        if isinstance(x, pytest.Item):
-            try:
-                # Call our setup (which may do a skip, in which
-                # case we won't count it).
-                pytest_runtest_setup(x)
-                i += 1
-            except:
-                continue
-    State.numcollected += i
-
-
-def pytest_collection_modifyitems():
-    connect_to_server_for_communication_to_xml_rpc_on_xdist()
-    pydev_runfiles_xml_rpc.notifyTestsCollected(State.numcollected)
-    State.numcollected = 0
-
-
-def pytest_unconfigure(*args, **kwargs):
+def pytest_unconfigure():
     _UninstallMockFileRepresentation()
-    pydev_runfiles_xml_rpc.notifyTestRunFinished('Finished in: %.2f secs.' % (time.time() - State.start_time,))
+    if is_in_xdist_node():
+        return
+    # Only report that it finished when on the main node (we don't want to report
+    # the finish on each separate node).
+    pydev_runfiles_xml_rpc.notifyTestRunFinished(
+        'Finished in: %.2f secs.' % (time.time() - State.start_time,))
 
 
-def pytest_runtest_setup(item):
-    filename = item.fspath.strpath
-    test = item.location[2]
-    State.start_test_time = time.time()
+def pytest_collection_modifyitems(session, config, items):
+    # A note: in xdist, this is not called on the main process, only in the
+    # secondary nodes, so, we'll actually make the filter and report it multiple
+    # times.
+    connect_to_server_for_communication_to_xml_rpc_on_xdist()
 
-    pydev_runfiles_xml_rpc.notifyStartTest(filename, test)
+    _load_filters()
+    if not py_test_accept_filter:
+        pydev_runfiles_xml_rpc.notifyTestsCollected(len(items))
+        return  # Keep on going (nothing to filter)
 
+    new_items = []
+    for item in items:
+        f = _NormFile(str(item.parent.fspath))
+        name = item.name
 
-def report_test(cond, filename, test, captured_output, error_contents, delta):
-    '''
-    @param filename: 'D:\\src\\mod1\\hello.py'
-    @param test: 'TestCase.testMet1'
-    @param cond: fail, error, ok
-    '''
-    time_str = '%.2f' % (delta,)
-    pydev_runfiles_xml_rpc.notifyTest(cond, captured_output, error_contents, filename, test, time_str)
+        if f not in py_test_accept_filter:
+            # print('Skip file: %s' % (f,))
+            continue  # Skip the file
 
+        accept_tests = py_test_accept_filter[f]
 
-def pytest_runtest_makereport(item, call):
-    report_when = call.when
-    report_duration = call.stop-call.start
-    excinfo = call.excinfo
-
-    if not call.excinfo:
-        evalxfail = getattr(item, '_evalxfail', None)
-        if evalxfail and report_when == 'call' and (not hasattr(evalxfail, 'expr') or evalxfail.expr):
-            # I.e.: a method marked with xfail passed... let the user know.
-            report_outcome = "failed"
-            report_longrepr = "XFAIL: Unexpected pass"
-
+        if item.cls is not None:
+            class_name = item.cls.__name__
         else:
-            report_outcome = "passed"
-            report_longrepr = None
-    else:
-        excinfo = call.excinfo
+            class_name = None
+        for test in accept_tests:
+            # This happens when parameterizing pytest tests.
+            i = name.find('[')
+            if i > 0:
+                name = name[:i]
+            if test == name:
+                # Direct match of the test (just go on with the default
+                # loading)
+                new_items.append(item)
+                break
 
-        handled = False
+            if class_name is not None:
+                if test == class_name + '.' + name:
+                    new_items.append(item)
+                    break
 
-        if not (call.excinfo and
-                    call.excinfo.errisinstance(pytest.xfail.Exception)):
-            evalxfail = getattr(item, '_evalxfail', None)
-            # Something which had an xfail failed: this is expected.
-            if evalxfail and (not hasattr(evalxfail, 'expr') or evalxfail.expr):
-                report_outcome = "passed"
-                report_longrepr = None
-                handled = True
-
-        if handled:
+                if class_name == test:
+                    new_items.append(item)
+                    break
+        else:
             pass
+            # print('Skip test: %s.%s. Accept: %s' % (class_name, name, accept_tests))
 
-        if excinfo.errisinstance(pytest.xfail.Exception):
-            # Case where an explicit xfail is raised (i.e.: pytest.xfail("reason") is called
-            # programatically).
-            report_outcome = "passed"
-            report_longrepr = None
+    # Modify the original list
+    items[:] = new_items
+    pydev_runfiles_xml_rpc.notifyTestsCollected(len(items))
 
-        elif excinfo.errisinstance(py.test.skip.Exception):  # @UndefinedVariable
-            report_outcome = "skipped"
-            r = excinfo._getreprcrash()
-            report_longrepr = None #(str(r.path), r.lineno, r.message)
 
-        elif not isinstance(excinfo, py.code.ExceptionInfo):  # @UndefinedVariable
-            report_outcome = "failed"
-            report_longrepr = excinfo
+from py.io import TerminalWriter
 
-        else:
-            report_outcome = "failed"
-            if call.when == "call":
-                report_longrepr = item.repr_failure(excinfo)
+def _get_error_contents_from_report(report):
+    if report.longrepr is not None:
+        tw = TerminalWriter(stringio=True)
+        tw.hasmarkup = False
+        report.toterminal(tw)
+        exc = tw.stringio.getvalue()
+        s = exc.strip()
+        if s:
+            return s
 
-            else: # exception in setup or teardown
-                report_longrepr = item._repr_failure_py(excinfo, style=item.config.option.tbstyle)
+    return ''
 
-    filename = item.fspath.strpath
-    test = item.location[2]
+def pytest_collectreport(report):
+    error_contents = _get_error_contents_from_report(report)
+    if error_contents:
+        report_test('fail', '<collect errors>', '<collect errors>', '', error_contents, 0.0)
 
-    status = 'ok'
-    captured_output = ''
-    error_contents = ''
+def append_strings(s1, s2):
+    if s1.__class__ == s2.__class__:
+        return s1 + s2
 
-    if report_outcome in ('passed', 'skipped'):
-        #passed or skipped: no need to report if in setup or teardown (only on the actual test if it passed).
+    if sys.version_info[0] == 2:
+        if not isinstance(s1, basestring):
+            s1 = str(s1)
+
+        if not isinstance(s2, basestring):
+            s2 = str(s2)
+
+        # Prefer bytes
+        if isinstance(s1, unicode):
+            s1 = s1.encode('utf-8')
+
+        if isinstance(s2, unicode):
+            s2 = s2.encode('utf-8')
+
+        return s1 + s2
+    else:
+        # Prefer str
+        if isinstance(s1, bytes):
+            s1 = s1.decode('utf-8', 'replace')
+
+        if isinstance(s2, bytes):
+            s2 = s2.decode('utf-8', 'replace')
+
+        return s1 + s2
+
+
+
+def pytest_runtest_logreport(report):
+    if is_in_xdist_node():
+        # When running with xdist, we don't want the report to be called from the node, only
+        # from the main process.
+        return
+    report_duration = report.duration
+    report_when = report.when
+    report_outcome = report.outcome
+
+    if hasattr(report, 'wasxfail'):
+        if report_outcome != 'skipped':
+            report_outcome = 'passed'
+
+    if report_outcome == 'passed':
+        # passed on setup/teardown: no need to report if in setup or teardown
+        # (only on the actual test if it passed).
         if report_when in ('setup', 'teardown'):
             return
 
-    else:
-        #It has only passed, skipped and failed (no error), so, let's consider error if not on call.
-        if report_when == 'setup':
-            if status == 'ok':
-                status = 'error'
+        status = 'ok'
 
-        elif report_when == 'teardown':
-            if status == 'ok':
-                status = 'error'
+    elif report_outcome == 'skipped':
+        status = 'skip'
+
+    else:
+        # It has only passed, skipped and failed (no error), so, let's consider
+        # error if not on call.
+        if report_when in ('setup', 'teardown'):
+            status = 'error'
 
         else:
-            #any error in the call (not in setup or teardown) is considered a regular failure.
+            # any error in the call (not in setup or teardown) is considered a
+            # regular failure.
             status = 'fail'
 
+    # This will work if pytest is not capturing it, if it is, nothing will
+    # come from here...
+    captured_output, error_contents = report.pydev_captured_output, report.pydev_error_contents
+    for type_section, value in report.sections:
+        if value:
+            if type_section in ('err', 'stderr', 'Captured stderr call'):
+                error_contents = append_strings(error_contents, value)
+            else:
+                captured_output = append_strings(error_contents, value)
 
-    if call.excinfo:
-        rep = report_longrepr
-        if hasattr(rep, 'reprcrash'):
-            reprcrash = rep.reprcrash
-            error_contents += str(reprcrash)
-            error_contents += '\n'
+    filename = report.pydev_fspath_strpath
+    test = report.location[2]
 
-        if hasattr(rep, 'reprtraceback'):
-            error_contents += str(rep.reprtraceback)
+    if report_outcome != 'skipped':
+        # On skipped, we'll have a traceback for the skip, which is not what we
+        # want.
+        exc = _get_error_contents_from_report(report)
+        if exc:
+            if error_contents:
+                error_contents = append_strings(error_contents, '----------------------------- Exceptions -----------------------------\n')
+            error_contents = append_strings(error_contents, exc)
 
-        if hasattr(rep, 'sections'):
-            for name, content, sep in rep.sections:
-                error_contents += sep * 40
-                error_contents += name
-                error_contents += sep * 40
-                error_contents += '\n'
-                error_contents += content
-                error_contents += '\n'
-    else:
-        if report_longrepr:
-            error_contents += str(report_longrepr)
+    report_test(status, filename, test, captured_output, error_contents, report_duration)
 
-    if status != 'skip': #I.e.: don't event report skips...
-        report_test(status, filename, test, captured_output, error_contents, report_duration)
 
+def report_test(status, filename, test, captured_output, error_contents, duration):
+    '''
+    @param filename: 'D:\\src\\mod1\\hello.py'
+    @param test: 'TestCase.testMet1'
+    @param status: fail, error, ok
+    '''
+    time_str = '%.2f' % (duration,)
+    pydev_runfiles_xml_rpc.notifyTest(
+        status, captured_output, error_contents, filename, test, time_str)
+
+
+@pytest.hookimpl(hookwrapper=True)
+def pytest_runtest_makereport(item, call):
+    outcome = yield
+    report = outcome.get_result()
+    report.pydev_fspath_strpath = item.fspath.strpath
+    report.pydev_captured_output, report.pydev_error_contents = get_curr_output()
 
 
 @pytest.mark.tryfirst
-def pytest_runtest_setup(item):  # @DuplicatedSignature
+def pytest_runtest_setup(item):
     '''
-    Skips tests. With xdist will be on a secondary process.
+    Note: with xdist will be on a secondary process.
     '''
-    _load_filters()
-    if not py_test_accept_filter:
-        return #Keep on going (nothing to filter)
-
-    f = _NormFile(str(item.parent.fspath))
-    name = item.name
-
-    if f not in py_test_accept_filter:
-        pytest.skip() # Skip the file
-
-    accept_tests = py_test_accept_filter[f]
-
-    if item.cls is not None:
-        class_name = item.cls.__name__
-    else:
-        class_name = None
-    for test in accept_tests:
-        # This happens when parameterizing pytest tests.
-        i = name.find('[')
-        if i > 0:
-            name = name[:i]
-        if test == name:
-            #Direct match of the test (just go on with the default loading)
-            return
-
-        if class_name is not None:
-            if test == class_name + '.' + name:
-                return
-
-            if class_name == test:
-                return
-
-    # If we had a match it'd have returned already.
-    pytest.skip() # Skip the test
-
+    # We have our own redirection: if xdist does its redirection, we'll have
+    # nothing in our contents (which is OK), but if it does, we'll get nothing
+    # from pytest but will get our own here.
+    start_redirect()
+    filename = item.fspath.strpath
+    test = item.location[2]
 
+    pydev_runfiles_xml_rpc.notifyStartTest(filename, test)

python/helpers/pydev/_pydev_runfiles/pydev_runfiles_unittest.py

@@ -26,6 +26,17 @@ _PythonTextTestResult = python_unittest.TextTestRunner()._makeResult().__class__
 #=======================================================================================================================
 class PydevTestResult(_PythonTextTestResult):
 
+    def addSubTest(self, test, subtest, err):
+        """Called at the end of a subtest.
+        'err' is None if the subtest ended successfully, otherwise it's a
+        tuple of values as returned by sys.exc_info().
+        """
+        _PythonTextTestResult.addSubTest(self, test, subtest, err)
+        if err is not None:
+            subdesc = subtest._subDescription()
+            error = (test, self._exc_info_to_string(err, test))
+            self._reportErrors([error], [], '', '%s %s' % (self.get_test_name(test), subdesc))
+
 
     def startTest(self, test):
         _PythonTextTestResult.startTest(self, test)

python/helpers/pydev/_pydevd_bundle/pydevd_additional_thread_info_regular.py

@@ -1,8 +1,9 @@
 import sys
-import weakref
-from _pydev_imps._pydev_saved_modules import thread
-from _pydevd_bundle.pydevd_constants import STATE_RUN, PYTHON_SUSPEND, dict_iter_items
+from _pydevd_bundle.pydevd_constants import STATE_RUN, PYTHON_SUSPEND
+# IFDEF CYTHON
+# ELSE
 from _pydevd_bundle.pydevd_frame import PyDBFrame
+# ENDIF
 
 
 #=======================================================================================================================
@@ -84,110 +85,3 @@ class PyDBAdditionalThreadInfo(object):
     def __str__(self):
         return 'State:%s Stop:%s Cmd: %s Kill:%s' % (
             self.pydev_state, self.pydev_step_stop, self.pydev_step_cmd, self.pydev_notify_kill)
-
-#=======================================================================================================================
-# Note that the Cython version has only the contents above
-#=======================================================================================================================
-
-# IFDEF CYTHON
-# ELSE
-
-PyDBAdditionalThreadInfoOriginal = PyDBAdditionalThreadInfo
-#=======================================================================================================================
-# PyDBAdditionalThreadInfoWithoutCurrentFramesSupport
-#=======================================================================================================================
-class PyDBAdditionalThreadInfoWithoutCurrentFramesSupport(PyDBAdditionalThreadInfoOriginal):
-
-    def __init__(self):
-        PyDBAdditionalThreadInfoOriginal.__init__(self)
-        #That's where the last frame entered is kept. That's needed so that we're able to
-        #trace contexts that were previously untraced and are currently active. So, the bad thing
-        #is that the frame may be kept alive longer than it would if we go up on the frame stack,
-        #and is only disposed when some other frame is removed.
-        #A better way would be if we could get the topmost frame for each thread, but that's
-        #not possible (until python 2.5 -- which is the PyDBAdditionalThreadInfo version)
-        #Or if the user compiled threadframe (from http://www.majid.info/mylos/stories/2004/06/10/threadframe.html)
-
-        #NOT RLock!! (could deadlock if it was)
-        self.lock = thread.allocate_lock()
-        self._acquire_lock = self.lock.acquire
-        self._release_lock = self.lock.release
-
-        #collection with the refs
-        d = {}
-        self.pydev_existing_frames = d
-        try:
-            self._iter_frames = d.iterkeys
-        except AttributeError:
-            self._iter_frames = d.keys
-
-
-    def _OnDbFrameCollected(self, ref):
-        '''
-            Callback to be called when a given reference is garbage-collected.
-        '''
-        self._acquire_lock()
-        try:
-            del self.pydev_existing_frames[ref]
-        finally:
-            self._release_lock()
-
-
-    def _AddDbFrame(self, db_frame):
-        self._acquire_lock()
-        try:
-            #create the db frame with a callback to remove it from the dict when it's garbage-collected
-            #(could be a set, but that's not available on all versions we want to target).
-            r = weakref.ref(db_frame, self._OnDbFrameCollected)
-            self.pydev_existing_frames[r] = r
-        finally:
-            self._release_lock()
-
-
-    def create_db_frame(self, args):
-        #the frame must be cached as a weak-ref (we return the actual db frame -- which will be kept
-        #alive until its trace_dispatch method is not referenced anymore).
-        #that's a large workaround because:
-        #1. we can't have weak-references to python frame object
-        #2. only from 2.5 onwards we have _current_frames support from the interpreter
-        db_frame = PyDBFrame(args)
-        db_frame.frame = args[-1]
-        self._AddDbFrame(db_frame)
-        return db_frame
-
-
-    def iter_frames(self, t):
-        #We cannot use yield (because of the lock)
-        self._acquire_lock()
-        try:
-            ret = []
-
-            for weak_db_frame in self._iter_frames():
-                try:
-                    ret.append(weak_db_frame().frame)
-                except AttributeError:
-                    pass  # ok, garbage-collected already
-            return ret
-        finally:
-            self._release_lock()
-
-    def __str__(self):
-        return 'State:%s Stop:%s Cmd: %s Kill:%s Frames:%s' % (
-            self.pydev_state, self.pydev_step_stop, self.pydev_step_cmd, self.pydev_notify_kill, len(self.iter_frames(None)))
-
-#=======================================================================================================================
-# NOW, WE HAVE TO DEFINE WHICH THREAD INFO TO USE
-# (whether we have to keep references to the frames or not)
-# from version 2.5 onwards, we can use sys._current_frames to get a dict with the threads
-# and frames, but to support other versions, we can't rely on that.
-#=======================================================================================================================
-if not hasattr(sys, '_current_frames'):
-    try:
-        import threadframe  #@UnresolvedImport
-        sys._current_frames = threadframe.dict
-        assert sys._current_frames is threadframe.dict  #Just check if it was correctly set
-    except:
-        #If all fails, let's use the support without frames
-        PyDBAdditionalThreadInfo = PyDBAdditionalThreadInfoWithoutCurrentFramesSupport
-
-# ENDIF

python/helpers/pydev/_pydevd_bundle/pydevd_command_line_handling.py

@@ -0,0 +1,145 @@
+class ArgHandlerWithParam:
+    '''
+    Handler for some arguments which needs a value
+    '''
+
+    def __init__(self, arg_name, convert_val=None, default_val=None):
+        self.arg_name = arg_name
+        self.arg_v_rep = '--%s' % (arg_name,)
+        self.convert_val = convert_val
+        self.default_val = default_val
+
+    def to_argv(self, lst, setup):
+        v = setup.get(self.arg_name)
+        if v is not None and v != self.default_val:
+            lst.append(self.arg_v_rep)
+            lst.append('%s' % (v,))
+
+    def handle_argv(self, argv, i, setup):
+        assert argv[i] == self.arg_v_rep
+        del argv[i]
+
+        val = argv[i]
+        if self.convert_val:
+            val = self.convert_val(val)
+
+        setup[self.arg_name] = val
+        del argv[i]
+
+class ArgHandlerBool:
+    '''
+    If a given flag is received, mark it as 'True' in setup.
+    '''
+
+    def __init__(self, arg_name, default_val=False):
+        self.arg_name = arg_name
+        self.arg_v_rep = '--%s' % (arg_name,)
+        self.default_val = default_val
+
+    def to_argv(self, lst, setup):
+        v = setup.get(self.arg_name)
+        if v:
+            lst.append(self.arg_v_rep)
+
+    def handle_argv(self, argv, i, setup):
+        assert argv[i] == self.arg_v_rep
+        del argv[i]
+        setup[self.arg_name] = True
+
+
+ACCEPTED_ARG_HANDLERS = [
+    ArgHandlerWithParam('port', int, 0),
+    ArgHandlerWithParam('vm_type'),
+    ArgHandlerWithParam('client'),
+
+    ArgHandlerBool('server'),
+    ArgHandlerBool('DEBUG_RECORD_SOCKET_READS'),
+    ArgHandlerBool('multiproc'), # Used by PyCharm (reuses connection: ssh tunneling)
+    ArgHandlerBool('multiprocess'), # Used by PyDev (creates new connection to ide)
+    ArgHandlerBool('save-signatures'),
+    ArgHandlerBool('save-threading'),
+    ArgHandlerBool('save-asyncio'),
+    ArgHandlerBool('print-in-debugger-startup'),
+    ArgHandlerBool('cmd-line'),
+    ArgHandlerBool('module'),
+]
+
+ARGV_REP_TO_HANDLER = {}
+for handler in ACCEPTED_ARG_HANDLERS:
+    ARGV_REP_TO_HANDLER[handler.arg_v_rep] = handler
+
+def get_pydevd_file():
+    import pydevd
+    f = pydevd.__file__
+    if f.endswith('.pyc'):
+        f = f[:-1]
+    return f
+
+def setup_to_argv(setup):
+    '''
+    :param dict setup:
+        A dict previously gotten from process_command_line.
+        
+    :note: does not handle --file nor --DEBUG.
+    '''
+    ret = [get_pydevd_file()]
+
+    for handler in ACCEPTED_ARG_HANDLERS:
+        if handler.arg_name in setup:
+            handler.to_argv(ret, setup)
+    return ret
+
+def process_command_line(argv):
+    """ parses the arguments.
+        removes our arguments from the command line """
+    setup = {}
+    for handler in ACCEPTED_ARG_HANDLERS:
+        setup[handler.arg_name] = handler.default_val
+    setup['file'] = ''
+    setup['qt-support'] = ''
+
+    i = 0
+    del argv[0]
+    while i < len(argv):
+        handler = ARGV_REP_TO_HANDLER.get(argv[i])
+        if handler is not None:
+            handler.handle_argv(argv, i, setup)
+
+        elif argv[i].startswith('--qt-support'):
+            # The --qt-support is special because we want to keep backward compatibility: 
+            # Previously, just passing '--qt-support' meant that we should use the auto-discovery mode
+            # whereas now, if --qt-support is passed, it should be passed as --qt-support=<mode>, where
+            # mode can be one of 'auto', 'none', 'pyqt5', 'pyqt4', 'pyside'.
+            if argv[i] == '--qt-support':
+                setup['qt-support'] = 'auto'
+
+            elif argv[i].startswith('--qt-support='):
+                qt_support = argv[i][len('--qt-support='):]
+                valid_modes = ('none', 'auto', 'pyqt5', 'pyqt4', 'pyside')
+                if qt_support not in valid_modes:
+                    raise ValueError("qt-support mode invalid: " + qt_support)
+                if qt_support == 'none':
+                    # On none, actually set an empty string to evaluate to False.
+                    setup['qt-support'] = ''
+                else:
+                    setup['qt-support'] = qt_support
+            else:
+                raise ValueError("Unexpected definition for qt-support flag: " + argv[i])
+
+            del argv[i]
+
+
+        elif argv[i] == '--file':
+            # --file is special because it's the last one (so, no handler for it).
+            del argv[i]
+            setup['file'] = argv[i]
+            i = len(argv) # pop out, file is our last argument
+
+        elif argv[i] == '--DEBUG':
+            from pydevd import set_debug
+            del argv[i]
+            set_debug(setup)
+
+        else:
+            raise ValueError("Unexpected option: " + argv[i])
+    return setup

python/helpers/pydev/_pydevd_bundle/pydevd_constants.py

@@ -157,20 +157,6 @@ except:
         except NameError:
             def dict_contains(d, key):
                 return d.has_key(key)
-try:
-    dict_pop = dict.pop
-except:
-    #=======================================================================================================================
-    # Jython 2.1
-    #=======================================================================================================================
-    def dict_pop(d, key, default=None):
-        try:
-            ret = d[key]
-            del d[key]
-            return ret
-        except:
-            return default
-
 
 if IS_PY3K:
     def dict_keys(d):

python/helpers/pydev/_pydevd_bundle/pydevd_custom_frames.py

@@ -1,4 +1,4 @@
-from _pydevd_bundle.pydevd_constants import dict_pop, get_thread_id, Null
+from _pydevd_bundle.pydevd_constants import get_thread_id, Null
 from pydevd_file_utils import get_abs_path_real_path_and_base_from_frame
 from _pydev_imps._pydev_saved_modules import thread, threading
 import sys
@@ -125,7 +125,7 @@ def remove_custom_frame(frame_id):
     try:
         if DEBUG:
             sys.stderr.write('remove_custom_frame: %s\n' % frame_id)
-        dict_pop(CustomFramesContainer.custom_frames, frame_id, None)
+        CustomFramesContainer.custom_frames.pop(frame_id, None)
         CustomFramesContainer._py_db_command_thread_event.set()
     finally:
         CustomFramesContainer.custom_frames_lock.release()

python/helpers/pydev/_pydevd_bundle/pydevd_cython.pyx

@@ -3,10 +3,11 @@
 # DO NOT edit manually!
 # DO NOT edit manually!
 import sys
-import weakref
-from _pydev_imps._pydev_saved_modules import thread
-from _pydevd_bundle.pydevd_constants import STATE_RUN, PYTHON_SUSPEND, dict_iter_items
-from _pydevd_bundle.pydevd_frame import PyDBFrame
+from _pydevd_bundle.pydevd_constants import STATE_RUN, PYTHON_SUSPEND
+# IFDEF CYTHON -- DONT EDIT THIS FILE (it is automatically generated)
+# ELSE
+# from _pydevd_bundle.pydevd_frame import PyDBFrame
+# ENDIF
 
 
 #=======================================================================================================================
@@ -88,113 +89,6 @@ cdef class PyDBAdditionalThreadInfo:
     def __str__(self):
         return 'State:%s Stop:%s Cmd: %s Kill:%s' % (
             self.pydev_state, self.pydev_step_stop, self.pydev_step_cmd, self.pydev_notify_kill)
-
-#=======================================================================================================================
-# Note that the Cython version has only the contents above
-#=======================================================================================================================
-
-# IFDEF CYTHON -- DONT EDIT THIS FILE (it is automatically generated)
-# ELSE
-# 
-# PyDBAdditionalThreadInfoOriginal = PyDBAdditionalThreadInfo
-# #=======================================================================================================================
-# # PyDBAdditionalThreadInfoWithoutCurrentFramesSupport
-# #=======================================================================================================================
-# class PyDBAdditionalThreadInfoWithoutCurrentFramesSupport(PyDBAdditionalThreadInfoOriginal):
-# 
-#     def __init__(self):
-#         PyDBAdditionalThreadInfoOriginal.__init__(self)
-#         #That's where the last frame entered is kept. That's needed so that we're able to
-#         #trace contexts that were previously untraced and are currently active. So, the bad thing
-#         #is that the frame may be kept alive longer than it would if we go up on the frame stack,
-#         #and is only disposed when some other frame is removed.
-#         #A better way would be if we could get the topmost frame for each thread, but that's
-#         #not possible (until python 2.5 -- which is the PyDBAdditionalThreadInfo version)
-#         #Or if the user compiled threadframe (from http://www.majid.info/mylos/stories/2004/06/10/threadframe.html)
-# 
-#         #NOT RLock!! (could deadlock if it was)
-#         self.lock = thread.allocate_lock()
-#         self._acquire_lock = self.lock.acquire
-#         self._release_lock = self.lock.release
-# 
-#         #collection with the refs
-#         d = {}
-#         self.pydev_existing_frames = d
-#         try:
-#             self._iter_frames = d.iterkeys
-#         except AttributeError:
-#             self._iter_frames = d.keys
-# 
-# 
-#     def _OnDbFrameCollected(self, ref):
-#         '''
-#             Callback to be called when a given reference is garbage-collected.
-#         '''
-#         self._acquire_lock()
-#         try:
-#             del self.pydev_existing_frames[ref]
-#         finally:
-#             self._release_lock()
-# 
-# 
-#     def _AddDbFrame(self, db_frame):
-#         self._acquire_lock()
-#         try:
-#             #create the db frame with a callback to remove it from the dict when it's garbage-collected
-#             #(could be a set, but that's not available on all versions we want to target).
-#             r = weakref.ref(db_frame, self._OnDbFrameCollected)
-#             self.pydev_existing_frames[r] = r
-#         finally:
-#             self._release_lock()
-# 
-# 
-#     def create_db_frame(self, args):
-#         #the frame must be cached as a weak-ref (we return the actual db frame -- which will be kept
-#         #alive until its trace_dispatch method is not referenced anymore).
-#         #that's a large workaround because:
-#         #1. we can't have weak-references to python frame object
-#         #2. only from 2.5 onwards we have _current_frames support from the interpreter
-#         db_frame = PyDBFrame(args)
-#         db_frame.frame = args[-1]
-#         self._AddDbFrame(db_frame)
-#         return db_frame
-# 
-# 
-#     def iter_frames(self, t):
-#         #We cannot use yield (because of the lock)
-#         self._acquire_lock()
-#         try:
-#             ret = []
-# 
-#             for weak_db_frame in self._iter_frames():
-#                 try:
-#                     ret.append(weak_db_frame().frame)
-#                 except AttributeError:
-#                     pass  # ok, garbage-collected already
-#             return ret
-#         finally:
-#             self._release_lock()
-# 
-#     def __str__(self):
-#         return 'State:%s Stop:%s Cmd: %s Kill:%s Frames:%s' % (
-#             self.pydev_state, self.pydev_step_stop, self.pydev_step_cmd, self.pydev_notify_kill, len(self.iter_frames(None)))
-# 
-# #=======================================================================================================================
-# # NOW, WE HAVE TO DEFINE WHICH THREAD INFO TO USE
-# # (whether we have to keep references to the frames or not)
-# # from version 2.5 onwards, we can use sys._current_frames to get a dict with the threads
-# # and frames, but to support other versions, we can't rely on that.
-# #=======================================================================================================================
-# if not hasattr(sys, '_current_frames'):
-#     try:
-#         import threadframe  #@UnresolvedImport
-#         sys._current_frames = threadframe.dict
-#         assert sys._current_frames is threadframe.dict  #Just check if it was correctly set
-#     except:
-#         #If all fails, let's use the support without frames
-#         PyDBAdditionalThreadInfo = PyDBAdditionalThreadInfoWithoutCurrentFramesSupport
-# 
-# ENDIF
 import linecache
 import os.path
 import re
@@ -208,7 +102,7 @@ from _pydevd_bundle.pydevd_breakpoints import get_exception_breakpoint
 from _pydevd_bundle.pydevd_comm import CMD_STEP_CAUGHT_EXCEPTION, CMD_STEP_RETURN, CMD_STEP_OVER, CMD_SET_BREAK, \
     CMD_STEP_INTO, CMD_SMART_STEP_INTO, CMD_RUN_TO_LINE, CMD_SET_NEXT_STATEMENT, CMD_STEP_INTO_MY_CODE
 from _pydevd_bundle.pydevd_constants import STATE_SUSPEND, dict_contains, get_thread_id, STATE_RUN, dict_iter_values, IS_PY3K, \
-    dict_keys, dict_pop, RETURN_VALUES_DICT
+    dict_keys, RETURN_VALUES_DICT
 from _pydevd_bundle.pydevd_dont_trace_files import DONT_TRACE, PYDEV_FILE
 from _pydevd_bundle.pydevd_frame_utils import add_exception_to_frame, just_raised
 from _pydevd_bundle.pydevd_utils import get_clsname_for_code
@@ -237,11 +131,16 @@ get_file_type = DONT_TRACE.get
 #=======================================================================================================================
 # PyDBFrame
 #=======================================================================================================================
-class PyDBFrame: # No longer cdef because object was dying when only a reference to trace_dispatch was kept (need to check alternatives).
-    '''This makes the tracing for a given frame, so, the trace_dispatch
-    is used initially when we enter into a new context ('call') and then
-    is reused for the entire context.
-    '''
+# IFDEF CYTHON -- DONT EDIT THIS FILE (it is automatically generated)
+cdef class PyDBFrame:
+# ELSE
+# class PyDBFrame:
+#     '''This makes the tracing for a given frame, so, the trace_dispatch
+#     is used initially when we enter into a new context ('call') and then
+#     is reused for the entire context.
+#     '''
+    # ENDIF
+
 
     #Note: class (and not instance) attributes.
 
@@ -249,16 +148,20 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
     #considers the user input (so, the actual result must be a join of both).
     filename_to_lines_where_exceptions_are_ignored = {}
     filename_to_stat_info = {}
-    should_skip = -1
 
     # IFDEF CYTHON -- DONT EDIT THIS FILE (it is automatically generated)
-    def __init__(self, args):
+    cdef tuple _args
+    cdef int should_skip
+    def __init__(self, tuple args):
         self._args = args # In the cython version we don't need to pass the frame
+        self.should_skip = -1  # On cythonized version, put in instance.
     # ELSE
+#     should_skip = -1  # Default value in class (put in instance on set).
+# 
 #     def __init__(self, args):
 #         #args = main_debugger, filename, base, info, t, frame
 #         #yeap, much faster than putting in self and then getting it from self later on
-#         self._args = args[:-1] # Remove the frame (we don't want to have a reference to it).
+#         self._args = args
     # ENDIF
 
     def set_suspend(self, *args, **kwargs):
@@ -301,7 +204,8 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
         info = self._args[2]
         flag = False
 
-        if info.pydev_state != STATE_SUSPEND:  #and breakpoint is not None:
+        # STATE_SUSPEND = 2
+        if info.pydev_state != 2:  #and breakpoint is not None:
             exception, value, trace = arg
 
             if trace is not None: #on jython trace is None on the first event
@@ -336,7 +240,7 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
                         if main_debugger.plugin is not None:
                             result = main_debugger.plugin.exception_break(main_debugger, self, frame, self._args, arg)
                             if result:
-                                (flag, frame) = result
+                                flag, frame = result
                     except:
                         flag = False
 
@@ -463,6 +367,19 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
             main_debugger = None
             thread = None
 
+    def get_func_name(self, frame):
+        code_obj = frame.f_code
+        func_name = code_obj.co_name
+        try:
+            cls_name = get_clsname_for_code(code_obj, frame)
+            if cls_name is not None:
+                return "%s.%s" % (cls_name, func_name)
+            else:
+                return func_name
+        except:
+            traceback.print_exc()
+            return func_name
+
     def manage_return_values(self, main_debugger, frame, event, arg):
 
         def get_func_name(frame):
@@ -490,17 +407,17 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
                 # Showing return values was turned off, we should remove them from locals dict.
                 # The values can be in the current frame or in the back one
                 if RETURN_VALUES_DICT in dict_keys(frame.f_locals):
-                    dict_pop(frame.f_locals, RETURN_VALUES_DICT)
+                    frame.f_locals.pop(RETURN_VALUES_DICT)
                 if hasattr(frame, "f_back") and hasattr(frame.f_back, "f_locals"):
                     if RETURN_VALUES_DICT in dict_keys(frame.f_back.f_locals):
-                        dict_pop(frame.f_back.f_locals, RETURN_VALUES_DICT)
+                        frame.f_back.f_locals.pop(RETURN_VALUES_DICT)
                 main_debugger.remove_return_values_flag = False
         except:
             main_debugger.remove_return_values_flag = False
             traceback.print_exc()
 
     # IFDEF CYTHON -- DONT EDIT THIS FILE (it is automatically generated)
-    def trace_dispatch(self, frame, str event, arg):
+    cpdef trace_dispatch(self, frame, str event, arg):
         cdef str filename;
         cdef bint is_exception_event;
         cdef bint has_exception_breakpoints;
@@ -508,25 +425,30 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
         cdef PyDBAdditionalThreadInfo info;
         cdef int step_cmd;
         cdef int line;
+        cdef bint is_line;
+        cdef bint is_call;
+        cdef bint is_return;
         cdef str curr_func_name;
         cdef bint exist_result;
+        cdef dict frame_skips_cache;
+        cdef tuple frame_cache_key;
+        cdef tuple line_cache_key;
+        cdef int breakpoints_in_line_cache;
+        cdef int breakpoints_in_frame_cache;
+        cdef bint has_breakpoint_in_frame;
     # ELSE
 #     def trace_dispatch(self, frame, event, arg):
     # ENDIF
 
-        main_debugger, filename, info, thread = self._args
+        main_debugger, filename, info, thread, frame_skips_cache, frame_cache_key = self._args
+        # print('frame trace_dispatch', frame.f_lineno, frame.f_code.co_name, event, info.pydev_step_cmd)
         try:
-            # print 'frame trace_dispatch', frame.f_lineno, frame.f_code.co_name, event
             info.is_tracing = True
-            need_trace_return = False
+            line = frame.f_lineno
+            line_cache_key = (frame_cache_key, line)
 
             if main_debugger._finish_debugging_session:
-                return None
-
-            if event == 'call' and main_debugger.signature_factory:
-                need_trace_return = send_signature_call_trace(main_debugger, frame, filename)
-            if event == 'return' and main_debugger.signature_factory:
-                send_signature_return_trace(main_debugger, frame, filename, arg)
+                    return None
 
             plugin_manager = main_debugger.plugin
 
@@ -539,19 +461,32 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
                     if flag:
                         self.handle_exception(frame, event, arg)
                         return self.trace_dispatch
+                is_line = False
+                is_return = False
+                is_call = False
+            else:
+                is_line = event == 'line'
+                is_return = event == 'return'
+                is_call = event == 'call'
+                if not is_line and not is_return and not is_call:
+                    # I believe this can only happen in jython on some frontiers on jython and java code, which we don't want to trace.
+                    return None
 
-            elif event not in ('line', 'call', 'return'):
-                #I believe this can only happen in jython on some frontiers on jython and java code, which we don't want to trace.
-                return None
+            need_trace_return = False
+            if is_call and main_debugger.signature_factory:
+                need_trace_return = send_signature_call_trace(main_debugger, frame, filename)
+            if is_return and main_debugger.signature_factory:
+                send_signature_return_trace(main_debugger, frame, filename, arg)
 
             stop_frame = info.pydev_step_stop
             step_cmd = info.pydev_step_cmd
 
             if is_exception_event:
                 breakpoints_for_file = None
-                if stop_frame and stop_frame is not frame and step_cmd is CMD_STEP_OVER and \
+                # CMD_STEP_OVER = 108
+                if stop_frame and stop_frame is not frame and step_cmd == 108 and \
                                 arg[0] in (StopIteration, GeneratorExit) and arg[2] is None:
-                    info.pydev_step_cmd = CMD_STEP_INTO
+                    info.pydev_step_cmd = 107  # CMD_STEP_INTO = 107
                     info.pydev_step_stop = None
             else:
                 # If we are in single step mode and something causes us to exit the current frame, we need to make sure we break
@@ -560,29 +495,31 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
                 # to make a step in or step over at that location).
                 # Note: this is especially troublesome when we're skipping code with the
                 # @DontTrace comment.
-                if stop_frame is frame and event is 'return' and step_cmd in (CMD_STEP_RETURN, CMD_STEP_OVER):
-                    if not frame.f_code.co_flags & CO_GENERATOR:
-                        info.pydev_step_cmd = CMD_STEP_INTO
+                if stop_frame is frame and is_return and step_cmd in (109, 108):  # CMD_STEP_RETURN = 109, CMD_STEP_OVER = 108
+                    if not frame.f_code.co_flags & 0x20:  # CO_GENERATOR = 0x20 (inspect.CO_GENERATOR)
+                        info.pydev_step_cmd = 107  # CMD_STEP_INTO = 107
                         info.pydev_step_stop = None
 
                 breakpoints_for_file = main_debugger.breakpoints.get(filename)
 
                 can_skip = False
 
-                if info.pydev_state == STATE_RUN:
+                if info.pydev_state == 1:  # STATE_RUN = 1
                     #we can skip if:
                     #- we have no stop marked
                     #- we should make a step return/step over and we're not in the current frame
-                    can_skip = (step_cmd == -1 and stop_frame is None)\
-                        or (step_cmd in (CMD_STEP_RETURN, CMD_STEP_OVER) and stop_frame is not frame)
+                    # CMD_STEP_RETURN = 109, CMD_STEP_OVER = 108
+                    can_skip = (step_cmd == -1 and stop_frame is None) \
+                               or (step_cmd in (109, 108) and stop_frame is not frame)
 
-                if can_skip and plugin_manager is not None and main_debugger.has_plugin_line_breaks:
-                    can_skip = not plugin_manager.can_not_skip(main_debugger, self, frame)
+                    if can_skip:
+                        if plugin_manager is not None and main_debugger.has_plugin_line_breaks:
+                            can_skip = not plugin_manager.can_not_skip(main_debugger, self, frame)
 
-                if can_skip and main_debugger.show_return_values:
-                    # trace function for showing return values after step over
-                    if info.pydev_step_cmd == CMD_STEP_OVER and hasattr(frame, "f_back") and frame.f_back == info.pydev_step_stop:
-                        can_skip = False
+                        # CMD_STEP_OVER = 108
+                        if can_skip and is_return and main_debugger.show_return_values and info.pydev_step_cmd == 108 and frame.f_back is info.pydev_step_stop:
+                            # trace function for showing return values after step over
+                            can_skip = False
 
                 if main_debugger.frame_eval_func and event == 'return' and info.pydev_step_cmd == -1:
                     frames_set = main_debugger.disable_tracing_after_exit_frames.get(get_thread_id(thread), None)
@@ -596,8 +533,8 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
 
                 # Let's check to see if we are in a function that has a breakpoint. If we don't have a breakpoint,
                 # we will return nothing for the next trace
-                #also, after we hit a breakpoint and go to some other debugging state, we have to force the set trace anyway,
-                #so, that's why the additional checks are there.
+                # also, after we hit a breakpoint and go to some other debugging state, we have to force the set trace anyway,
+                # so, that's why the additional checks are there.
                 if not breakpoints_for_file:
                     if can_skip:
                         if has_exception_breakpoints:
@@ -609,34 +546,51 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
                                 return None
 
                 else:
-                    #checks the breakpoint to see if there is a context match in some function
-                    curr_func_name = frame.f_code.co_name
+                    # When cached, 0 means we don't have a breakpoint and 1 means we have.
+                    if can_skip:
+                        breakpoints_in_line_cache = frame_skips_cache.get(line_cache_key, -1)
+                        if breakpoints_in_line_cache == 0:
+                            return self.trace_dispatch
 
-                    #global context is set with an empty name
-                    if curr_func_name in ('?', '<module>'):
-                        curr_func_name = ''
+                    breakpoints_in_frame_cache = frame_skips_cache.get(frame_cache_key, -1)
+                    if breakpoints_in_frame_cache != -1:
+                        # Gotten from cache.
+                        has_breakpoint_in_frame = breakpoints_in_frame_cache == 1
 
-                    for breakpoint in dict_iter_values(breakpoints_for_file): #jython does not support itervalues()
-                        #will match either global or some function
-                        if breakpoint.func_name in ('None', curr_func_name):
-                            break
+                    else:
+                        has_breakpoint_in_frame = False
+                        # Checks the breakpoint to see if there is a context match in some function
+                        curr_func_name = frame.f_code.co_name
 
-                    else: # if we had some break, it won't get here (so, that's a context that we want to skip)
-                        if can_skip:
-                            if has_exception_breakpoints:
-                                return self.trace_exception
+                        #global context is set with an empty name
+                        if curr_func_name in ('?', '<module>'):
+                            curr_func_name = ''
+
+                        for breakpoint in dict_iter_values(breakpoints_for_file): #jython does not support itervalues()
+                            #will match either global or some function
+                            if breakpoint.func_name in ('None', curr_func_name):
+                                has_breakpoint_in_frame = True
+                                break
+
+                        # Cache the value (1 or 0 or -1 for default because of cython).
+                        if has_breakpoint_in_frame:
+                            frame_skips_cache[frame_cache_key] = 1
+                        else:
+                            frame_skips_cache[frame_cache_key] = 0
+
+                    if can_skip and not has_breakpoint_in_frame:
+                        if has_exception_breakpoints:
+                            return self.trace_exception
+                        else:
+                            if need_trace_return:
+                                return self.trace_return
                             else:
-                                if need_trace_return:
-                                    return self.trace_return
-                                else:
-                                    return None
-
+                                return None
 
             #We may have hit a breakpoint or we are already in step mode. Either way, let's check what we should do in this frame
-            #print 'NOT skipped', frame.f_lineno, frame.f_code.co_name, event
+            #print('NOT skipped', frame.f_lineno, frame.f_code.co_name, event)
 
             try:
-                line = frame.f_lineno
                 flag = False
                 #return is not taken into account for breakpoint hit because we'd have a double-hit in this case
                 #(one for the line and the other for the return).
@@ -646,18 +600,17 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
                 exist_result = False
                 stop = False
                 bp_type = None
-                if not flag and event != 'return' and info.pydev_state != STATE_SUSPEND and breakpoints_for_file is not None \
-                        and dict_contains(breakpoints_for_file, line):
+                if not is_return and info.pydev_state != STATE_SUSPEND and breakpoints_for_file is not None and dict_contains(breakpoints_for_file, line):
                     breakpoint = breakpoints_for_file[line]
                     new_frame = frame
                     stop = True
-                    if step_cmd == CMD_STEP_OVER and stop_frame is frame and event in ('line', 'return'):
+                    if step_cmd == CMD_STEP_OVER and stop_frame is frame and (is_line or is_return):
                         stop = False #we don't stop on breakpoint if we have to stop by step-over (it will be processed later)
                 elif plugin_manager is not None and main_debugger.has_plugin_line_breaks:
                     result = plugin_manager.get_breakpoint(main_debugger, self, frame, event, self._args)
                     if result:
                         exist_result = True
-                        (flag, breakpoint, new_frame, bp_type) = result
+                        flag, breakpoint, new_frame, bp_type = result
 
                 if breakpoint:
                     #ok, hit breakpoint, now, we have to discover if it is a conditional breakpoint
@@ -709,17 +662,16 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
                                     info.pydev_message = str(val)
 
                         if not main_debugger.first_breakpoint_reached:
-                            if event == 'call':
-                                if hasattr(frame, 'f_back'):
-                                    back = frame.f_back
-                                    if back is not None:
-                                        # When we start debug session, we call execfile in pydevd run function. It produces an additional
-                                        # 'call' event for tracing and we stop on the first line of code twice.
-                                        _, back_filename, base = get_abs_path_real_path_and_base_from_frame(back)
-                                        if (base == DEBUG_START[0] and back.f_code.co_name == DEBUG_START[1]) or \
-                                                (base == DEBUG_START_PY3K[0] and back.f_code.co_name == DEBUG_START_PY3K[1]):
-                                            stop = False
-                                            main_debugger.first_breakpoint_reached = True
+                            if is_call:
+                                back = frame.f_back
+                                if back is not None:
+                                    # When we start debug session, we call execfile in pydevd run function. It produces an additional
+                                    # 'call' event for tracing and we stop on the first line of code twice.
+                                    _, back_filename, base = get_abs_path_real_path_and_base_from_frame(back)
+                                    if (base == DEBUG_START[0] and back.f_code.co_name == DEBUG_START[1]) or \
+                                            (base == DEBUG_START_PY3K[0] and back.f_code.co_name == DEBUG_START_PY3K[1]):
+                                        stop = False
+                                        main_debugger.first_breakpoint_reached = True
                 else:
                     # if the frame is traced after breakpoint stop,
                     # but the file should be ignored while stepping because of filters
@@ -747,6 +699,10 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
                 if info.pydev_state == STATE_SUSPEND:
                     self.do_wait_suspend(thread, frame, event, arg)
                     return self.trace_dispatch
+                else:
+                    if not breakpoint and not is_return:
+                        # No stop from anyone and no breakpoint found in line (cache that).
+                        frame_skips_cache[line_cache_key] = 0
 
             except:
                 traceback.print_exc()
@@ -773,7 +729,7 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
                     stop = False
 
                 elif step_cmd == CMD_STEP_INTO:
-                    stop = event in ('line', 'return')
+                    stop = is_line or is_return
                     if plugin_manager is not None:
                         result = plugin_manager.cmd_step_into(main_debugger, frame, event, self._args, stop_info, stop)
                         if result:
@@ -781,13 +737,13 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
 
                 elif step_cmd == CMD_STEP_INTO_MY_CODE:
                     if not main_debugger.not_in_scope(frame.f_code.co_filename):
-                        stop = event == 'line'
+                        stop = is_line
 
                 elif step_cmd == CMD_STEP_OVER:
-                    stop = stop_frame is frame and event in ('line', 'return')
+                    stop = stop_frame is frame and (is_line or is_return)
 
                     if frame.f_code.co_flags & CO_GENERATOR:
-                        if event is 'return':
+                        if is_return:
                             stop = False
 
                     if plugin_manager is not None:
@@ -801,7 +757,7 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
                         info.pydev_func_name = '.invalid.' # Must match the type in cython
                         info.pydev_smart_step_stop = None
 
-                    if event == 'line' or event == 'exception':
+                    if is_line or is_exception_event:
                         curr_func_name = frame.f_code.co_name
 
                         #global context is set with an empty name
@@ -812,12 +768,12 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
                             stop = True
 
                 elif step_cmd == CMD_STEP_RETURN:
-                    stop = event == 'return' and stop_frame is frame
+                    stop = is_return and stop_frame is frame
 
                 elif step_cmd == CMD_RUN_TO_LINE or step_cmd == CMD_SET_NEXT_STATEMENT:
                     stop = False
 
-                    if event == 'line' or event == 'exception':
+                    if is_line or is_exception_event:
                         #Yes, we can only act on line events (weird hum?)
                         #Note: This code is duplicated at pydevd.py
                         #Acting on exception events after debugger breaks with exception
@@ -841,19 +797,18 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
                 else:
                     stop = False
 
-                if stop and step_cmd != -1 and IS_PY3K:
-                    # in Py3k we start script via our custom `execfile` function, and we shouldn't stop there
-                    # while stepping when execution is finished
-                    if event == 'return' and hasattr(frame, "f_back") and hasattr(frame.f_back, "f_code"):
-                        back_filename = os.path.basename(frame.f_back.f_code.co_filename)
-                        file_type = get_file_type(back_filename)
-                        if file_type == PYDEV_FILE:
-                            stop = False
+                    if is_return and hasattr(frame, "f_back"):
+                        f_code = getattr(frame.f_back, 'f_code', None)
+                        if f_code is not None:
+                            back_filename = os.path.basename(f_code.co_filename)
+                            file_type = get_file_type(back_filename)
+                            if file_type == PYDEV_FILE:
+                                stop = False
 
                 if plugin_stop:
                     stopped_on_plugin = plugin_manager.stop(main_debugger, frame, event, self._args, stop_info, arg, step_cmd)
                 elif stop:
-                    if event == 'line':
+                    if is_line:
                         self.set_suspend(thread, step_cmd)
                         self.do_wait_suspend(thread, frame, event, arg)
                     else: #return event
@@ -920,25 +875,34 @@ from _pydevd_bundle.pydevd_dont_trace_files import DONT_TRACE
 from _pydevd_bundle.pydevd_kill_all_pydevd_threads import kill_all_pydev_threads
 from pydevd_file_utils import get_abs_path_real_path_and_base_from_frame, NORM_PATHS_AND_BASE_CONTAINER
 from _pydevd_bundle.pydevd_tracing import SetTrace
-
 # IFDEF CYTHON -- DONT EDIT THIS FILE (it is automatically generated)
 # In Cython, PyDBAdditionalThreadInfo is bundled in the file.
+from cpython.object cimport PyObject
+from cpython.ref cimport Py_INCREF, Py_XDECREF
 # ELSE
 # from _pydevd_bundle.pydevd_additional_thread_info import PyDBAdditionalThreadInfo
+# from _pydevd_bundle.pydevd_frame import PyDBFrame
 # ENDIF
 
 threadingCurrentThread = threading.currentThread
 get_file_type = DONT_TRACE.get
 
+# IFDEF CYTHON -- DONT EDIT THIS FILE (it is automatically generated)
+# cdef dict global_cache_skips
+# cdef dict global_cache_frame_skips
+# ELSE
+# ENDIF
+
+
+# Cache where we should keep that we completely skipped entering some context.
+# It needs to be invalidated when:
+# - Breakpoints are changed
+# It can be used when running regularly (without step over/step in/step return)
+global_cache_skips = {}
+global_cache_frame_skips = {}
+
 def trace_dispatch(py_db, frame, event, arg):
-    #try:
     t = threadingCurrentThread()
-    #except:
-    #this could give an exception (python 2.5 bug), but should not be there anymore...
-    #see http://mail.python.org/pipermail/python-bugs-list/2007-June/038796.html
-    #and related bug: http://bugs.python.org/issue1733757
-    #frame.f_trace = py_db.trace_dispatch
-    #return py_db.trace_dispatch
 
     if getattr(t, 'pydev_do_not_trace', None):
         return None
@@ -950,7 +914,7 @@ def trace_dispatch(py_db, frame, event, arg):
     except:
         additional_info = t.additional_info = PyDBAdditionalThreadInfo()
 
-    thread_tracer = ThreadTracer((py_db, t, additional_info))
+    thread_tracer = ThreadTracer((py_db, t, additional_info, global_cache_skips, global_cache_frame_skips))
 # IFDEF CYTHON -- DONT EDIT THIS FILE (it is automatically generated)
     t._tracer = thread_tracer # Hack for cython to keep it alive while the thread is alive (just the method in the SetTrace is not enough).
 # ELSE
@@ -959,6 +923,18 @@ def trace_dispatch(py_db, frame, event, arg):
     return thread_tracer.__call__(frame, event, arg)
 
 # IFDEF CYTHON -- DONT EDIT THIS FILE (it is automatically generated)
+cdef class SafeCallWrapper:
+  cdef method_object
+  def __init__(self, method_object):
+      self.method_object = method_object
+  def  __call__(self, *args):
+      #Cannot use 'self' once inside the delegate call since we are borrowing the self reference f_trace field
+      #in the frame, and that reference might get destroyed by set trace on frame and parents
+      cdef PyObject* method_obj = <PyObject*> self.method_object
+      Py_INCREF(<object>method_obj)
+      ret = (<object>method_obj)(*args)
+      Py_XDECREF (method_obj)
+      return SafeCallWrapper(ret) if ret is not None else None
 cdef class ThreadTracer:
     cdef public tuple _args;
     def __init__(self, tuple args):
@@ -967,7 +943,7 @@ cdef class ThreadTracer:
 # class ThreadTracer:
 #     def __init__(self, args):
 #         self._args = args
-# ENDIF
+    # ENDIF
 
 
     def __call__(self, frame, event, arg):
@@ -986,10 +962,17 @@ cdef class ThreadTracer:
         # IFDEF CYTHON -- DONT EDIT THIS FILE (it is automatically generated)
         cdef str filename;
         cdef str base;
+        cdef int pydev_step_cmd;
+        cdef tuple cache_key;
+        cdef dict cache_skips;
+        cdef bint is_stepping;
         cdef tuple abs_path_real_path_and_base;
         cdef PyDBAdditionalThreadInfo additional_info;
         # ENDIF
-        py_db, t, additional_info = self._args
+        # print('ENTER: trace_dispatch', frame.f_code.co_filename, frame.f_lineno, event, frame.f_code.co_name)
+        py_db, t, additional_info, cache_skips, frame_skips_cache = self._args
+        pydev_step_cmd = additional_info.pydev_step_cmd
+        is_stepping = pydev_step_cmd != -1
 
         try:
             if py_db._finish_debugging_session:
@@ -1020,22 +1003,32 @@ cdef class ThreadTracer:
             if py_db.asyncio_analyser is not None:
                 py_db.asyncio_analyser.log_event(frame)
 
+            filename = abs_path_real_path_and_base[1]
+            # Note: it's important that the context name is also given because we may hit something once
+            # in the global context and another in the local context.
+            cache_key = (frame.f_lineno, frame.f_code.co_name, filename)
+            if not is_stepping and cache_key in cache_skips:
+                # print('skipped: trace_dispatch (cache hit)', cache_key, frame.f_lineno, event, frame.f_code.co_name)
+                return None
+
             file_type = get_file_type(abs_path_real_path_and_base[-1]) #we don't want to debug threading or anything related to pydevd
 
             if file_type is not None:
                 if file_type == 1: # inlining LIB_FILE = 1
-                    if py_db.not_in_scope(abs_path_real_path_and_base[1]):
-                        # print('skipped: trace_dispatch (not in scope)', base, frame.f_lineno, event, frame.f_code.co_name, file_type)
+                    if py_db.not_in_scope(filename):
+                        # print('skipped: trace_dispatch (not in scope)', abs_path_real_path_and_base[-1], frame.f_lineno, event, frame.f_code.co_name, file_type)
+                        cache_skips[cache_key] = 1
                         return None
                 else:
-                    # print('skipped: trace_dispatch', base, frame.f_lineno, event, frame.f_code.co_name, file_type)
+                    # print('skipped: trace_dispatch', abs_path_real_path_and_base[-1], frame.f_lineno, event, frame.f_code.co_name, file_type)
+                    cache_skips[cache_key] = 1
                     return None
 
-            if additional_info.pydev_step_cmd != -1:
-                if py_db.is_filter_enabled and py_db.is_ignored_by_filters(abs_path_real_path_and_base[1]):
+            if is_stepping:
+                if py_db.is_filter_enabled and py_db.is_ignored_by_filters(filename):
                     # ignore files matching stepping filters
                     return None
-                if py_db.is_filter_libraries and py_db.not_in_scope(abs_path_real_path_and_base[1]):
+                if py_db.is_filter_libraries and py_db.not_in_scope(filename):
                     # ignore library files while stepping
                     return None
 
@@ -1043,13 +1036,17 @@ cdef class ThreadTracer:
             if additional_info.is_tracing:
                 return None  #we don't wan't to trace code invoked from pydevd_frame.trace_dispatch
 
+            # Just create PyDBFrame directly (removed support for Python versions < 2.5, which required keeping a weak
+            # reference to the frame).
+            ret = PyDBFrame((py_db, filename, additional_info, t, frame_skips_cache, (frame.f_code.co_name, frame.f_code.co_firstlineno, filename))).trace_dispatch(frame, event, arg)
+            if ret is None:
+                cache_skips[cache_key] = 1
+                return None
 
-            # each new frame...
             # IFDEF CYTHON -- DONT EDIT THIS FILE (it is automatically generated)
-            # Note that on Cython we only support more modern idioms (no support for < Python 2.5)
-            return PyDBFrame((py_db, abs_path_real_path_and_base[1], additional_info, t)).trace_dispatch(frame, event, arg)
+            return SafeCallWrapper(ret)
             # ELSE
-#             return additional_info.create_db_frame((py_db, abs_path_real_path_and_base[1], additional_info, t, frame)).trace_dispatch(frame, event, arg)
+#             return ret
             # ENDIF
 
         except SystemExit:

python/helpers/pydev/_pydevd_bundle/pydevd_cython_wrapper.py

@@ -1,5 +1,5 @@
 try:
-    from _pydevd_bundle.pydevd_cython import trace_dispatch, PyDBAdditionalThreadInfo
+    from _pydevd_bundle.pydevd_cython import trace_dispatch, PyDBAdditionalThreadInfo, global_cache_skips, global_cache_frame_skips
 except ImportError:
     try:
         import struct
@@ -25,6 +25,7 @@ except ImportError:
         check_name = '_pydevd_bundle.%s' % (mod_name,)
         mod = __import__(check_name)
         mod = getattr(mod, mod_name)
-        trace_dispatch, PyDBAdditionalThreadInfo = mod.trace_dispatch, mod.PyDBAdditionalThreadInfo
+        trace_dispatch, PyDBAdditionalThreadInfo, global_cache_skips, global_cache_frame_skips = \
+            mod.trace_dispatch, mod.PyDBAdditionalThreadInfo, mod.global_cache_skips, mod.global_cache_frame_skips
     except ImportError:
         raise

python/helpers/pydev/_pydevd_bundle/pydevd_dont_trace_files.py

@@ -65,6 +65,7 @@ DONT_TRACE = {
     'pydevd_additional_thread_info_regular.py': PYDEV_FILE,
     'pydevd_breakpoints.py': PYDEV_FILE,
     'pydevd_comm.py': PYDEV_FILE,
+    'pydevd_command_line_handling.py': PYDEV_FILE,
     'pydevd_concurrency_logger.py': PYDEV_FILE,
     'pydevd_console.py': PYDEV_FILE,
     'pydevd_constants.py': PYDEV_FILE,

python/helpers/pydev/_pydevd_bundle/pydevd_frame.py

@@ -11,7 +11,7 @@ from _pydevd_bundle.pydevd_breakpoints import get_exception_breakpoint
 from _pydevd_bundle.pydevd_comm import CMD_STEP_CAUGHT_EXCEPTION, CMD_STEP_RETURN, CMD_STEP_OVER, CMD_SET_BREAK, \
     CMD_STEP_INTO, CMD_SMART_STEP_INTO, CMD_RUN_TO_LINE, CMD_SET_NEXT_STATEMENT, CMD_STEP_INTO_MY_CODE
 from _pydevd_bundle.pydevd_constants import STATE_SUSPEND, dict_contains, get_thread_id, STATE_RUN, dict_iter_values, IS_PY3K, \
-    dict_keys, dict_pop, RETURN_VALUES_DICT
+    dict_keys, RETURN_VALUES_DICT
 from _pydevd_bundle.pydevd_dont_trace_files import DONT_TRACE, PYDEV_FILE
 from _pydevd_bundle.pydevd_frame_utils import add_exception_to_frame, just_raised
 from _pydevd_bundle.pydevd_utils import get_clsname_for_code
@@ -40,11 +40,16 @@ get_file_type = DONT_TRACE.get
 #=======================================================================================================================
 # PyDBFrame
 #=======================================================================================================================
-class PyDBFrame: # No longer cdef because object was dying when only a reference to trace_dispatch was kept (need to check alternatives).
+# IFDEF CYTHON
+# cdef class PyDBFrame:
+# ELSE
+class PyDBFrame:
     '''This makes the tracing for a given frame, so, the trace_dispatch
     is used initially when we enter into a new context ('call') and then
     is reused for the entire context.
     '''
+    # ENDIF
+
 
     #Note: class (and not instance) attributes.
 
@@ -52,16 +57,20 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
     #considers the user input (so, the actual result must be a join of both).
     filename_to_lines_where_exceptions_are_ignored = {}
     filename_to_stat_info = {}
-    should_skip = -1
 
     # IFDEF CYTHON
-    # def __init__(self, args):
-        # self._args = args # In the cython version we don't need to pass the frame
+    # cdef tuple _args
+    # cdef int should_skip
+    # def __init__(self, tuple args):
+    #     self._args = args # In the cython version we don't need to pass the frame
+    #     self.should_skip = -1  # On cythonized version, put in instance.
     # ELSE
+    should_skip = -1  # Default value in class (put in instance on set).
+
     def __init__(self, args):
         #args = main_debugger, filename, base, info, t, frame
         #yeap, much faster than putting in self and then getting it from self later on
-        self._args = args[:-1] # Remove the frame (we don't want to have a reference to it).
+        self._args = args
     # ENDIF
 
     def set_suspend(self, *args, **kwargs):
@@ -104,7 +113,8 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
         info = self._args[2]
         flag = False
 
-        if info.pydev_state != STATE_SUSPEND:  #and breakpoint is not None:
+        # STATE_SUSPEND = 2
+        if info.pydev_state != 2:  #and breakpoint is not None:
             exception, value, trace = arg
 
             if trace is not None: #on jython trace is None on the first event
@@ -139,7 +149,7 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
                         if main_debugger.plugin is not None:
                             result = main_debugger.plugin.exception_break(main_debugger, self, frame, self._args, arg)
                             if result:
-                                (flag, frame) = result
+                                flag, frame = result
                     except:
                         flag = False
 
@@ -266,6 +276,19 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
             main_debugger = None
             thread = None
 
+    def get_func_name(self, frame):
+        code_obj = frame.f_code
+        func_name = code_obj.co_name
+        try:
+            cls_name = get_clsname_for_code(code_obj, frame)
+            if cls_name is not None:
+                return "%s.%s" % (cls_name, func_name)
+            else:
+                return func_name
+        except:
+            traceback.print_exc()
+            return func_name
+
     def manage_return_values(self, main_debugger, frame, event, arg):
 
         def get_func_name(frame):
@@ -293,17 +316,17 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
                 # Showing return values was turned off, we should remove them from locals dict.
                 # The values can be in the current frame or in the back one
                 if RETURN_VALUES_DICT in dict_keys(frame.f_locals):
-                    dict_pop(frame.f_locals, RETURN_VALUES_DICT)
+                    frame.f_locals.pop(RETURN_VALUES_DICT)
                 if hasattr(frame, "f_back") and hasattr(frame.f_back, "f_locals"):
                     if RETURN_VALUES_DICT in dict_keys(frame.f_back.f_locals):
-                        dict_pop(frame.f_back.f_locals, RETURN_VALUES_DICT)
+                        frame.f_back.f_locals.pop(RETURN_VALUES_DICT)
                 main_debugger.remove_return_values_flag = False
         except:
             main_debugger.remove_return_values_flag = False
             traceback.print_exc()
 
     # IFDEF CYTHON
-    # def trace_dispatch(self, frame, str event, arg):
+    # cpdef trace_dispatch(self, frame, str event, arg):
     #     cdef str filename;
     #     cdef bint is_exception_event;
     #     cdef bint has_exception_breakpoints;
@@ -311,26 +334,31 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
     #     cdef PyDBAdditionalThreadInfo info;
     #     cdef int step_cmd;
     #     cdef int line;
+    #     cdef bint is_line;
+    #     cdef bint is_call;
+    #     cdef bint is_return;
     #     cdef str curr_func_name;
     #     cdef bint exist_result;
+    #     cdef dict frame_skips_cache;
+    #     cdef tuple frame_cache_key;
+    #     cdef tuple line_cache_key;
+    #     cdef int breakpoints_in_line_cache;
+    #     cdef int breakpoints_in_frame_cache;
+    #     cdef bint has_breakpoint_in_frame;
     # ELSE
     def trace_dispatch(self, frame, event, arg):
     # ENDIF
 
-        main_debugger, filename, info, thread = self._args
+        main_debugger, filename, info, thread, frame_skips_cache, frame_cache_key = self._args
+        # print('frame trace_dispatch', frame.f_lineno, frame.f_code.co_name, event, info.pydev_step_cmd)
         try:
-            # print 'frame trace_dispatch', frame.f_lineno, frame.f_code.co_name, event
             info.is_tracing = True
-            need_trace_return = False
+            line = frame.f_lineno
+            line_cache_key = (frame_cache_key, line)
 
             if main_debugger._finish_debugging_session:
                 return None
 
-            if event == 'call' and main_debugger.signature_factory:
-                need_trace_return = send_signature_call_trace(main_debugger, frame, filename)
-            if event == 'return' and main_debugger.signature_factory:
-                send_signature_return_trace(main_debugger, frame, filename, arg)
-
             plugin_manager = main_debugger.plugin
 
             is_exception_event = event == 'exception'
@@ -342,19 +370,32 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
                     if flag:
                         self.handle_exception(frame, event, arg)
                         return self.trace_dispatch
+                is_line = False
+                is_return = False
+                is_call = False
+            else:
+                is_line = event == 'line'
+                is_return = event == 'return'
+                is_call = event == 'call'
+                if not is_line and not is_return and not is_call:
+                    # I believe this can only happen in jython on some frontiers on jython and java code, which we don't want to trace.
+                    return None
 
-            elif event not in ('line', 'call', 'return'):
-                #I believe this can only happen in jython on some frontiers on jython and java code, which we don't want to trace.
-                return None
+            need_trace_return = False
+            if is_call and main_debugger.signature_factory:
+                need_trace_return = send_signature_call_trace(main_debugger, frame, filename)
+            if is_return and main_debugger.signature_factory:
+                send_signature_return_trace(main_debugger, frame, filename, arg)
 
             stop_frame = info.pydev_step_stop
             step_cmd = info.pydev_step_cmd
 
             if is_exception_event:
                 breakpoints_for_file = None
-                if stop_frame and stop_frame is not frame and step_cmd is CMD_STEP_OVER and \
+                # CMD_STEP_OVER = 108
+                if stop_frame and stop_frame is not frame and step_cmd == 108 and \
                                 arg[0] in (StopIteration, GeneratorExit) and arg[2] is None:
-                    info.pydev_step_cmd = CMD_STEP_INTO
+                    info.pydev_step_cmd = 107  # CMD_STEP_INTO = 107
                     info.pydev_step_stop = None
             else:
                 # If we are in single step mode and something causes us to exit the current frame, we need to make sure we break
@@ -363,29 +404,31 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
                 # to make a step in or step over at that location).
                 # Note: this is especially troublesome when we're skipping code with the
                 # @DontTrace comment.
-                if stop_frame is frame and event is 'return' and step_cmd in (CMD_STEP_RETURN, CMD_STEP_OVER):
-                    if not frame.f_code.co_flags & CO_GENERATOR:
-                        info.pydev_step_cmd = CMD_STEP_INTO
+                if stop_frame is frame and is_return and step_cmd in (109, 108):  # CMD_STEP_RETURN = 109, CMD_STEP_OVER = 108
+                    if not frame.f_code.co_flags & 0x20:  # CO_GENERATOR = 0x20 (inspect.CO_GENERATOR)
+                        info.pydev_step_cmd = 107  # CMD_STEP_INTO = 107
                         info.pydev_step_stop = None
 
                 breakpoints_for_file = main_debugger.breakpoints.get(filename)
 
                 can_skip = False
 
-                if info.pydev_state == STATE_RUN:
+                if info.pydev_state == 1:  # STATE_RUN = 1
                     #we can skip if:
                     #- we have no stop marked
                     #- we should make a step return/step over and we're not in the current frame
-                    can_skip = (step_cmd == -1 and stop_frame is None)\
-                        or (step_cmd in (CMD_STEP_RETURN, CMD_STEP_OVER) and stop_frame is not frame)
+                    # CMD_STEP_RETURN = 109, CMD_STEP_OVER = 108
+                    can_skip = (step_cmd == -1 and stop_frame is None) \
+                               or (step_cmd in (109, 108) and stop_frame is not frame)
 
-                if can_skip and plugin_manager is not None and main_debugger.has_plugin_line_breaks:
-                    can_skip = not plugin_manager.can_not_skip(main_debugger, self, frame)
+                    if can_skip:
+                        if plugin_manager is not None and main_debugger.has_plugin_line_breaks:
+                            can_skip = not plugin_manager.can_not_skip(main_debugger, self, frame)
 
-                if can_skip and main_debugger.show_return_values:
-                    # trace function for showing return values after step over
-                    if info.pydev_step_cmd == CMD_STEP_OVER and hasattr(frame, "f_back") and frame.f_back == info.pydev_step_stop:
-                        can_skip = False
+                        # CMD_STEP_OVER = 108
+                        if can_skip and is_return and main_debugger.show_return_values and info.pydev_step_cmd == 108 and frame.f_back is info.pydev_step_stop:
+                            # trace function for showing return values after step over
+                            can_skip = False
 
                 if main_debugger.frame_eval_func and event == 'return' and info.pydev_step_cmd == -1:
                     frames_set = main_debugger.disable_tracing_after_exit_frames.get(get_thread_id(thread), None)
@@ -399,8 +442,8 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
 
                 # Let's check to see if we are in a function that has a breakpoint. If we don't have a breakpoint,
                 # we will return nothing for the next trace
-                #also, after we hit a breakpoint and go to some other debugging state, we have to force the set trace anyway,
-                #so, that's why the additional checks are there.
+                # also, after we hit a breakpoint and go to some other debugging state, we have to force the set trace anyway,
+                # so, that's why the additional checks are there.
                 if not breakpoints_for_file:
                     if can_skip:
                         if has_exception_breakpoints:
@@ -412,34 +455,51 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
                                 return None
 
                 else:
-                    #checks the breakpoint to see if there is a context match in some function
-                    curr_func_name = frame.f_code.co_name
+                    # When cached, 0 means we don't have a breakpoint and 1 means we have.
+                    if can_skip:
+                        breakpoints_in_line_cache = frame_skips_cache.get(line_cache_key, -1)
+                        if breakpoints_in_line_cache == 0:
+                            return self.trace_dispatch
 
-                    #global context is set with an empty name
-                    if curr_func_name in ('?', '<module>'):
-                        curr_func_name = ''
+                    breakpoints_in_frame_cache = frame_skips_cache.get(frame_cache_key, -1)
+                    if breakpoints_in_frame_cache != -1:
+                        # Gotten from cache.
+                        has_breakpoint_in_frame = breakpoints_in_frame_cache == 1
 
-                    for breakpoint in dict_iter_values(breakpoints_for_file): #jython does not support itervalues()
-                        #will match either global or some function
-                        if breakpoint.func_name in ('None', curr_func_name):
-                            break
+                    else:
+                        has_breakpoint_in_frame = False
+                        # Checks the breakpoint to see if there is a context match in some function
+                        curr_func_name = frame.f_code.co_name
 
-                    else: # if we had some break, it won't get here (so, that's a context that we want to skip)
-                        if can_skip:
-                            if has_exception_breakpoints:
-                                return self.trace_exception
+                        #global context is set with an empty name
+                        if curr_func_name in ('?', '<module>'):
+                            curr_func_name = ''
+
+                        for breakpoint in dict_iter_values(breakpoints_for_file): #jython does not support itervalues()
+                            #will match either global or some function
+                            if breakpoint.func_name in ('None', curr_func_name):
+                                has_breakpoint_in_frame = True
+                                break
+
+                        # Cache the value (1 or 0 or -1 for default because of cython).
+                        if has_breakpoint_in_frame:
+                            frame_skips_cache[frame_cache_key] = 1
+                        else:
+                            frame_skips_cache[frame_cache_key] = 0
+
+                    if can_skip and not has_breakpoint_in_frame:
+                        if has_exception_breakpoints:
+                            return self.trace_exception
+                        else:
+                            if need_trace_return:
+                                return self.trace_return
                             else:
-                                if need_trace_return:
-                                    return self.trace_return
-                                else:
-                                    return None
-
+                                return None
 
             #We may have hit a breakpoint or we are already in step mode. Either way, let's check what we should do in this frame
-            #print 'NOT skipped', frame.f_lineno, frame.f_code.co_name, event
+            #print('NOT skipped', frame.f_lineno, frame.f_code.co_name, event)
 
             try:
-                line = frame.f_lineno
                 flag = False
                 #return is not taken into account for breakpoint hit because we'd have a double-hit in this case
                 #(one for the line and the other for the return).
@@ -449,18 +509,17 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
                 exist_result = False
                 stop = False
                 bp_type = None
-                if not flag and event != 'return' and info.pydev_state != STATE_SUSPEND and breakpoints_for_file is not None \
-                        and dict_contains(breakpoints_for_file, line):
+                if not is_return and info.pydev_state != STATE_SUSPEND and breakpoints_for_file is not None and dict_contains(breakpoints_for_file, line):
                     breakpoint = breakpoints_for_file[line]
                     new_frame = frame
                     stop = True
-                    if step_cmd == CMD_STEP_OVER and stop_frame is frame and event in ('line', 'return'):
+                    if step_cmd == CMD_STEP_OVER and stop_frame is frame and (is_line or is_return):
                         stop = False #we don't stop on breakpoint if we have to stop by step-over (it will be processed later)
                 elif plugin_manager is not None and main_debugger.has_plugin_line_breaks:
                     result = plugin_manager.get_breakpoint(main_debugger, self, frame, event, self._args)
                     if result:
                         exist_result = True
-                        (flag, breakpoint, new_frame, bp_type) = result
+                        flag, breakpoint, new_frame, bp_type = result
 
                 if breakpoint:
                     #ok, hit breakpoint, now, we have to discover if it is a conditional breakpoint
@@ -512,17 +571,16 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
                                     info.pydev_message = str(val)
 
                         if not main_debugger.first_breakpoint_reached:
-                            if event == 'call':
-                                if hasattr(frame, 'f_back'):
-                                    back = frame.f_back
-                                    if back is not None:
-                                        # When we start debug session, we call execfile in pydevd run function. It produces an additional
-                                        # 'call' event for tracing and we stop on the first line of code twice.
-                                        _, back_filename, base = get_abs_path_real_path_and_base_from_frame(back)
-                                        if (base == DEBUG_START[0] and back.f_code.co_name == DEBUG_START[1]) or \
-                                                (base == DEBUG_START_PY3K[0] and back.f_code.co_name == DEBUG_START_PY3K[1]):
-                                            stop = False
-                                            main_debugger.first_breakpoint_reached = True
+                            if is_call:
+                                back = frame.f_back
+                                if back is not None:
+                                    # When we start debug session, we call execfile in pydevd run function. It produces an additional
+                                    # 'call' event for tracing and we stop on the first line of code twice.
+                                    _, back_filename, base = get_abs_path_real_path_and_base_from_frame(back)
+                                    if (base == DEBUG_START[0] and back.f_code.co_name == DEBUG_START[1]) or \
+                                            (base == DEBUG_START_PY3K[0] and back.f_code.co_name == DEBUG_START_PY3K[1]):
+                                        stop = False
+                                        main_debugger.first_breakpoint_reached = True
                 else:
                     # if the frame is traced after breakpoint stop,
                     # but the file should be ignored while stepping because of filters
@@ -550,6 +608,10 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
                 if info.pydev_state == STATE_SUSPEND:
                     self.do_wait_suspend(thread, frame, event, arg)
                     return self.trace_dispatch
+                else:
+                    if not breakpoint and not is_return:
+                        # No stop from anyone and no breakpoint found in line (cache that).
+                        frame_skips_cache[line_cache_key] = 0
 
             except:
                 traceback.print_exc()
@@ -576,7 +638,7 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
                     stop = False
 
                 elif step_cmd == CMD_STEP_INTO:
-                    stop = event in ('line', 'return')
+                    stop = is_line or is_return
                     if plugin_manager is not None:
                         result = plugin_manager.cmd_step_into(main_debugger, frame, event, self._args, stop_info, stop)
                         if result:
@@ -584,13 +646,13 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
 
                 elif step_cmd == CMD_STEP_INTO_MY_CODE:
                     if not main_debugger.not_in_scope(frame.f_code.co_filename):
-                        stop = event == 'line'
+                        stop = is_line
 
                 elif step_cmd == CMD_STEP_OVER:
-                    stop = stop_frame is frame and event in ('line', 'return')
+                    stop = stop_frame is frame and (is_line or is_return)
 
                     if frame.f_code.co_flags & CO_GENERATOR:
-                        if event is 'return':
+                        if is_return:
                             stop = False
 
                     if plugin_manager is not None:
@@ -604,7 +666,7 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
                         info.pydev_func_name = '.invalid.' # Must match the type in cython
                         info.pydev_smart_step_stop = None
 
-                    if event == 'line' or event == 'exception':
+                    if is_line or is_exception_event:
                         curr_func_name = frame.f_code.co_name
 
                         #global context is set with an empty name
@@ -615,12 +677,12 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
                             stop = True
 
                 elif step_cmd == CMD_STEP_RETURN:
-                    stop = event == 'return' and stop_frame is frame
+                    stop = is_return and stop_frame is frame
 
                 elif step_cmd == CMD_RUN_TO_LINE or step_cmd == CMD_SET_NEXT_STATEMENT:
                     stop = False
 
-                    if event == 'line' or event == 'exception':
+                    if is_line or is_exception_event:
                         #Yes, we can only act on line events (weird hum?)
                         #Note: This code is duplicated at pydevd.py
                         #Acting on exception events after debugger breaks with exception
@@ -644,19 +706,18 @@ class PyDBFrame: # No longer cdef because object was dying when only a reference
                 else:
                     stop = False
 
-                if stop and step_cmd != -1 and IS_PY3K:
-                    # in Py3k we start script via our custom `execfile` function, and we shouldn't stop there
-                    # while stepping when execution is finished
-                    if event == 'return' and hasattr(frame, "f_back") and hasattr(frame.f_back, "f_code"):
-                        back_filename = os.path.basename(frame.f_back.f_code.co_filename)
-                        file_type = get_file_type(back_filename)
-                        if file_type == PYDEV_FILE:
-                            stop = False
+                    if is_return and hasattr(frame, "f_back"):
+                        f_code = getattr(frame.f_back, 'f_code', None)
+                        if f_code is not None:
+                            back_filename = os.path.basename(f_code.co_filename)
+                            file_type = get_file_type(back_filename)
+                            if file_type == PYDEV_FILE:
+                                stop = False
 
                 if plugin_stop:
                     stopped_on_plugin = plugin_manager.stop(main_debugger, frame, event, self._args, stop_info, arg, step_cmd)
                 elif stop:
-                    if event == 'line':
+                    if is_line:
                         self.set_suspend(thread, step_cmd)
                         self.do_wait_suspend(thread, frame, event, arg)
                     else: #return event

python/helpers/pydev/_pydevd_bundle/pydevd_process_net_command.py

@@ -18,7 +18,7 @@ from _pydevd_bundle.pydevd_comm import CMD_RUN, CMD_VERSION, CMD_LIST_THREADS, C
     CMD_EVALUATE_CONSOLE_EXPRESSION, InternalEvaluateConsoleExpression, InternalConsoleGetCompletions, \
     CMD_RUN_CUSTOM_OPERATION, InternalRunCustomOperation, CMD_IGNORE_THROWN_EXCEPTION_AT, CMD_ENABLE_DONT_TRACE, \
     CMD_SHOW_RETURN_VALUES, ID_TO_MEANING, CMD_GET_DESCRIPTION, InternalGetDescription, pydevd_check_frame_for_new_breakpoint
-from _pydevd_bundle.pydevd_constants import get_thread_id, IS_PY3K, DebugInfoHolder, dict_contains, dict_keys, dict_pop, \
+from _pydevd_bundle.pydevd_constants import get_thread_id, IS_PY3K, DebugInfoHolder, dict_contains, dict_keys, \
     STATE_RUN
 
 
@@ -538,11 +538,11 @@ def process_net_command(py_db, cmd_id, seq, text):
                 if exception_type == 'python':
                     try:
                         cp = py_db.break_on_uncaught_exceptions.copy()
-                        dict_pop(cp, exception, None)
+                        cp.pop(exception, None)
                         py_db.break_on_uncaught_exceptions = cp
 
                         cp = py_db.break_on_caught_exceptions.copy()
-                        dict_pop(cp, exception, None)
+                        cp.pop(exception, None)
                         py_db.break_on_caught_exceptions = cp
                     except:
                         pydev_log.debug("Error while removing exception %s"%sys.exc_info()[0])

python/helpers/pydev/_pydevd_bundle/pydevd_trace_dispatch.py

@@ -7,24 +7,24 @@ import sys
 
 if use_cython == 'YES':
     # We must import the cython version if forcing cython
-    from _pydevd_bundle.pydevd_cython_wrapper import trace_dispatch as _trace_dispatch
+    from _pydevd_bundle.pydevd_cython_wrapper import trace_dispatch as _trace_dispatch, global_cache_skips, global_cache_frame_skips
     def trace_dispatch(py_db, frame, event, arg):
         return _trace_dispatch(py_db, frame, event, arg)
 
 elif use_cython == 'NO':
     # Use the regular version if not forcing cython
-    from _pydevd_bundle.pydevd_trace_dispatch_regular import trace_dispatch  # @UnusedImport
+    from _pydevd_bundle.pydevd_trace_dispatch_regular import trace_dispatch, global_cache_skips, global_cache_frame_skips  # @UnusedImport
 
 elif use_cython is None:
     # Regular: use fallback if not found and give message to user
     try:
-        from _pydevd_bundle.pydevd_cython_wrapper import trace_dispatch as _trace_dispatch
+        from _pydevd_bundle.pydevd_cython_wrapper import trace_dispatch as _trace_dispatch, global_cache_skips, global_cache_frame_skips
         def trace_dispatch(py_db, frame, event, arg):
             return _trace_dispatch(py_db, frame, event, arg)
 
     except ImportError:
         from _pydevd_bundle.pydevd_additional_thread_info_regular import PyDBAdditionalThreadInfo  # @UnusedImport
-        from _pydevd_bundle.pydevd_trace_dispatch_regular import trace_dispatch  # @UnusedImport
+        from _pydevd_bundle.pydevd_trace_dispatch_regular import trace_dispatch, global_cache_skips, global_cache_frame_skips  # @UnusedImport
         from _pydevd_bundle.pydevd_constants import CYTHON_SUPPORTED
 
         dirname = os.path.dirname(os.path.dirname(__file__))
@@ -39,5 +39,3 @@ elif use_cython is None:
 
 else:
     raise RuntimeError('Unexpected value for PYDEVD_USE_CYTHON: %s (accepted: YES, NO)' % (use_cython,))
-
-

python/helpers/pydev/_pydevd_bundle/pydevd_trace_dispatch_regular.py

@@ -7,25 +7,34 @@ from _pydevd_bundle.pydevd_dont_trace_files import DONT_TRACE
 from _pydevd_bundle.pydevd_kill_all_pydevd_threads import kill_all_pydev_threads
 from pydevd_file_utils import get_abs_path_real_path_and_base_from_frame, NORM_PATHS_AND_BASE_CONTAINER
 from _pydevd_bundle.pydevd_tracing import SetTrace
-
 # IFDEF CYTHON
 # # In Cython, PyDBAdditionalThreadInfo is bundled in the file.
+# from cpython.object cimport PyObject
+# from cpython.ref cimport Py_INCREF, Py_XDECREF
 # ELSE
 from _pydevd_bundle.pydevd_additional_thread_info import PyDBAdditionalThreadInfo
+from _pydevd_bundle.pydevd_frame import PyDBFrame
 # ENDIF
 
 threadingCurrentThread = threading.currentThread
 get_file_type = DONT_TRACE.get
 
+# IFDEF CYTHON -- DONT EDIT THIS FILE (it is automatically generated)
+# cdef dict global_cache_skips
+# cdef dict global_cache_frame_skips
+# ELSE
+# ENDIF
+
+
+# Cache where we should keep that we completely skipped entering some context.
+# It needs to be invalidated when:
+# - Breakpoints are changed
+# It can be used when running regularly (without step over/step in/step return)
+global_cache_skips = {}
+global_cache_frame_skips = {}
+
 def trace_dispatch(py_db, frame, event, arg):
-    #try:
     t = threadingCurrentThread()
-    #except:
-    #this could give an exception (python 2.5 bug), but should not be there anymore...
-    #see http://mail.python.org/pipermail/python-bugs-list/2007-June/038796.html
-    #and related bug: http://bugs.python.org/issue1733757
-    #frame.f_trace = py_db.trace_dispatch
-    #return py_db.trace_dispatch
 
     if getattr(t, 'pydev_do_not_trace', None):
         return None
@@ -37,7 +46,7 @@ def trace_dispatch(py_db, frame, event, arg):
     except:
         additional_info = t.additional_info = PyDBAdditionalThreadInfo()
 
-    thread_tracer = ThreadTracer((py_db, t, additional_info))
+    thread_tracer = ThreadTracer((py_db, t, additional_info, global_cache_skips, global_cache_frame_skips))
 # IFDEF CYTHON
 #     t._tracer = thread_tracer # Hack for cython to keep it alive while the thread is alive (just the method in the SetTrace is not enough).
 # ELSE
@@ -46,6 +55,18 @@ def trace_dispatch(py_db, frame, event, arg):
     return thread_tracer.__call__(frame, event, arg)
 
 # IFDEF CYTHON
+# cdef class SafeCallWrapper:
+#   cdef method_object
+#   def __init__(self, method_object):
+#       self.method_object = method_object
+#   def  __call__(self, *args):
+#       #Cannot use 'self' once inside the delegate call since we are borrowing the self reference f_trace field
+#       #in the frame, and that reference might get destroyed by set trace on frame and parents
+#       cdef PyObject* method_obj = <PyObject*> self.method_object
+#       Py_INCREF(<object>method_obj)
+#       ret = (<object>method_obj)(*args)
+#       Py_XDECREF (method_obj)
+#       return SafeCallWrapper(ret) if ret is not None else None
 # cdef class ThreadTracer:
 #     cdef public tuple _args;
 #     def __init__(self, tuple args):
@@ -54,7 +75,7 @@ def trace_dispatch(py_db, frame, event, arg):
 class ThreadTracer:
     def __init__(self, args):
         self._args = args
-# ENDIF
+    # ENDIF
 
 
     def __call__(self, frame, event, arg):
@@ -73,10 +94,17 @@ class ThreadTracer:
         # IFDEF CYTHON
         # cdef str filename;
         # cdef str base;
+        # cdef int pydev_step_cmd;
+        # cdef tuple cache_key;
+        # cdef dict cache_skips;
+        # cdef bint is_stepping;
         # cdef tuple abs_path_real_path_and_base;
         # cdef PyDBAdditionalThreadInfo additional_info;
         # ENDIF
-        py_db, t, additional_info = self._args
+        # print('ENTER: trace_dispatch', frame.f_code.co_filename, frame.f_lineno, event, frame.f_code.co_name)
+        py_db, t, additional_info, cache_skips, frame_skips_cache = self._args
+        pydev_step_cmd = additional_info.pydev_step_cmd
+        is_stepping = pydev_step_cmd != -1
 
         try:
             if py_db._finish_debugging_session:
@@ -107,22 +135,32 @@ class ThreadTracer:
             if py_db.asyncio_analyser is not None:
                 py_db.asyncio_analyser.log_event(frame)
 
+            filename = abs_path_real_path_and_base[1]
+            # Note: it's important that the context name is also given because we may hit something once
+            # in the global context and another in the local context.
+            cache_key = (frame.f_lineno, frame.f_code.co_name, filename)
+            if not is_stepping and cache_key in cache_skips:
+                # print('skipped: trace_dispatch (cache hit)', cache_key, frame.f_lineno, event, frame.f_code.co_name)
+                return None
+
             file_type = get_file_type(abs_path_real_path_and_base[-1]) #we don't want to debug threading or anything related to pydevd
 
             if file_type is not None:
                 if file_type == 1: # inlining LIB_FILE = 1
-                    if py_db.not_in_scope(abs_path_real_path_and_base[1]):
-                        # print('skipped: trace_dispatch (not in scope)', base, frame.f_lineno, event, frame.f_code.co_name, file_type)
+                    if py_db.not_in_scope(filename):
+                        # print('skipped: trace_dispatch (not in scope)', abs_path_real_path_and_base[-1], frame.f_lineno, event, frame.f_code.co_name, file_type)
+                        cache_skips[cache_key] = 1
                         return None
                 else:
-                    # print('skipped: trace_dispatch', base, frame.f_lineno, event, frame.f_code.co_name, file_type)
+                    # print('skipped: trace_dispatch', abs_path_real_path_and_base[-1], frame.f_lineno, event, frame.f_code.co_name, file_type)
+                    cache_skips[cache_key] = 1
                     return None
 
-            if additional_info.pydev_step_cmd != -1:
-                if py_db.is_filter_enabled and py_db.is_ignored_by_filters(abs_path_real_path_and_base[1]):
+            if is_stepping:
+                if py_db.is_filter_enabled and py_db.is_ignored_by_filters(filename):
                     # ignore files matching stepping filters
                     return None
-                if py_db.is_filter_libraries and py_db.not_in_scope(abs_path_real_path_and_base[1]):
+                if py_db.is_filter_libraries and py_db.not_in_scope(filename):
                     # ignore library files while stepping
                     return None
 
@@ -130,13 +168,17 @@ class ThreadTracer:
             if additional_info.is_tracing:
                 return None  #we don't wan't to trace code invoked from pydevd_frame.trace_dispatch
 
+            # Just create PyDBFrame directly (removed support for Python versions < 2.5, which required keeping a weak
+            # reference to the frame).
+            ret = PyDBFrame((py_db, filename, additional_info, t, frame_skips_cache, (frame.f_code.co_name, frame.f_code.co_firstlineno, filename))).trace_dispatch(frame, event, arg)
+            if ret is None:
+                cache_skips[cache_key] = 1
+                return None
 
-            # each new frame...
             # IFDEF CYTHON
-            # # Note that on Cython we only support more modern idioms (no support for < Python 2.5)
-            # return PyDBFrame((py_db, abs_path_real_path_and_base[1], additional_info, t)).trace_dispatch(frame, event, arg)
+            # return SafeCallWrapper(ret)
             # ELSE
-            return additional_info.create_db_frame((py_db, abs_path_real_path_and_base[1], additional_info, t, frame)).trace_dispatch(frame, event, arg)
+            return ret
             # ENDIF
 
         except SystemExit:

python/helpers/pydev/_pydevd_frame_eval/pydevd_frame_evaluator.pyx

@@ -2,7 +2,7 @@ import dis
 from _pydev_imps._pydev_saved_modules import threading
 from _pydevd_bundle.pydevd_additional_thread_info import PyDBAdditionalThreadInfo
 from _pydevd_bundle.pydevd_comm import get_global_debugger, CMD_THREAD_SUSPEND
-from _pydevd_bundle.pydevd_constants import STATE_SUSPEND
+from _pydevd_bundle.pydevd_constants import STATE_RUN, STATE_SUSPEND
 from _pydevd_bundle.pydevd_dont_trace_files import DONT_TRACE
 from _pydevd_frame_eval.pydevd_frame_tracing import pydev_trace_code_wrapper, update_globals_dict
 from _pydevd_frame_eval.pydevd_modify_bytecode import insert_code
@@ -93,7 +93,8 @@ cdef PyObject* get_bytecode_while_frame_eval(PyFrameObject *frame_obj, int exc):
             return _PyEval_EvalFrameDefault(frame_obj, exc)
 
         main_debugger = get_global_debugger()
-        if additional_info.pydev_state == STATE_SUSPEND and t.stop_reason == CMD_THREAD_SUSPEND:
+        if (additional_info.pydev_state == STATE_SUSPEND and t.stop_reason == CMD_THREAD_SUSPEND) or \
+            (additional_info.pydev_state == STATE_RUN and main_debugger.disable_tracing_after_exit_frames):
             main_debugger.process_internal_commands()
 
         was_break = False

python/helpers/pydev/pydevd.py

@@ -10,7 +10,7 @@ import os
 import sys
 import traceback
 
-from _pydevd_bundle.pydevd_constants import IS_JYTH_LESS25, IS_PY3K, IS_PY34_OLDER, get_thread_id, dict_keys, dict_pop, dict_contains, \
+from _pydevd_bundle.pydevd_constants import IS_JYTH_LESS25, IS_PY3K, IS_PY34_OLDER, get_thread_id, dict_keys, dict_contains, \
     dict_iter_items, DebugInfoHolder, PYTHON_SUSPEND, STATE_SUSPEND, STATE_RUN, get_frame, xrange, \
     clear_cached_thread_id, INTERACTIVE_MODE_AVAILABLE
 from _pydev_bundle import fix_getpass
@@ -42,7 +42,7 @@ from pydevd_concurrency_analyser.pydevd_concurrency_logger import ThreadingLogge
 from pydevd_concurrency_analyser.pydevd_thread_wrappers import wrap_threads
 
 
-__version_info__ = (0, 0, 5)
+__version_info__ = (1, 0, 0)
 __version_info_str__ = []
 for v in __version_info__:
     __version_info_str__.append(str(v))
@@ -247,6 +247,9 @@ class PyDB:
         self.mpl_hooks_in_debug_console = False
         self.mpl_modules_for_patching = {}
 
+        # this dict is used for frame evaluation, it holds thread_id -> set of frames
+        self.disable_tracing_after_exit_frames = {}
+
         self._filename_to_not_in_scope = {}
         self.first_breakpoint_reached = False
         self.is_filter_enabled = pydevd_utils.is_filter_enabled()
@@ -254,9 +257,6 @@ class PyDB:
         self.show_return_values = False
         self.remove_return_values_flag = False
 
-        # this dict is used for frame evaluation, it holds thread_id -> set of frames
-        self.disable_tracing_after_exit_frames = {}
-
     def get_plugin_lazy_init(self):
         if self.plugin is None and SUPPORT_PLUGINS:
             self.plugin = PluginManager(self)
@@ -379,7 +379,7 @@ class PyDB:
         # import hook and patches for matplotlib support in debug console
         from _pydev_bundle.pydev_import_hook import import_hook_manager
         for module in dict_keys(self.mpl_modules_for_patching):
-            import_hook_manager.add_module_name(module, dict_pop(self.mpl_modules_for_patching, module))
+            import_hook_manager.add_module_name(module, self.mpl_modules_for_patching.pop(module))
 
     def init_matplotlib_support(self):
         # prepare debugger for integration with matplotlib GUI event loop
@@ -408,7 +408,7 @@ class PyDB:
         if len(self.mpl_modules_for_patching) > 0:
             for module in dict_keys(self.mpl_modules_for_patching):
                 if module in sys.modules:
-                    activate_function = dict_pop(self.mpl_modules_for_patching, module)
+                    activate_function = self.mpl_modules_for_patching.pop(module)
                     activate_function()
                     self.mpl_in_use = True
 
@@ -586,11 +586,14 @@ class PyDB:
 
 
     def consolidate_breakpoints(self, file, id_to_breakpoint, breakpoints):
+        from _pydevd_bundle.pydevd_trace_dispatch import global_cache_skips, global_cache_frame_skips
         break_dict = {}
         for breakpoint_id, pybreakpoint in dict_iter_items(id_to_breakpoint):
             break_dict[pybreakpoint.line] = pybreakpoint
 
         breakpoints[file] = break_dict
+        global_cache_skips.clear()
+        global_cache_frame_skips.clear()
 
     def add_break_on_exception(
             self,
@@ -931,8 +934,10 @@ class PyDB:
                     return meth(mod_name)
         return None
 
-    def run(self, file, globals=None, locals=None, module=False, set_trace=True):
-        if module:
+    def run(self, file, globals=None, locals=None, is_module=False, set_trace=True):
+        module_name = None
+        if is_module:
+            module_name = file
             filename = self.get_fullname(file)
             if filename is None:
                 sys.stderr.write("No module named %s\n" % file)
@@ -971,11 +976,12 @@ class PyDB:
                 # print >> sys.stderr, 'Deleting: ', sys.path[0]
                 del sys.path[0]
 
-            # now, the local directory has to be added to the pythonpath
-            # sys.path.insert(0, os.getcwd())
-            # Changed: it's not the local directory, but the directory of the file launched
-            # The file being run ust be in the pythonpath (even if it was not before)
-            sys.path.insert(0, os.path.split(file)[0])
+            if not is_module:
+                # now, the local directory has to be added to the pythonpath
+                # sys.path.insert(0, os.getcwd())
+                # Changed: it's not the local directory, but the directory of the file launched
+                # The file being run must be in the pythonpath (even if it was not before)
+                sys.path.insert(0, os.path.split(file)[0])
 
             while not self.ready_to_run:
                 time.sleep(0.1)  # busy wait until we receive run command
@@ -1005,7 +1011,16 @@ class PyDB:
             sys.stderr.write("Matplotlib support in debugger failed\n")
             traceback.print_exc()
 
-        pydev_imports.execfile(file, globals, locals)  # execute the script
+        if not is_module:
+            pydev_imports.execfile(file, globals, locals)  # execute the script
+        else:
+            # Run with the -m switch
+            import runpy
+            if hasattr(runpy, '_run_module_as_main'):
+                # Newer versions of Python actually use this when the -m switch is used.
+                runpy._run_module_as_main(module_name, alter_argv=False)
+            else:
+                runpy.run_module(module_name)
         return globals
 
     def exiting(self):
@@ -1045,91 +1060,14 @@ def set_debug(setup):
     setup['DEBUG_TRACE_LEVEL'] = 3
 
 
-def enable_qt_support():
+def enable_qt_support(qt_support_mode):
     from _pydev_bundle import pydev_monkey_qt
-    pydev_monkey_qt.patch_qt()
+    pydev_monkey_qt.patch_qt(qt_support_mode)
 
 
-def process_command_line(argv):
-    """ parses the arguments.
-        removes our arguments from the command line """
-    setup = {}
-    setup['client'] = None
-    setup['server'] = False
-    setup['port'] = 0
-    setup['file'] = ''
-    setup['multiproc'] = False #Used by PyCharm (reuses connection: ssh tunneling)
-    setup['multiprocess'] = False # Used by PyDev (creates new connection to ide)
-    setup['save-signatures'] = False
-    setup['save-threading'] = False
-    setup['save-asyncio'] = False
-    setup['qt-support'] = False
-    setup['print-in-debugger-startup'] = False
-    setup['cmd-line'] = False
-    setup['module'] = False
-    i = 0
-    del argv[0]
-    while (i < len(argv)):
-        if argv[i] == '--port':
-            del argv[i]
-            setup['port'] = int(argv[i])
-            del argv[i]
-        elif argv[i] == '--vm_type':
-            del argv[i]
-            setup['vm_type'] = argv[i]
-            del argv[i]
-        elif argv[i] == '--client':
-            del argv[i]
-            setup['client'] = argv[i]
-            del argv[i]
-        elif argv[i] == '--server':
-            del argv[i]
-            setup['server'] = True
-        elif argv[i] == '--file':
-            del argv[i]
-            setup['file'] = argv[i]
-            i = len(argv) # pop out, file is our last argument
-        elif argv[i] == '--DEBUG_RECORD_SOCKET_READS':
-            del argv[i]
-            setup['DEBUG_RECORD_SOCKET_READS'] = True
-        elif argv[i] == '--DEBUG':
-            del argv[i]
-            set_debug(setup)
-        elif argv[i] == '--multiproc':
-            del argv[i]
-            setup['multiproc'] = True
-        elif argv[i] == '--multiprocess':
-            del argv[i]
-            setup['multiprocess'] = True
-        elif argv[i] == '--save-signatures':
-            del argv[i]
-            setup['save-signatures'] = True
-        elif argv[i] == '--save-threading':
-            del argv[i]
-            setup['save-threading'] = True
-        elif argv[i] == '--save-asyncio':
-            del argv[i]
-            setup['save-asyncio'] = True
-        elif argv[i] == '--qt-support':
-            del argv[i]
-            setup['qt-support'] = True
-
-        elif argv[i] == '--print-in-debugger-startup':
-            del argv[i]
-            setup['print-in-debugger-startup'] = True
-        elif (argv[i] == '--cmd-line'):
-            del argv[i]
-            setup['cmd-line'] = True
-        elif (argv[i] == '--module'):
-            del argv[i]
-            setup['module'] = True
-        else:
-            raise ValueError("unexpected option " + argv[i])
-    return setup
-
 def usage(doExit=0):
     sys.stdout.write('Usage:\n')
-    sys.stdout.write('pydevd.py --port=N [(--client hostname) | --server] --file executable [file_options]\n')
+    sys.stdout.write('pydevd.py --port N [(--client hostname) | --server] --file executable [file_options]\n')
     if doExit:
         sys.exit(0)
 
@@ -1237,6 +1175,15 @@ def _locked_settrace(
     if not connected :
         pydevd_vm_type.setup_type()
 
+        if SetupHolder.setup is None:
+            setup = {
+                'client': host,  # dispatch expects client to be set to the host address when server is False
+                'server': False,
+                'port': int(port),
+                'multiprocess': patch_multiprocessing,
+            }
+            SetupHolder.setup = setup
+
         debugger = PyDB()
         debugger.connect(host, port)  # Note: connect can raise error.
 
@@ -1437,7 +1384,7 @@ def apply_debugger_options(setup_options):
 
     :type setup_options: dict[str, bool]
     """
-    default_options = {'save-signatures': False, 'qt-support': False}
+    default_options = {'save-signatures': False, 'qt-support': ''}
     default_options.update(setup_options)
     setup_options = default_options
 
@@ -1451,7 +1398,7 @@ def apply_debugger_options(setup_options):
             debugger.signature_factory = SignatureFactory()
 
     if setup_options['qt-support']:
-        enable_qt_support()
+        enable_qt_support(setup_options['qt-support'])
 
 
 def patch_stdin(debugger):
@@ -1467,7 +1414,7 @@ if __name__ == '__main__':
 
     # parse the command line. --file is our last argument that is required
     try:
-        sys.original_argv = sys.argv[:]
+        from _pydevd_bundle.pydevd_command_line_handling import process_command_line
         setup = process_command_line(sys.argv)
         SetupHolder.setup = setup
     except ValueError:

python/helpers/pydev/pydevd_attach_to_process/add_code_to_python_process.py

@@ -280,8 +280,8 @@ def run_python_code_windows(pid, python_code, connect_debugger_tracing=False, sh
 
     if is_64 != is_python_64bit():
         raise RuntimeError("The architecture of the Python used to connect doesn't match the architecture of the target.\n"
-        "Target 64 bits: %s\n"
-        "Current Python 64 bits: %s" % (is_64, is_python_64bit()))
+                           "Target 64 bits: %s\n"
+                           "Current Python 64 bits: %s" % (is_64, is_python_64bit()))
 
     print('Connecting to %s bits target' % (bits,))
     assert resolve_label(process, compat.b('PyGILState_Ensure'))
@@ -366,16 +366,16 @@ def run_python_code_windows(pid, python_code, connect_debugger_tracing=False, sh
 
 
     # Uncomment to see the disassembled version of what we just did...
-#     with open('f.asm', 'wb') as stream:
-#         stream.write(code)
-#
-#     exe = r'x:\nasm\nasm-2.07-win32\nasm-2.07\ndisasm.exe'
-#     if is_64:
-#         arch = '64'
-#     else:
-#         arch = '32'
-#
-#     subprocess.call((exe + ' -b %s f.asm' % arch).split())
+    #     with open('f.asm', 'wb') as stream:
+    #         stream.write(code)
+    #
+    #     exe = r'x:\nasm\nasm-2.07-win32\nasm-2.07\ndisasm.exe'
+    #     if is_64:
+    #         arch = '64'
+    #     else:
+    #         arch = '32'
+    #
+    #     subprocess.call((exe + ' -b %s f.asm' % arch).split())
 
     print('Injecting code to target process')
     thread, _thread_address = process.inject_code(code, 0)
@@ -414,7 +414,7 @@ def run_python_code_linux(pid, python_code, connect_debugger_tracing=False, show
     print('Attaching with arch: %s'% (arch,))
 
     target_dll = os.path.join(filedir, 'attach_linux_%s.so' % suffix)
-    target_dll = os.path.normpath(target_dll)
+    target_dll = os.path.abspath(os.path.normpath(target_dll))
     if not os.path.exists(target_dll):
         raise RuntimeError('Could not find dll file to inject: %s' % target_dll)
 
@@ -428,11 +428,11 @@ def run_python_code_linux(pid, python_code, connect_debugger_tracing=False, show
         '--nw',  # no gui interface
         '--nh',  # no ~/.gdbinit
         '--nx',  # no .gdbinit
-#         '--quiet',  # no version number on startup
+        #         '--quiet',  # no version number on startup
         '--pid',
         str(pid),
         '--batch',
-#         '--batch-silent',
+        #         '--batch-silent',
     ]
 
     cmd.extend(["--eval-command='set scheduler-locking off'"])  # If on we'll deadlock.
@@ -449,7 +449,7 @@ def run_python_code_linux(pid, python_code, connect_debugger_tracing=False, show
     if connect_debugger_tracing:
         cmd.extend([
             "--command='%s'" % (gdb_threads_settrace_file,),
-        ])
+            ])
 
     #print ' '.join(cmd)
 
@@ -527,8 +527,8 @@ def run_python_code_mac(pid, python_code, connect_debugger_tracing=False, show_d
         "-o 'process attach --pid %d'"%pid,
         "-o 'command script import \"%s\"'" % (lldb_prepare_file,),
         "-o 'load_lib_and_attach \"%s\" %s \"%s\" %s'" % (target_dll,
-            is_debug, python_code, show_debug_info),
-    ])
+                                                          is_debug, python_code, show_debug_info),
+        ])
 
 
     if connect_debugger_tracing:
@@ -556,7 +556,7 @@ def run_python_code_mac(pid, python_code, connect_debugger_tracing=False, show_d
         env=env,
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
-        )
+    )
     print('Running lldb in target process.')
     out, err = p.communicate()
     print('stdout: %s' % (out,))

python/helpers/pydev/pydevd_plugins/django_debug.py

@@ -352,7 +352,8 @@ def has_line_breaks(plugin):
 
 
 def cmd_step_into(plugin, main_debugger, frame, event, args, stop_info, stop):
-    main_debugger, filename, info, thread = args
+    info = args[2]
+    thread = args[3]
     plugin_stop = False
     if _is_django_suspended(thread):
         stop_info['django_stop'] = event == 'call' and _is_django_render_call(frame)
@@ -364,7 +365,8 @@ def cmd_step_into(plugin, main_debugger, frame, event, args, stop_info, stop):
 
 
 def cmd_step_over(plugin, main_debugger, frame, event, args, stop_info, stop):
-    main_debugger, filename, info, thread = args
+    info = args[2]
+    thread = args[3]
     plugin_stop = False
     if _is_django_suspended(thread):
         stop_info['django_stop'] = event == 'call' and _is_django_render_call(frame)
@@ -382,7 +384,8 @@ def cmd_step_over(plugin, main_debugger, frame, event, args, stop_info, stop):
 
 
 def stop(plugin, main_debugger, frame, event, args, stop_info, arg, step_cmd):
-    main_debugger, filename, info, thread = args
+    main_debugger = args[0]
+    thread = args[3]
     if dict_contains(stop_info, 'django_stop') and stop_info['django_stop']:
         frame = suspend_django(main_debugger, thread, frame, step_cmd)
         if frame:
@@ -392,7 +395,9 @@ def stop(plugin, main_debugger, frame, event, args, stop_info, arg, step_cmd):
 
 
 def get_breakpoint(plugin, main_debugger, pydb_frame, frame, event, args):
-    main_debugger, filename, info, thread = args
+    main_debugger = args[0]
+    filename = args[1]
+    info = args[2]
     flag = False
     django_breakpoint = None
     new_frame = None
@@ -421,10 +426,11 @@ def suspend(plugin, main_debugger, thread, frame, bp_type):
     return None
 
 def exception_break(plugin, main_debugger, pydb_frame, frame, args, arg):
-    main_debugger, filename, info, thread = args
+    main_debugger = args[0]
+    thread = args[3]
     exception, value, trace = arg
     if main_debugger.django_exception_break and \
-            get_exception_name(exception) in ['VariableDoesNotExist', 'TemplateDoesNotExist', 'TemplateSyntaxError'] and \
+                    get_exception_name(exception) in ['VariableDoesNotExist', 'TemplateDoesNotExist', 'TemplateSyntaxError'] and \
             just_raised(trace) and _is_django_exception_break_context(frame):
         render_frame = _find_django_render_frame(frame)
         if render_frame:

python/helpers/pydev/pydevd_plugins/jinja2_debug.py

@@ -238,7 +238,8 @@ def can_not_skip(plugin, pydb, pydb_frame, frame):
 
 
 def cmd_step_into(plugin, pydb, frame, event, args, stop_info, stop):
-    pydb, filename, info, thread = args
+    info = args[2]
+    thread = args[3]
     plugin_stop = False
     stop_info['jinja2_stop'] = False
     if _is_jinja2_suspended(thread):
@@ -271,7 +272,8 @@ def cmd_step_into(plugin, pydb, frame, event, args, stop_info, stop):
 
 
 def cmd_step_over(plugin, pydb, frame, event, args, stop_info, stop):
-    pydb, filename, info, thread = args
+    info = args[2]
+    thread = args[3]
     plugin_stop = False
     stop_info['jinja2_stop'] = False
     if _is_jinja2_suspended(thread):
@@ -311,7 +313,8 @@ def cmd_step_over(plugin, pydb, frame, event, args, stop_info, stop):
 
 
 def stop(plugin, pydb, frame, event, args, stop_info, arg, step_cmd):
-    pydb, filename, info, thread = args
+    pydb = args[0]
+    thread = args[3]
     if dict_contains(stop_info, 'jinja2_stop') and stop_info['jinja2_stop']:
         frame = _suspend_jinja2(pydb, thread, frame, step_cmd)
         if frame:
@@ -321,7 +324,9 @@ def stop(plugin, pydb, frame, event, args, stop_info, arg, step_cmd):
 
 
 def get_breakpoint(plugin, pydb, pydb_frame, frame, event, args):
-    pydb, filename, info, thread = args
+    pydb= args[0]
+    filename = args[1]
+    info = args[2]
     new_frame = None
     jinja2_breakpoint = None
     flag = False
@@ -357,7 +362,8 @@ def suspend(plugin, pydb, thread, frame, bp_type):
 
 
 def exception_break(plugin, pydb, pydb_frame, frame, args, arg):
-    pydb, filename, info, thread = args
+    pydb = args[0]
+    thread = args[3]
     exception, value, trace = arg
     if pydb.jinja2_exception_break:
         exception_type = dict_keys(pydb.jinja2_exception_break)[0]

python/helpers/pydev/setup.py

@@ -104,6 +104,7 @@ args = dict(
         'pydevconsole',
         'pydevd_file_utils',
         'pydevd',
+        'pydevd_tracing',
         # 'runfiles', -- Not needed for debugger
         # 'setup_cython', -- Should not be included as a module
         # 'setup', -- Should not be included as a module

python/helpers/pydev/tests_pydevd_python/_debugger_case_m_switch.py

@@ -0,0 +1,3 @@
+from . import _debugger_case_m_switch_2
+print(_debugger_case_m_switch_2.ClassToBeImported)
+print('TEST SUCEEDED!')

python/helpers/pydev/tests_pydevd_python/_debugger_case_m_switch_2.py

@@ -0,0 +1,2 @@
+class ClassToBeImported(object):
+    pass

python/helpers/pydev/tests_pydevd_python/_debugger_case_qthread4.py

@@ -0,0 +1,37 @@
+try:
+    from PySide import QtCore
+except:
+    from PyQt4 import QtCore
+
+class TestObject(QtCore.QObject):
+    """
+    Test class providing some non-argument signal
+    """
+
+    try:
+        testSignal = QtCore.Signal()  # @UndefinedVariable
+    except:
+        testSignal = QtCore.pyqtSignal()  # @UndefinedVariable
+
+
+class TestThread(QtCore.QThread):
+
+    def run(self):
+        QtCore.QThread.sleep(2)
+        print('Done sleeping')
+
+def on_start():
+    print('On start called')
+
+app = QtCore.QCoreApplication([])
+some_thread = TestThread()
+some_object = TestObject()
+
+# connect QThread.started to the signal
+some_thread.started.connect(some_object.testSignal)
+some_object.testSignal.connect(on_start)
+some_thread.finished.connect(app.quit)
+
+some_thread.start()
+app.exec_()
+print('TEST SUCEEDED!')

python/helpers/pydev/tests_pydevd_python/_debugger_case_remote.py

@@ -0,0 +1,14 @@
+if __name__ == '__main__':
+    import os
+    import sys
+    root_dirname = os.path.dirname(os.path.dirname(__file__))
+    
+    if root_dirname not in sys.path:
+        sys.path.append(root_dirname)
+        
+    import pydevd
+    print('before pydevd.settrace')
+    pydevd.settrace(port=8787)
+    print('after pydevd.settrace')
+    print('TEST SUCEEDED!')
+    

python/helpers/pydev/tests_pydevd_python/_debugger_case_remote_1.py

@@ -0,0 +1,25 @@
+if __name__ == '__main__':
+    import subprocess
+    import sys
+    import os
+    import _debugger_case_remote_2
+    root_dirname = os.path.dirname(os.path.dirname(__file__))
+    
+    if root_dirname not in sys.path:
+        sys.path.append(root_dirname)
+        
+    import pydevd
+    
+    print('before pydevd.settrace')
+    sys.stdout.flush()
+    pydevd.settrace(port=8787, patch_multiprocessing=True)
+    print('after pydevd.settrace')
+    sys.stdout.flush()
+    f = _debugger_case_remote_2.__file__
+    if f.endswith('.pyc'):
+        f = f[:-1]
+    print('before call')
+    sys.stdout.flush()
+    subprocess.check_call([sys.executable, '-u', f])
+    print('after call')
+    sys.stdout.flush()

python/helpers/pydev/tests_pydevd_python/_debugger_case_remote_2.py

@@ -0,0 +1,11 @@
+if __name__ == '__main__':
+    print('Run as main: %s' % (__file__,))
+    import sys
+    sys.stdout.flush()
+    import pydevd
+    # Just check that we're already connected
+    assert pydevd.GetGlobalDebugger() is not None
+    print('finish')
+    sys.stdout.flush()
+    print('TEST SUCEEDED!')
+    sys.stdout.flush()

python/helpers/pydev/tests_pydevd_python/_performance_2.py

@@ -0,0 +1,26 @@
+import time
+start_time = time.time()
+
+try:
+    xrange  # @UndefinedVariable
+except NameError:
+    xrange = range
+    
+from itertools import groupby
+from random import randrange
+
+letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+
+# create an array of random strings of 40 characters each
+l = sorted([''.join([letters[randrange(0, 26)] for _ in range(40)]) for _ in xrange(10000)])
+
+# group by the first two characters
+g = {k: list(v) for k, v in groupby(l, lambda x: x[:2])}
+
+print(len(g.get('AA')))
+
+if False:
+    pass  # Breakpoint here
+
+print('TotalTime>>%s<<' % (time.time()-start_time,))
+print('TEST SUCEEDED')

python/helpers/pydev/tests_pydevd_python/_performance_3.py

@@ -0,0 +1,20 @@
+import time
+start_time = time.time()
+
+try:
+    xrange  # @UndefinedVariable
+except NameError:
+    xrange = range
+
+# do some busy work in parallel
+print("Started main task")
+x = 0
+for i in xrange(1000000):
+    x += 1
+print("Completed main task")
+
+if False:
+    pass  # Breakpoint here
+
+print('TotalTime>>%s<<' % (time.time()-start_time,))
+print('TEST SUCEEDED')

python/helpers/pydev/tests_pydevd_python/debugger_unittest.py

@@ -103,12 +103,13 @@ class ReaderThread(threading.Thread):
         self.sock = sock
         self.last_received = ''
         self.all_received = []
+        self._kill = False
 
     def run(self):
         last_printed = None
         try:
             buf = ''
-            while True:
+            while not self._kill:
                 l = self.sock.recv(1024)
                 if IS_PY3K:
                     l = l.decode('utf-8')
@@ -127,9 +128,13 @@ class ReaderThread(threading.Thread):
                             print('Test Reader Thread Received %s' % last_printed)
         except:
             pass  # ok, finished it
+        finally:
+            del self.all_received[:]
 
     def do_kill(self):
-        self.sock.close()
+        self._kill = True
+        if hasattr(self, 'sock'):
+            self.sock.close()
 
 
 class DebuggerRunner(object):
@@ -145,7 +150,7 @@ class DebuggerRunner(object):
         port = int(writer_thread.port)
 
         localhost = pydev_localhost.get_localhost()
-        return args + [
+        ret = args + [
             writer_thread.get_pydevd_file(),
             '--DEBUG_RECORD_SOCKET_READS',
             '--qt-support',
@@ -153,25 +158,40 @@ class DebuggerRunner(object):
             localhost,
             '--port',
             str(port),
-            '--file',
-        ] + writer_thread.get_command_line_args()
-        return args
+        ]
+
+        if writer_thread.IS_MODULE:
+            ret += ['--module']
+
+        ret = ret + ['--file'] + writer_thread.get_command_line_args()
+        return ret
 
     def check_case(self, writer_thread_class):
         writer_thread = writer_thread_class()
-        writer_thread.start()
-        while not hasattr(writer_thread, 'port'):
-            time.sleep(.01)
-        self.writer_thread = writer_thread
+        try:
+            writer_thread.start()
+            for _i in xrange(40000):
+                if hasattr(writer_thread, 'port'):
+                    break
+                time.sleep(.01)
+            self.writer_thread = writer_thread
 
-        args = self.get_command_line()
+            args = self.get_command_line()
 
-        args = self.add_command_line_args(args)
+            args = self.add_command_line_args(args)
 
-        if SHOW_OTHER_DEBUG_INFO:
-            print('executing', ' '.join(args))
+            if SHOW_OTHER_DEBUG_INFO:
+                print('executing', ' '.join(args))
 
-        return self.run_process(args, writer_thread)
+            ret = self.run_process(args, writer_thread)
+        finally:
+            writer_thread.do_kill()
+            writer_thread.log = []
+
+        stdout = ret['stdout']
+        stderr = ret['stderr']
+        writer_thread.additional_output_checks(''.join(stdout), ''.join(stderr))
+        return ret
 
     def create_process(self, args, writer_thread):
         process = subprocess.Popen(
@@ -187,78 +207,88 @@ class DebuggerRunner(object):
         process = self.create_process(args, writer_thread)
         stdout = []
         stderr = []
+        finish = [False]
 
-        def read(stream, buffer):
-            for line in stream.readlines():
-                if IS_PY3K:
-                    line = line.decode('utf-8')
+        try:
+            def read(stream, buffer):
+                for line in stream.readlines():
+                    if finish[0]:
+                        return
+                    if IS_PY3K:
+                        line = line.decode('utf-8')
 
-                if SHOW_STDOUT:
-                    sys.stdout.write('stdout: %s' % (line,))
-                buffer.append(line)
+                    if SHOW_STDOUT:
+                        sys.stdout.write('stdout: %s' % (line,))
+                    buffer.append(line)
 
-        start_new_thread(read, (process.stdout, stdout))
+            start_new_thread(read, (process.stdout, stdout))
 
 
-        if SHOW_OTHER_DEBUG_INFO:
-            print('Both processes started')
+            if SHOW_OTHER_DEBUG_INFO:
+                print('Both processes started')
 
-        # polls can fail (because the process may finish and the thread still not -- so, we give it some more chances to
-        # finish successfully).
-        check = 0
-        while True:
-            if process.poll() is not None:
-                break
-            else:
-                if writer_thread is not None:
-                    if not writer_thread.isAlive():
-                        if writer_thread.FORCE_KILL_PROCESS_WHEN_FINISHED_OK:
-                            process.kill()
-                            continue
+            # polls can fail (because the process may finish and the thread still not -- so, we give it some more chances to
+            # finish successfully).
+            check = 0
+            while True:
+                if process.poll() is not None:
+                    break
+                else:
+                    if writer_thread is not None:
+                        if not writer_thread.isAlive():
+                            if writer_thread.FORCE_KILL_PROCESS_WHEN_FINISHED_OK:
+                                process.kill()
+                                continue
 
+                            check += 1
+                            if check == 20:
+                                print('Warning: writer thread exited and process still did not.')
+                            if check == 100:
+                                process.kill()
+                                time.sleep(.2)
+                                self.fail_with_message(
+                                    "The other process should've exited but still didn't (timeout for process to exit).",
+                                    stdout, stderr, writer_thread
+                                )
+                time.sleep(.2)
+
+
+            if writer_thread is not None:
+                if not writer_thread.FORCE_KILL_PROCESS_WHEN_FINISHED_OK:
+                    poll = process.poll()
+                    if poll < 0:
+                        self.fail_with_message(
+                            "The other process exited with error code: " + str(poll), stdout, stderr, writer_thread)
+
+
+                    if stdout is None:
+                        self.fail_with_message(
+                            "The other process may still be running -- and didn't give any output.", stdout, stderr, writer_thread)
+
+                    check = 0
+                    while 'TEST SUCEEDED' not in ''.join(stdout):
                         check += 1
-                        if check == 20:
-                            print('Warning: writer thread exited and process still did not.')
-                        if check == 100:
-                            process.kill()
-                            time.sleep(.2)
-                            self.fail_with_message(
-                                "The other process should've exited but still didn't (timeout for process to exit).",
-                                stdout, stderr, writer_thread
-                            )
-            time.sleep(.2)
+                        if check == 50:
+                            self.fail_with_message("TEST SUCEEDED not found in stdout.", stdout, stderr, writer_thread)
+                        time.sleep(.1)
 
+                for _i in xrange(100):
+                    if not writer_thread.finished_ok:
+                        time.sleep(.1)
 
-        if writer_thread is not None:
-            if not writer_thread.FORCE_KILL_PROCESS_WHEN_FINISHED_OK:
-                poll = process.poll()
-                if poll < 0:
-                    self.fail_with_message(
-                        "The other process exited with error code: " + str(poll), stdout, stderr, writer_thread)
-
-
-                if stdout is None:
-                    self.fail_with_message(
-                        "The other process may still be running -- and didn't give any output.", stdout, stderr, writer_thread)
-
-                if 'TEST SUCEEDED' not in ''.join(stdout):
-                    self.fail_with_message("TEST SUCEEDED not found in stdout.", stdout, stderr, writer_thread)
-
-            for i in xrange(100):
                 if not writer_thread.finished_ok:
-                    time.sleep(.1)
-
-            if not writer_thread.finished_ok:
-                self.fail_with_message(
-                    "The thread that was doing the tests didn't finish successfully.", stdout, stderr, writer_thread)
+                    self.fail_with_message(
+                        "The thread that was doing the tests didn't finish successfully.", stdout, stderr, writer_thread)
+        finally:
+            finish[0] = True
 
         return {'stdout':stdout, 'stderr':stderr}
 
     def fail_with_message(self, msg, stdout, stderr, writerThread):
         raise AssertionError(msg+
-            "\nStdout: \n"+'\n'.join(stdout)+
-            "\nStderr:"+'\n'.join(stderr)+
-            "\nLog:\n"+'\n'.join(getattr(writerThread, 'log', [])))
+                             "\n\n===========================\nStdout: \n"+''.join(stdout)+
+                             "\n\n===========================\nStderr:"+''.join(stderr)+
+                             "\n\n===========================\nLog:\n"+'\n'.join(getattr(writerThread, 'log', [])))
 
 
 
@@ -268,6 +298,7 @@ class DebuggerRunner(object):
 class AbstractWriterThread(threading.Thread):
 
     FORCE_KILL_PROCESS_WHEN_FINISHED_OK = False
+    IS_MODULE = False
 
     def __init__(self):
         threading.Thread.__init__(self)
@@ -276,6 +307,9 @@ class AbstractWriterThread(threading.Thread):
         self._next_breakpoint_id = 0
         self.log = []
 
+    def additional_output_checks(self, stdout, stderr):
+        pass
+
     def get_environ(self):
         return None
 
@@ -291,12 +325,17 @@ class AbstractWriterThread(threading.Thread):
         return [self.TEST_FILE]
 
     def do_kill(self):
+        if hasattr(self, 'server_socket'):
+            self.server_socket.close()
+
         if hasattr(self, 'reader_thread'):
             # if it's not created, it's not there...
             self.reader_thread.do_kill()
-        self.sock.close()
+        if hasattr(self, 'sock'):
+            self.sock.close()
 
     def write(self, s):
+        self.log.append('write: %s' % (s,))
 
         last = self.reader_thread.last_received
         if SHOW_WRITES_AND_READS:
@@ -313,16 +352,17 @@ class AbstractWriterThread(threading.Thread):
             time.sleep(0.1)
 
 
-    def start_socket(self):
+    def start_socket(self, port=0):
         if SHOW_WRITES_AND_READS:
             print('start_socket')
 
         s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        s.bind(('', 0))
+        s.bind(('', port))
         self.port = s.getsockname()[1]
         s.listen(1)
         if SHOW_WRITES_AND_READS:
             print('Waiting in socket.accept()')
+        self.server_socket = s
         newSock, addr = s.accept()
         if SHOW_WRITES_AND_READS:
             print('Test Writer Thread Socket:', newSock, addr)
@@ -359,7 +399,7 @@ class AbstractWriterThread(threading.Thread):
         thread_id = splitted[3]
         return thread_id
 
-    def wait_for_breakpoint_hit(self, reason='111', get_line=False):
+    def wait_for_breakpoint_hit(self, reason='111', get_line=False, get_name=False):
         '''
             108 is over
             109 is return
@@ -375,22 +415,29 @@ class AbstractWriterThread(threading.Thread):
             last = self.reader_thread.last_received
             if i >= 10:
                 raise AssertionError('After %s seconds, a break with reason: %s was not hit. Found: %s' % \
-                    (i, reason, last))
+                                     (i, reason, last))
 
-        # we have something like <xml><thread id="12152656" stop_reason="111"><frame id="12453120" ...
+        # we have something like <xml><thread id="12152656" stop_reason="111"><frame id="12453120" name="encode" ...
         splitted = last.split('"')
         thread_id = splitted[1]
         frameId = splitted[7]
+        name = splitted[9]
         if get_line:
-            self.log.append('End(0): wait_for_breakpoint_hit')
+            self.log.append('End(0): wait_for_breakpoint_hit: %s' % (last,))
             try:
-                return thread_id, frameId, int(splitted[13])
+                if not get_name:
+                    return thread_id, frameId, int(splitted[13])
+                else:
+                    return thread_id, frameId, int(splitted[13]), name
             except:
                 raise AssertionError('Error with: %s, %s, %s.\nLast: %s.\n\nAll: %s\n\nSplitted: %s' % (
                     thread_id, frameId, splitted[13], last, '\n'.join(self.reader_thread.all_received), splitted))
 
-        self.log.append('End(1): wait_for_breakpoint_hit')
-        return thread_id, frameId
+        self.log.append('End(1): wait_for_breakpoint_hit: %s' % (last,))
+        if not get_name:
+            return thread_id, frameId
+        else:
+            return thread_id, frameId, name
 
     def wait_for_custom_operation(self, expected):
         i = 0
@@ -401,7 +448,7 @@ class AbstractWriterThread(threading.Thread):
             time.sleep(1)
             if i >= 10:
                 raise AssertionError('After %s seconds, the custom operation not received. Last found:\n%s\nExpected (encoded)\n%s' %
-                    (i, self.reader_thread.last_received, expectedEncoded))
+                                     (i, self.reader_thread.last_received, expectedEncoded))
 
         return True
 
@@ -417,7 +464,7 @@ class AbstractWriterThread(threading.Thread):
             time.sleep(1)
             if i >= 10:
                 raise AssertionError('After %s seconds, the vars were not found. Last found:\n%s' %
-                    (i, self.reader_thread.last_received))
+                                     (i, self.reader_thread.last_received))
 
         return True
 
@@ -461,7 +508,7 @@ class AbstractWriterThread(threading.Thread):
             time.sleep(1)
             if i >= 10:
                 raise AssertionError('After %s seconds, %s. Last found:\n%s' %
-                    (i, error_msg, last))
+                                     (i, error_msg, last))
 
         return True
 
@@ -479,7 +526,7 @@ class AbstractWriterThread(threading.Thread):
             time.sleep(1)
             if i >= 10:
                 raise AssertionError('After %s seconds, the vars were not found. Last found:\n%s' %
-                    (i, self.reader_thread.last_received))
+                                     (i, self.reader_thread.last_received))
 
         return True
 
@@ -490,12 +537,15 @@ class AbstractWriterThread(threading.Thread):
     def write_version(self):
         self.write("501\t%s\t1.0\tWINDOWS\tID" % self.next_seq())
 
+    def get_main_filename(self):
+        return self.TEST_FILE
+
     def write_add_breakpoint(self, line, func):
         '''
             @param line: starts at 1
         '''
         breakpoint_id = self.next_breakpoint_id()
-        self.write("111\t%s\t%s\t%s\t%s\t%s\t%s\tNone\tNone" % (self.next_seq(), breakpoint_id, 'python-line', self.TEST_FILE, line, func))
+        self.write("111\t%s\t%s\t%s\t%s\t%s\t%s\tNone\tNone" % (self.next_seq(), breakpoint_id, 'python-line', self.get_main_filename(), line, func))
         self.log.append('write_add_breakpoint: %s line: %s func: %s' % (breakpoint_id, line, func))
         return breakpoint_id
 
@@ -504,7 +554,7 @@ class AbstractWriterThread(threading.Thread):
         self.log.append('write_add_exception_breakpoint: %s' % (exception,))
 
     def write_remove_breakpoint(self, breakpoint_id):
-        self.write("112\t%s\t%s\t%s\t%s" % (self.next_seq(), 'python-line', self.TEST_FILE, breakpoint_id))
+        self.write("112\t%s\t%s\t%s\t%s" % (self.next_seq(), 'python-line', self.get_main_filename(), breakpoint_id))
 
     def write_change_variable(self, thread_id, frame_id, varname, value):
         self.write("117\t%s\t%s\t%s\t%s\t%s\t%s" % (self.next_seq(), thread_id, frame_id, 'FRAME', varname, value))

python/helpers/pydev/tests_pydevd_python/performance_check.py

@@ -82,6 +82,30 @@ class WriterThreadPerformance4(PerformanceWriterThread):
         self.write_make_initial_run()
         self.finished_ok = True
 
+class WriterThreadPerformance5(PerformanceWriterThread):
+
+    TEST_FILE = debugger_unittest._get_debugger_test_file('_performance_2.py')
+    BENCHMARK_NAME = 'global_scope_1_with_breakpoint'
+
+    def run(self):
+        self.start_socket()
+        self.write_add_breakpoint(23, None)
+
+        self.write_make_initial_run()
+        self.finished_ok = True
+
+class WriterThreadPerformance6(PerformanceWriterThread):
+
+    TEST_FILE = debugger_unittest._get_debugger_test_file('_performance_3.py')
+    BENCHMARK_NAME = 'global_scope_2_with_breakpoint'
+
+    def run(self):
+        self.start_socket()
+        self.write_add_breakpoint(17, None)
+
+        self.write_make_initial_run()
+        self.finished_ok = True
+
 
 class CheckDebuggerPerformance(debugger_unittest.DebuggerRunner):
 
@@ -95,13 +119,19 @@ class CheckDebuggerPerformance(debugger_unittest.DebuggerRunner):
         return float(time_taken)
 
     def obtain_results(self, writer_thread_class):
-        time_when_debugged = self._get_time_from_result(self.check_case(writer_thread_class))
+        runs = 5
+        all_times = []
+        for _ in range(runs):
+            all_times.append(self._get_time_from_result(self.check_case(writer_thread_class)))
+            print('partial for: %s: %.3fs' % (writer_thread_class.BENCHMARK_NAME, all_times[-1]))
+        all_times.remove(min(all_times))
+        all_times.remove(max(all_times))
+        time_when_debugged = sum(all_times) / float(len(all_times))
 
         args = self.get_command_line()
         args.append(writer_thread_class.TEST_FILE)
-        regular_time = self._get_time_from_result(self.run_process(args, writer_thread=None))
-        simple_trace_time = self._get_time_from_result(self.run_process(args+['--regular-trace'], writer_thread=None))
-        print(writer_thread_class.BENCHMARK_NAME, time_when_debugged, regular_time, simple_trace_time)
+        # regular_time = self._get_time_from_result(self.run_process(args, writer_thread=None))
+        # simple_trace_time = self._get_time_from_result(self.run_process(args+['--regular-trace'], writer_thread=None))
 
         if 'SPEEDTIN_AUTHORIZATION_KEY' in os.environ:
 
@@ -133,43 +163,62 @@ class CheckDebuggerPerformance(debugger_unittest.DebuggerRunner):
                 commit_id, branch, commit_date = api.git_commit_id_branch_and_date_from_path(pydevd.__file__)
                 api.add_benchmark(benchmark_name)
                 api.add_measurement(
-                        benchmark_name,
-                        value=time_when_debugged,
-                        version=version,
-                        released=False,
-                        branch=branch,
-                        commit_id=commit_id,
-                        commit_date=commit_date,
+                    benchmark_name,
+                    value=time_when_debugged,
+                    version=version,
+                    released=False,
+                    branch=branch,
+                    commit_id=commit_id,
+                    commit_date=commit_date,
                 )
                 api.commit()
 
+        return '%s: %.3fs ' % (writer_thread_class.BENCHMARK_NAME, time_when_debugged)
+
 
     def check_performance1(self):
-        self.obtain_results(WriterThreadPerformance1)
+        return self.obtain_results(WriterThreadPerformance1)
 
     def check_performance2(self):
-        self.obtain_results(WriterThreadPerformance2)
+        return self.obtain_results(WriterThreadPerformance2)
 
     def check_performance3(self):
-        self.obtain_results(WriterThreadPerformance3)
+        return self.obtain_results(WriterThreadPerformance3)
 
     def check_performance4(self):
-        self.obtain_results(WriterThreadPerformance4)
+        return self.obtain_results(WriterThreadPerformance4)
+
+    def check_performance5(self):
+        return self.obtain_results(WriterThreadPerformance5)
+
+    def check_performance6(self):
+        return self.obtain_results(WriterThreadPerformance6)
 
 if __name__ == '__main__':
     debugger_unittest.SHOW_WRITES_AND_READS = False
     debugger_unittest.SHOW_OTHER_DEBUG_INFO = False
     debugger_unittest.SHOW_STDOUT = False
 
+    import time
+    start_time = time.time()
+
+    msgs = []
     for check in (
             # CHECK_BASELINE, -- Checks against the version checked out at X:\PyDev.Debugger.baseline.
             CHECK_REGULAR,
             CHECK_CYTHON
     ):
         PerformanceWriterThread.CHECK = check
-        print('Checking: %s' % (check,))
+        msgs.append('Checking: %s' % (check,))
         check_debugger_performance = CheckDebuggerPerformance()
-        check_debugger_performance.check_performance1()
-        check_debugger_performance.check_performance2()
-        check_debugger_performance.check_performance3()
-        check_debugger_performance.check_performance4()
+        msgs.append(check_debugger_performance.check_performance1())
+        msgs.append(check_debugger_performance.check_performance2())
+        msgs.append(check_debugger_performance.check_performance3())
+        msgs.append(check_debugger_performance.check_performance4())
+        msgs.append(check_debugger_performance.check_performance5())
+        msgs.append(check_debugger_performance.check_performance6())
+
+    for msg in msgs:
+        print(msg)
+
+    print('TotalTime for profile: %.2fs' % (time.time()-start_time,))

python/helpers/pydev/tests_pydevd_python/test_additional_thread_info.py

@@ -20,35 +20,6 @@ except:
 # TestCase
 #=======================================================================================================================
 class TestCase(unittest.TestCase):
-    '''
-        Used for profiling the PyDBAdditionalThreadInfoWithoutCurrentFramesSupport version
-    '''
-
-    def test_met_no_frames_support(self):
-        from _pydevd_bundle.pydevd_additional_thread_info_regular import PyDBAdditionalThreadInfoWithoutCurrentFramesSupport
-        info = PyDBAdditionalThreadInfoWithoutCurrentFramesSupport()
-
-        main_debugger = Null()
-        filename = ''
-        base = ''
-        additional_info = Null()
-        t = Null()
-        frame = Null()
-
-        times = 10
-        for i in range(times):
-            info.create_db_frame((main_debugger, filename, additional_info, t, frame))
-
-        #we haven't kept any reference, so, they must have been garbage-collected already!
-        self.assertEqual(0, len(info.iter_frames(t)))
-
-        kept_frames = []
-        for i in range(times):
-            kept_frames.append(info.create_db_frame((main_debugger, filename, additional_info, t, frame)))
-
-        for i in range(times):
-            self.assertEqual(times, len(info.iter_frames(t)))
-
 
     def test_start_new_thread(self):
         pydev_monkey.patch_thread_modules()

python/helpers/pydev/tests_pydevd_python/test_debugger.py

@@ -5,7 +5,7 @@
 
     Note that it's a python script but it'll spawn a process to run as jython, ironpython and as python.
 '''
-from tests_python.debugger_unittest import get_free_port
+from tests_pydevd_python.debugger_unittest import get_free_port
 import threading
 
 
@@ -17,12 +17,16 @@ IRONPYTHON_EXE = None
 JYTHON_JAR_LOCATION = None
 JAVA_LOCATION = None
 
+try:
+    xrange
+except:
+    xrange = range
 
 import unittest
 import os
 import sys
 import time
-from tests_python import debugger_unittest
+from tests_pydevd_python import debugger_unittest
 
 TEST_DJANGO = False
 if sys.version_info[:2] == (2, 7):
@@ -239,10 +243,13 @@ class WriterThreadCase17a(debugger_unittest.AbstractWriterThread):
         self.write_make_initial_run()
 
         thread_id, frame_id, line = self.wait_for_breakpoint_hit('111', True)
+        assert line == 2, 'Expected return to be in line 2, was: %s' % line
 
         self.write_step_in(thread_id)
-        thread_id, frame_id, line = self.wait_for_breakpoint_hit('107', True)
+        thread_id, frame_id, line, name = self.wait_for_breakpoint_hit('107', get_line=True, get_name=True)
+
         # Should Skip step into properties setter
+        assert name == 'm3'
         assert line == 10, 'Expected return to be in line 10, was: %s' % line
         self.write_run_thread(thread_id)
 
@@ -284,15 +291,17 @@ class WriterThreadCase16(debugger_unittest.AbstractWriterThread):
         self.wait_for_var('<var name="%27size%27')
 
         self.write_get_variable(thread_id, frame_id, 'bigarray')
+        # isContainer could be true on some numpy versions, so, we only check for the var begin.
         self.wait_for_var([
-            '<var name="min" type="int64" qualifier="numpy" value="int64%253A 0" />',
-            '<var name="min" type="int64" qualifier="numpy" value="int64%3A 0" />',
-            '<var name="size" type="int" qualifier="{}" value="int%3A 100000" />'.format(builtin_qualifier),
+            '<var name="min" type="int64" qualifier="numpy" value="int64%253A 0"',
+            '<var name="min" type="int64" qualifier="numpy" value="int64%3A 0"',
+            '<var name="size" type="int" qualifier="{}" value="int%3A 100000"'.format(builtin_qualifier),
         ])
         self.wait_for_var([
-            '<var name="max" type="int64" qualifier="numpy" value="int64%253A 99999" />',
-            '<var name="max" type="int32" qualifier="numpy" value="int32%253A 99999" />',
-            '<var name="max" type="int64" qualifier="numpy" value="int64%3A 99999"'
+            '<var name="max" type="int64" qualifier="numpy" value="int64%253A 99999"',
+            '<var name="max" type="int32" qualifier="numpy" value="int32%253A 99999"',
+            '<var name="max" type="int64" qualifier="numpy" value="int64%3A 99999"',
+            '<var name="max" type="int32" qualifier="numpy" value="int32%253A 99999"',
         ])
         self.wait_for_var('<var name="shape" type="tuple"')
         self.wait_for_var('<var name="dtype" type="dtype"')
@@ -306,10 +315,14 @@ class WriterThreadCase16(debugger_unittest.AbstractWriterThread):
         self.wait_for_var([
             '<var name="min" type="str" qualifier={} value="str%253A ndarray too big%252C calculating min would slow down debugging" />'.format(builtin_qualifier),
             '<var name="min" type="str" qualifier={} value="str%3A ndarray too big%252C calculating min would slow down debugging" />'.format(builtin_qualifier),
+            '<var name="min" type="str" qualifier="{}" value="str%253A ndarray too big%252C calculating min would slow down debugging" />'.format(builtin_qualifier),
+            '<var name="min" type="str" qualifier="{}" value="str%3A ndarray too big%252C calculating min would slow down debugging" />'.format(builtin_qualifier),
         ])
         self.wait_for_var([
             '<var name="max" type="str" qualifier={} value="str%253A ndarray too big%252C calculating max would slow down debugging" />'.format(builtin_qualifier),
             '<var name="max" type="str" qualifier={} value="str%3A ndarray too big%252C calculating max would slow down debugging" />'.format(builtin_qualifier),
+            '<var name="max" type="str" qualifier="{}" value="str%253A ndarray too big%252C calculating max would slow down debugging" />'.format(builtin_qualifier),
+            '<var name="max" type="str" qualifier="{}" value="str%3A ndarray too big%252C calculating max would slow down debugging" />'.format(builtin_qualifier),
         ])
         self.wait_for_var('<var name="shape" type="tuple"')
         self.wait_for_var('<var name="dtype" type="dtype"')
@@ -487,7 +500,9 @@ class WriterThreadCase11(debugger_unittest.AbstractWriterThread):
         self.write_add_breakpoint(2, 'Method1')
         self.write_make_initial_run()
 
-        thread_id, frame_id = self.wait_for_breakpoint_hit('111')
+        thread_id, frame_id, line = self.wait_for_breakpoint_hit('111', True)
+
+        assert line == 2, 'Expected return to be in line 2, was: %s' % line
 
         self.write_step_over(thread_id)
 
@@ -877,6 +892,37 @@ class WriterThreadCaseQThread3(debugger_unittest.AbstractWriterThread):
         self.log.append('Marking finished ok.')
         self.finished_ok = True
 
+#=======================================================================================================================
+# WriterThreadCaseQThread4
+#=======================================================================================================================
+class WriterThreadCaseQThread4(debugger_unittest.AbstractWriterThread):
+
+    TEST_FILE = debugger_unittest._get_debugger_test_file('_debugger_case_qthread4.py')
+
+    def run(self):
+        self.start_socket()
+        breakpoint_id = self.write_add_breakpoint(24, 'on_start')
+        self.write_make_initial_run()
+
+        thread_id, frame_id = self.wait_for_breakpoint_hit()
+
+        self.write_remove_breakpoint(breakpoint_id)
+        self.write_run_thread(thread_id)
+
+        self.log.append('Checking sequence. Found: %s' % (self._sequence))
+        assert 9 == self._sequence, 'Expected 9. Had: %s' % self._sequence
+
+        self.log.append('Marking finished ok.')
+        self.finished_ok = True
+
+    def additional_output_checks(self, stdout, stderr):
+        if 'On start called' not in stdout:
+            raise AssertionError('Expected "On start called" to be in stdout:\n%s' % (stdout,))
+        if 'Done sleeping' not in stdout:
+            raise AssertionError('Expected "Done sleeping" to be in stdout:\n%s' % (stdout,))
+        if 'native Qt signal is not callable' in stderr:
+            raise AssertionError('Did not expect "native Qt signal is not callable" to be in stderr:\n%s' % (stderr,))
+
 #=======================================================================================================================
 # WriterThreadCase1
 #=======================================================================================================================
@@ -918,6 +964,162 @@ class WriterThreadCase1(debugger_unittest.AbstractWriterThread):
 
         self.finished_ok = True
 
+#=======================================================================================================================
+# WriterThreadCaseMSwitch
+#=======================================================================================================================
+class WriterThreadCaseMSwitch(debugger_unittest.AbstractWriterThread):
+
+    TEST_FILE = 'tests_pydevd_python._debugger_case_m_switch'
+    IS_MODULE = True
+
+    def get_environ(self):
+        env = os.environ.copy()
+        curr_pythonpath = env.get('PYTHONPATH', '')
+
+        root_dirname = os.path.dirname(os.path.dirname(__file__))
+
+        curr_pythonpath += root_dirname + os.pathsep
+        env['PYTHONPATH'] = curr_pythonpath
+        return env
+
+    def get_main_filename(self):
+        return debugger_unittest._get_debugger_test_file('_debugger_case_m_switch.py')
+
+    def run(self):
+        self.start_socket()
+
+        self.log.append('writing add breakpoint')
+        breakpoint_id = self.write_add_breakpoint(1, None)
+
+        self.log.append('making initial run')
+        self.write_make_initial_run()
+
+        self.log.append('waiting for breakpoint hit')
+        thread_id, frame_id = self.wait_for_breakpoint_hit()
+
+        self.write_remove_breakpoint(breakpoint_id)
+
+        self.log.append('run thread')
+        self.write_run_thread(thread_id)
+
+        self.log.append('asserting')
+        try:
+            assert 9 == self._sequence, 'Expected 9. Had: %s' % self._sequence
+        except:
+            self.log.append('assert failed!')
+            raise
+        self.log.append('asserted')
+
+        self.finished_ok = True
+
+#=======================================================================================================================
+# WriterThreadCaseRemoteDebugger
+#=======================================================================================================================
+class WriterThreadCaseRemoteDebugger(debugger_unittest.AbstractWriterThread):
+
+    TEST_FILE = debugger_unittest._get_debugger_test_file('_debugger_case_remote.py')
+
+    def run(self):
+        self.start_socket(8787)
+
+        self.log.append('making initial run')
+        self.write_make_initial_run()
+
+        self.log.append('waiting for breakpoint hit')
+        thread_id, frame_id = self.wait_for_breakpoint_hit('105')
+
+        self.log.append('run thread')
+        self.write_run_thread(thread_id)
+
+        self.log.append('asserting')
+        try:
+            assert 5 == self._sequence, 'Expected 5. Had: %s' % self._sequence
+        except:
+            self.log.append('assert failed!')
+            raise
+        self.log.append('asserted')
+
+        self.finished_ok = True
+
+#=======================================================================================================================
+# _SecondaryMultiProcProcessWriterThread
+#=======================================================================================================================
+class _SecondaryMultiProcProcessWriterThread(debugger_unittest.AbstractWriterThread):
+
+    FORCE_KILL_PROCESS_WHEN_FINISHED_OK = True
+
+    def __init__(self, server_socket):
+        debugger_unittest.AbstractWriterThread.__init__(self)
+        self.server_socket = server_socket
+
+    def run(self):
+        print('waiting for second process')
+        self.sock, addr = self.server_socket.accept()
+        print('accepted second process')
+
+        from tests_pydevd_python.debugger_unittest import ReaderThread
+        self.reader_thread = ReaderThread(self.sock)
+        self.reader_thread.start()
+
+        self._sequence = -1
+        # initial command is always the version
+        self.write_version()
+        self.log.append('start_socket')
+        self.write_make_initial_run()
+        time.sleep(.5)
+        self.finished_ok = True
+
+#=======================================================================================================================
+# WriterThreadCaseRemoteDebuggerMultiProc
+#=======================================================================================================================
+class WriterThreadCaseRemoteDebuggerMultiProc(debugger_unittest.AbstractWriterThread):
+
+    # It seems sometimes it becomes flaky on the ci because the process outlives the writer thread...
+    # As we're only interested in knowing if a second connection was received, just kill the related
+    # process.
+    FORCE_KILL_PROCESS_WHEN_FINISHED_OK = True
+
+    TEST_FILE = debugger_unittest._get_debugger_test_file('_debugger_case_remote_1.py')
+
+    def run(self):
+        self.start_socket(8787)
+
+        self.log.append('making initial run')
+        self.write_make_initial_run()
+
+        self.log.append('waiting for breakpoint hit')
+        thread_id, frame_id = self.wait_for_breakpoint_hit('105')
+
+        self.secondary_multi_proc_process_writer_thread  = secondary_multi_proc_process_writer_thread = \
+            _SecondaryMultiProcProcessWriterThread(self.server_socket)
+        secondary_multi_proc_process_writer_thread.start()
+
+        self.log.append('run thread')
+        self.write_run_thread(thread_id)
+
+        for _i in xrange(400):
+            if secondary_multi_proc_process_writer_thread.finished_ok:
+                break
+            time.sleep(.1)
+        else:
+            self.log.append('Secondary process not finished ok!')
+            raise AssertionError('Secondary process not finished ok!')
+
+        self.log.append('Secondary process finished!')
+        try:
+            assert 5 == self._sequence, 'Expected 5. Had: %s' % self._sequence
+        except:
+            self.log.append('assert failed!')
+            raise
+        self.log.append('asserted')
+
+        self.finished_ok = True
+
+    def do_kill(self):
+        debugger_unittest.AbstractWriterThread.do_kill(self)
+        if hasattr(self, 'secondary_multi_proc_process_writer_thread'):
+            self.secondary_multi_proc_process_writer_thread.do_kill()
+
 #=======================================================================================================================
 # DebuggerBase
 #=======================================================================================================================
@@ -1016,6 +1218,29 @@ class DebuggerBase(debugger_unittest.DebuggerRunner):
         if self._has_qt():
             self.check_case(WriterThreadCaseQThread3)
 
+    def test_case_qthread4(self):
+        if self._has_qt():
+            self.check_case(WriterThreadCaseQThread4)
+
+    def test_m_switch(self):
+        self.check_case(WriterThreadCaseMSwitch)
+
+
+
+# class TestPythonRemoteDebugger(unittest.TestCase, debugger_unittest.DebuggerRunner):
+#
+#     def get_command_line(self):
+#         return [PYTHON_EXE, '-u']
+#
+#     def add_command_line_args(self, args):
+#         return args + [self.writer_thread.TEST_FILE]
+#
+#     def test_remote_debugger(self):
+#         self.check_case(WriterThreadCaseRemoteDebugger)
+#
+#     def test_remote_debugger2(self):
+#         self.check_case(WriterThreadCaseRemoteDebuggerMultiProc)
+
 
 class TestPython(unittest.TestCase, DebuggerBase):
     def get_command_line(self):
@@ -1154,7 +1379,7 @@ if __name__ == '__main__':
         #
         #         suite.addTests(unittest.makeSuite(TestIronPython))
         #
-        suite.addTests(unittest.makeSuite(TestPython))
+        #         suite.addTests(unittest.makeSuite(TestPython))
 
 
 
@@ -1178,6 +1403,6 @@ if __name__ == '__main__':
         #         unittest.TextTestRunner(verbosity=3).run(suite)
         #     suite.addTest(TestPython('test_case_17'))
         #     suite.addTest(TestPython('test_case_18'))
-        #     suite.addTest(TestPython('test_case_19'))
+        suite.addTest(TestPython('test_case_qthread4'))
 
         unittest.TextTestRunner(verbosity=3).run(suite)

python/helpers/pydev/tests_pydevd_python/test_process_command_line.py

@@ -0,0 +1,28 @@
+import unittest
+
+class Test(unittest.TestCase):
+    
+    def testProcessCommandLine(self):
+        from _pydevd_bundle.pydevd_command_line_handling import process_command_line, setup_to_argv
+        setup = process_command_line(['pydevd.py', '--port', '1', '--save-threading'])
+        assert setup['save-threading']
+        assert setup['port'] == 1
+        assert not setup['qt-support']
+        
+        argv = setup_to_argv(setup)
+        assert argv[0].endswith('pydevd.py')
+        argv = argv[1:]
+        assert argv == ['--port', '1', '--save-threading']
+        
+    def testProcessCommandLine2(self):
+        from _pydevd_bundle.pydevd_command_line_handling import process_command_line, setup_to_argv
+        setup = process_command_line(['pydevd.py', '--port', '1', '--qt-support=auto'])
+        assert setup['qt-support'] == 'auto'
+        
+        setup = process_command_line(['pydevd.py', '--port', '1', '--qt-support'])
+        assert setup['qt-support'] == 'auto'
+        
+        setup = process_command_line(['pydevd.py', '--port', '1', '--qt-support=pyqt4'])
+        assert setup['qt-support'] == 'pyqt4'
+        
+        self.assertRaises(ValueError, process_command_line, ['pydevd.py', '--port', '1', '--qt-support=wrong'])

python/helpers/pydev/tests_pydevd_python/test_pydev_monkey.py

@@ -19,14 +19,13 @@ class TestCase(unittest.TestCase):
         try:
             SetupHolder.setup = {'client':'127.0.0.1', 'port': '0'}
             check='''C:\\bin\\python.exe -u -c connect(\\"127.0.0.1\\")'''
-            sys.original_argv = []
             debug_command = (
-                'import sys; '
-                'sys.path.append(r\'%s\'); '
-                'import pydevd; pydevd.settrace(host=\'127.0.0.1\', port=0, suspend=False, '
-                'trace_only_current_thread=False, patch_multiprocessing=True); '
-                'sys.original_argv = []; '
-                'connect("127.0.0.1")') % pydev_src_dir
+                                'import sys; '
+                                'sys.path.append(r\'%s\'); '
+                                'import pydevd; pydevd.settrace(host=\'127.0.0.1\', port=0, suspend=False, '
+                                'trace_only_current_thread=False, patch_multiprocessing=True); '
+                                ''
+                                'connect("127.0.0.1")') % pydev_src_dir
             if sys.platform == "win32":
                 debug_command = debug_command.replace('"', '\\"')
                 debug_command = '"%s"' % debug_command
@@ -45,12 +44,11 @@ class TestCase(unittest.TestCase):
         try:
             SetupHolder.setup = {'client':'127.0.0.1', 'port': '0'}
             check=['C:\\bin\\python.exe', '-u', '-c', 'connect("127.0.0.1")']
-            sys.original_argv = []
             debug_command = (
-                'import sys; sys.path.append(r\'%s\'); import pydevd; '
-                'pydevd.settrace(host=\'127.0.0.1\', port=0, suspend=False, trace_only_current_thread=False, patch_multiprocessing=True); '
-                'sys.original_argv = []; '
-                'connect("127.0.0.1")') % pydev_src_dir
+                                'import sys; sys.path.append(r\'%s\'); import pydevd; '
+                                'pydevd.settrace(host=\'127.0.0.1\', port=0, suspend=False, trace_only_current_thread=False, patch_multiprocessing=True); '
+                                ''
+                                'connect("127.0.0.1")') % pydev_src_dir
             if sys.platform == "win32":
                 debug_command = debug_command.replace('"', '\\"')
                 debug_command = '"%s"' % debug_command
@@ -68,14 +66,19 @@ class TestCase(unittest.TestCase):
         original = SetupHolder.setup
 
         try:
-            SetupHolder.setup = {'client':'127.0.0.1', 'port': '0'}
+            SetupHolder.setup = {'client':'127.0.0.1', 'port': '0', 'multiprocess': True}
             check=['C:\\bin\\python.exe', '-m', 'test']
-            sys.original_argv = ['pydevd', '--multiprocess']
+            from _pydevd_bundle.pydevd_command_line_handling import get_pydevd_file
             self.assertEqual(pydev_monkey.patch_args(check), [
                 'C:\\bin\\python.exe',
-                'pydevd',
+                get_pydevd_file(),
                 '--module',
+                '--port',
+                '0',
+                '--client',
+                '127.0.0.1',
                 '--multiprocess',
+                '--file',
                 'test',
             ])
         finally:
@@ -87,9 +90,16 @@ class TestCase(unittest.TestCase):
         try:
             SetupHolder.setup = {'client':'127.0.0.1', 'port': '0'}
             check=['C:\\bin\\python.exe', 'connect(\\"127.0.0.1\\")']
-            sys.original_argv = ['my', 'original', 'argv']
+            from _pydevd_bundle.pydevd_command_line_handling import get_pydevd_file
             self.assertEqual(pydev_monkey.patch_args(check), [
-                'C:\\bin\\python.exe', 'my', 'original', 'argv', 'connect(\\"127.0.0.1\\")'])
+                'C:\\bin\\python.exe',
+                get_pydevd_file(),
+                '--port',
+                '0',
+                '--client',
+                '127.0.0.1',
+                '--file',
+                'connect(\\"127.0.0.1\\")'])
         finally:
             SetupHolder.setup = original
 
@@ -99,7 +109,6 @@ class TestCase(unittest.TestCase):
         try:
             SetupHolder.setup = {'client':'127.0.0.1', 'port': '0'}
             check=['C:\\bin\\python.exe', 'pydevd.py', 'connect(\\"127.0.0.1\\")', 'bar']
-            sys.original_argv = ['my', 'original', 'argv']
 
             self.assertEqual(pydev_monkey.patch_args(check), [
                 'C:\\bin\\python.exe', 'pydevd.py', 'connect(\\"127.0.0.1\\")', 'bar'])
@@ -108,18 +117,19 @@ class TestCase(unittest.TestCase):
 
     def test_monkey_patch_args_no_indc_without_pydevd(self):
         original = SetupHolder.setup
+        from _pydevd_bundle.pydevd_command_line_handling import get_pydevd_file
 
         try:
             SetupHolder.setup = {'client':'127.0.0.1', 'port': '0'}
             check=['C:\\bin\\python.exe', 'target.py', 'connect(\\"127.0.0.1\\")', 'bar']
-            sys.original_argv = ['pydevd.py', '--a=1', 'b', '--c=2', '--file', 'ignore_this.py']
 
             self.assertEqual(pydev_monkey.patch_args(check), [
                 'C:\\bin\\python.exe',
-                'pydevd.py',
-                '--a=1',
-                'b',
-                '--c=2',
+                get_pydevd_file(),
+                '--port',
+                '0',
+                '--client',
+                '127.0.0.1',
                 '--file',
                 'target.py',
                 'connect(\\"127.0.0.1\\")',

python/testData/debug/test_runtoline.py

@@ -1,7 +1,12 @@
 from time import sleep
 
+
+def foo(x):
+    print(x)
+
 x = 0
 print(x)
+foo(x)
 while x<2:
     x+=1
     print(x)

python/testSrc/com/jetbrains/env/python/PythonDebuggerTest.java

@@ -379,15 +379,15 @@ public class PythonDebuggerTest extends PyEnvTestCase {
     runPythonTest(new PyDebuggerTask("/debug", "test_runtoline.py") {
       @Override
       public void before() throws Exception {
-        toggleBreakpoint(getFilePath(getScriptName()), 3);
-        toggleBreakpoint(getFilePath(getScriptName()), 9);
+        toggleBreakpoint(getFilePath(getScriptName()), 7);
+        toggleBreakpoint(getFilePath(getScriptName()), 14);
       }
 
       @Override
       public void testing() throws Exception {
         waitForPause();
         eval("x").hasValue("0");
-        runToLine(6);
+        runToLine(11);
         eval("x").hasValue("1");
         resume();
         waitForPause();
@@ -961,8 +961,7 @@ public class PythonDebuggerTest extends PyEnvTestCase {
     runPythonTest(new PyDebuggerTask("/debug", "test_return_values.py") {
       @Override
       public void before() throws Exception {
-        toggleBreakpoint(getScriptName(), 7);
-        toggleBreakpoint(getScriptName(), 11);
+        toggleBreakpoint(getScriptName(), 2);
         final PyDebuggerSettings debuggerSettings = PyDebuggerSettings.getInstance();
         debuggerSettings.setWatchReturnValues(true);
       }
@@ -975,9 +974,13 @@ public class PythonDebuggerTest extends PyEnvTestCase {
 
       @Override
       public void testing() throws Exception {
+        waitForPause();
+        stepOver();
         waitForPause();
         eval(PyDebugValue.RETURN_VALUES_PREFIX + "['bar'][0]").hasValue("1");
-        resume();
+        stepOver();
+        waitForPause();
+        stepOver();
         waitForPause();
         eval(PyDebugValue.RETURN_VALUES_PREFIX + "['foo']").hasValue("33");
         resume();
@@ -986,7 +989,7 @@ public class PythonDebuggerTest extends PyEnvTestCase {
       @NotNull
       @Override
       public Set<String> getTags() {
-        return ImmutableSet.of("-iron");
+        return ImmutableSet.of("-iron", "-python36");
       }
     });
   }
@@ -1155,6 +1158,7 @@ public class PythonDebuggerTest extends PyEnvTestCase {
       @Override
       public void before() throws Exception {
         toggleBreakpoint(getScriptName(), 2);
+        toggleBreakpoint(getScriptName(), 3);
       }
 
       @Override
@@ -1164,6 +1168,9 @@ public class PythonDebuggerTest extends PyEnvTestCase {
         toggleBreakpoint(getScriptName(), 5);
         resume();
         waitForPause();
+        eval("b").hasValue("2");
+        resume();
+        waitForPause();
         eval("d").hasValue("4");
         resume();
       }