PY-9132 Load big Python collections by parts

python/helpers/pydev/_pydevd_bundle/pydevd_resolver.py

@@ -6,12 +6,13 @@ import traceback
 from os.path import basename
 
 from _pydevd_bundle import pydevd_constants
-from _pydevd_bundle.pydevd_constants import dict_iter_items, dict_keys, xrange
+from _pydevd_bundle.pydevd_constants import dict_iter_items, dict_keys, xrange, IS_PYCHARM
+from _pydevd_bundle.pydevd_utils import get_var_and_offset
 
 
 # Note: 300 is already a lot to see in the outline (after that the user should really use the shell to get things)
 # and this also means we'll pass less information to the client side (which makes debugging faster).
-MAX_ITEMS_TO_HANDLE = 300 
+MAX_ITEMS_TO_HANDLE = 300 if not IS_PYCHARM else 100
 
 TOO_LARGE_MSG = 'Too large to show contents. Max items to show: ' + str(MAX_ITEMS_TO_HANDLE)
 TOO_LARGE_ATTR = 'Unable to handle:'
@@ -146,7 +147,7 @@ class DefaultResolver:
 
         if not names:
             names = self.get_names(var)
-        d = {}
+        d = OrderedDict()
 
         #Be aware that the order in which the filters are applied attempts to
         #optimize the operation by removing as many items as possible in the
@@ -221,25 +222,32 @@ class DictResolver:
             return key
 
     def init_dict(self):
-        return {}
+        return OrderedDict()
 
     def get_dictionary(self, dict):
+        dict, offset = get_var_and_offset(dict)
+
         ret = self.init_dict()
 
-        i = 0
-        for key, val in dict_iter_items(dict):
-            i += 1
-            #we need to add the id because otherwise we cannot find the real object to get its contents later on.
-            key = '%s (%s)' % (self.key_to_str(key), id(key))
-            ret[key] = val
-            if i > MAX_ITEMS_TO_HANDLE:
-                ret[TOO_LARGE_ATTR] = TOO_LARGE_MSG
-                break
+        for i, (key, val) in enumerate(dict_iter_items(dict)):
+            if i >= offset:
+                if i >= offset + MAX_ITEMS_TO_HANDLE:
+                    if not IS_PYCHARM:
+                        ret[TOO_LARGE_ATTR] = TOO_LARGE_MSG
+                    break
+                # we need to add the id because otherwise we cannot find the real object to get its contents later on.
+                key = '%s (%s)' % (self.key_to_str(key), id(key))
+                ret[key] = val
 
         ret['__len__'] = len(dict)
         # in case if the class extends built-in type and has some additional fields
         additional_fields = defaultResolver.get_dictionary(dict)
-        ret.update(additional_fields)
+        if IS_PYCHARM:
+            if offset == 0:
+                additional_fields.update(ret)
+                ret = additional_fields
+        else:
+            ret.update(additional_fields)
         return ret
 
 
@@ -261,24 +269,32 @@ class TupleResolver: #to enumerate tuples and lists
             return getattr(var, attribute)
 
     def get_dictionary(self, var):
+        var, offset = get_var_and_offset(var)
+
         l = len(var)
-        d = {}
+        d = OrderedDict()
 
         format_str = '%0' + str(int(len(str(l)))) + 'd'
 
-        i = 0
-        for item in var:
+        i = offset
+        for item in var[offset:offset+MAX_ITEMS_TO_HANDLE]:
             d[format_str % i] = item
             i += 1
-            
-            if i > MAX_ITEMS_TO_HANDLE:
-                d[TOO_LARGE_ATTR] = TOO_LARGE_MSG
+
+            if i > MAX_ITEMS_TO_HANDLE + offset:
+                if not IS_PYCHARM:
+                    d[TOO_LARGE_ATTR] = TOO_LARGE_MSG
                 break
-                
+
         d['__len__'] = len(var)
         # in case if the class extends built-in type and has some additional fields
         additional_fields = defaultResolver.get_dictionary(var)
-        d.update(additional_fields)
+        if IS_PYCHARM:
+            if offset == 0:
+                additional_fields.update(d)
+                d = additional_fields
+        else:
+            d.update(additional_fields)
         return d
 
 
@@ -307,21 +323,28 @@ class SetResolver:
         raise UnableToResolveVariableException('Unable to resolve %s in %s' % (attribute, var))
 
     def get_dictionary(self, var):
-        d = {}
+        var, offset = get_var_and_offset(var)
+
+        d = OrderedDict()
         i = 0
         for item in var:
+            if i >= offset:
+                if i >= offset + MAX_ITEMS_TO_HANDLE:
+                    if not IS_PYCHARM:
+                        d[TOO_LARGE_ATTR] = TOO_LARGE_MSG
+                    break
+                d[str(id(item))] = item
             i += 1
-            d[str(id(item))] = item
-            
-            if i > MAX_ITEMS_TO_HANDLE:
-                d[TOO_LARGE_ATTR] = TOO_LARGE_MSG
-                break
 
-            
         d['__len__'] = len(var)
         # in case if the class extends built-in type and has some additional fields
         additional_fields = defaultResolver.get_dictionary(var)
-        d.update(additional_fields)
+        if IS_PYCHARM:
+            if offset == 0:
+                additional_fields.update(d)
+                d = additional_fields
+        else:
+            d.update(additional_fields)
         return d
 
 
@@ -429,8 +452,33 @@ class DjangoFormResolver(DefaultResolver):
 #=======================================================================================================================
 class DequeResolver(TupleResolver):
     def get_dictionary(self, var):
-        d = TupleResolver.get_dictionary(self, var)
-        d['maxlen'] = getattr(var, 'maxlen', None)
+        var, offset = get_var_and_offset(var)
+
+        l = len(var)
+        d = OrderedDict()
+
+        format_str = '%0' + str(int(len(str(l)))) + 'd'
+
+        i = 0
+        for item in var:
+            if i >= offset:
+                if i >= offset + MAX_ITEMS_TO_HANDLE:
+                    if not IS_PYCHARM:
+                        d[TOO_LARGE_ATTR] = TOO_LARGE_MSG
+                    break
+                d[format_str % i] = item
+            i += 1
+
+        d['__len__'] = len(var)
+        # in case if the class extends built-in type and has some additional fields
+        additional_fields = defaultResolver.get_dictionary(var)
+        if IS_PYCHARM:
+            if offset == 0:
+                additional_fields['maxlen'] = getattr(var, 'maxlen', None)
+                additional_fields.update(d)
+                d = additional_fields
+        else:
+            d.update(additional_fields)
         return d
 
 

python/helpers/pydev/_pydevd_bundle/pydevd_thrift.py

@@ -16,6 +16,7 @@ from _pydevd_bundle.pydevd_extension_api import TypeResolveProvider, StrPresenta
 from _pydevd_bundle.pydevd_vars import get_label, array_default_format, MAXIMUM_ARRAY_SIZE
 from pydev_console.protocol import DebugValue, GetArrayResponse, ArrayData, ArrayHeaders, ColHeader, RowHeader, \
     UnsupportedArrayTypeException
+from _pydevd_bundle.pydevd_utils import take_first_n_coll_elements
 
 try:
     import types
@@ -227,8 +228,17 @@ def is_builtin(x):
     return getattr(x, '__module__', None) == BUILTINS_MODULE_NAME
 
 
+def is_numpy(x):
+    if not getattr(x, '__module__', None) == 'numpy':
+        return False
+    type_name = x.__name__
+    return type_name == 'dtype' or type_name == 'bool_' or type_name == 'str_' or 'int' in type_name or 'uint' in type_name \
+           or 'float' in type_name or 'complex' in type_name
+
+
 def should_evaluate_full_value(val):
-    return LOAD_VALUES_POLICY == ValuesPolicy.SYNC or (is_builtin(type(val)) and not isinstance(val, (list, tuple, dict)))
+    return LOAD_VALUES_POLICY == ValuesPolicy.SYNC or ((is_builtin(type(val)) or is_numpy(type(val)))
+                                                       and not isinstance(val, (list, tuple, dict, set, frozenset)))
 
 
 def frame_vars_to_struct(frame_f_locals, hidden_ns=None):
@@ -300,10 +310,12 @@ def var_to_struct(val, name, do_trim=True, evaluate_full_value=True):
                     value = pydevd_resolver.frameResolver.get_frame_name(v)
 
                 elif v.__class__ in (list, tuple):
-                    if len(v) > 300:
-                        value = '<Too big to print. Len: %s>' % (len(v),)
+                    if len(v) > pydevd_resolver.MAX_ITEMS_TO_HANDLE:
+                        value = '%s: %s' % (str(v.__class__),  take_first_n_coll_elements(
+                            v, pydevd_resolver.MAX_ITEMS_TO_HANDLE))
+                        value = value.rstrip(')]}') + '...'
                     else:
-                        value = str(v)
+                        value = '%s: %s' % (str(v.__class__, v))
                 else:
                     value = str(v)
             else:

python/helpers/pydev/_pydevd_bundle/pydevd_utils.py

@@ -7,8 +7,13 @@ try:
 except:
     from urllib.parse import quote  # @UnresolvedImport
 
+try:
+    from collections import OrderedDict
+except:
+    OrderedDict = dict
+
 import inspect
-from _pydevd_bundle.pydevd_constants import IS_PY3K
+from _pydevd_bundle.pydevd_constants import IS_PY3K, dict_iter_items
 import sys
 from _pydev_bundle import pydev_log
 
@@ -206,3 +211,35 @@ def is_ignored_by_filter(filename, filename_to_ignored_by_filters_cache={}):
 
         return filename_to_ignored_by_filters_cache[filename]
 
+
+def take_first_n_coll_elements(coll, n):
+    if coll.__class__ in (list, tuple):
+        return coll[:n]
+    elif coll.__class__ in (set, frozenset):
+        buf = []
+        for i, x in enumerate(coll):
+            if i >= n:
+                break
+            buf.append(x)
+        return type(coll)(buf)
+    elif coll.__class__ in (dict, OrderedDict):
+        ret = type(coll)()
+        for i, (k, v) in enumerate(dict_iter_items(coll)):
+            if i >= n:
+                break
+            ret[k] = v
+        return ret
+    else:
+        raise TypeError("Unsupported collection type: '%s'" % str(coll.__class__))
+
+
+class VariableWithOffset(object):
+    def __init__(self, data, offset):
+        self.data, self.offset = data, offset
+
+
+def get_var_and_offset(var):
+    if isinstance(var, VariableWithOffset):
+        return var.data, var.offset
+    return var, 0
+

python/helpers/pydev/_pydevd_bundle/pydevd_vars.py

@@ -14,11 +14,16 @@ except ImportError:
     from io import StringIO
 import sys  # @Reimport
 
+try:
+    from collections import OrderedDict
+except:
+    OrderedDict = dict
+
 from _pydev_imps._pydev_saved_modules import threading
 import traceback
 from _pydevd_bundle import pydevd_save_locals
 from _pydev_bundle.pydev_imports import Exec, execfile
-from _pydevd_bundle.pydevd_utils import to_string
+from _pydevd_bundle.pydevd_utils import to_string, VariableWithOffset
 
 SENTINEL_VALUE = []
 
@@ -162,7 +167,7 @@ def getVariable(thread_id, frame_id, scope, attrs):
     BY_ID means we'll traverse the list of all objects alive to get the object.
 
     :attrs: after reaching the proper scope, we have to get the attributes until we find
-            the proper location (i.e.: obj\tattr1\tattr2)
+            the proper location (i.e.: obj\tattr1\tattr2).
 
     :note: when BY_ID is used, the frame_id is considered the id of the object to find and
            not the frame (as we don't care about the frame in this case).
@@ -229,6 +234,23 @@ def getVariable(thread_id, frame_id, scope, attrs):
     return var
 
 
+def get_offset(attrs):
+    """
+    Extract offset from the given attributes.
+
+    :param attrs: The string of a compound variable fields split by tabs.
+      If an offset is given, it must go the first element.
+    :return: The value of offset if given or 0.
+    """
+    offset = 0
+    if attrs is not None:
+        try:
+            offset = int(attrs.split('\t')[0])
+        except ValueError:
+            pass
+    return offset
+
+
 def resolve_compound_variable_fields(thread_id, frame_id, scope, attrs):
     """
     Resolve compound variable in debugger scopes by its name and attributes
@@ -239,16 +261,24 @@ def resolve_compound_variable_fields(thread_id, frame_id, scope, attrs):
     :param attrs: after reaching the proper scope, we have to get the attributes until we find
             the proper location (i.e.: obj\tattr1\tattr2)
     :return: a dictionary of variables's fields
+
+    :note: PyCharm supports progressive loading of large collections and uses the `attrs`
+           parameter to pass the offset, e.g. 300\t\obj\tattr1\tattr2 should return
+           the value of attr2 starting from the 300th element. This hack makes it possible
+           to add the support of progressive loading without extending of the protocol.
     """
+    offset = get_offset(attrs)
+
+    orig_attrs, attrs = attrs, attrs.split('\t', 1)[1] if offset else attrs
 
     var = getVariable(thread_id, frame_id, scope, attrs)
 
     try:
         _type, _typeName, resolver = get_type(var)
-        return _typeName, resolver.get_dictionary(var)
+        return _typeName, resolver.get_dictionary(VariableWithOffset(var, offset) if offset else var)
     except:
         sys.stderr.write('Error evaluating: thread_id: %s\nframe_id: %s\nscope: %s\nattrs: %s\n' % (
-            thread_id, frame_id, scope, attrs,))
+            thread_id, frame_id, scope, orig_attrs,))
         traceback.print_exc()
 
 
@@ -278,6 +308,10 @@ def resolve_compound_var_object_fields(var, attrs):
     :param attrs: a sequence of variable's attributes separated by \t (i.e.: obj\tattr1\tattr2)
     :return: a dictionary of variables's fields
     """
+    offset = get_offset(attrs)
+
+    attrs = attrs.split('\t', 1)[1] if offset else attrs
+
     attr_list = attrs.split('\t')
 
     for k in attr_list:
@@ -286,7 +320,7 @@ def resolve_compound_var_object_fields(var, attrs):
 
     try:
         type, _typeName, resolver = get_type(var)
-        return resolver.get_dictionary(var)
+        return resolver.get_dictionary(VariableWithOffset(var, offset) if offset else var)
     except:
         traceback.print_exc()
 

python/helpers/pydev/_pydevd_bundle/pydevd_xml.py

@@ -13,6 +13,7 @@ from _pydevd_bundle.pydevd_constants import dict_iter_items, dict_keys, IS_PY3K,
     BUILTINS_MODULE_NAME, MAXIMUM_VARIABLE_REPRESENTATION_SIZE, RETURN_VALUES_DICT, LOAD_VALUES_POLICY, ValuesPolicy, DEFAULT_VALUES_DICT
 from _pydev_bundle.pydev_imports import quote
 from _pydevd_bundle.pydevd_extension_api import TypeResolveProvider, StrPresentationProvider
+from _pydevd_bundle.pydevd_utils import take_first_n_coll_elements
 
 try:
     import types
@@ -242,8 +243,17 @@ def is_builtin(x):
     return getattr(x, '__module__', None) == BUILTINS_MODULE_NAME
 
 
+def is_numpy(x):
+    if not getattr(x, '__module__', None) == 'numpy':
+        return False
+    type_name = x.__name__
+    return type_name == 'dtype' or type_name == 'bool_' or type_name == 'str_' or 'int' in type_name or 'uint' in type_name \
+           or 'float' in type_name or 'complex' in type_name
+
+
 def should_evaluate_full_value(val):
-    return LOAD_VALUES_POLICY == ValuesPolicy.SYNC or (is_builtin(type(val)) and not isinstance(val, (list, tuple, dict)))
+    return LOAD_VALUES_POLICY == ValuesPolicy.SYNC or ((is_builtin(type(val)) or is_numpy(type(val)))
+                                                       and not isinstance(val, (list, tuple, dict, set, frozenset)))
 
 
 def frame_vars_to_xml(frame_f_locals, hidden_ns=None):
@@ -311,8 +321,10 @@ def var_to_xml(val, name, doTrim=True, additional_in_xml='', evaluate_full_value
                     value = pydevd_resolver.frameResolver.get_frame_name(v)
 
                 elif v.__class__ in (list, tuple, set, frozenset, dict):
-                    if len(v) > 300:
-                        value = '%s: %s' % (str(v.__class__), '<Too big to print. Len: %s>' % (len(v),))
+                    if len(v) > pydevd_resolver.MAX_ITEMS_TO_HANDLE:
+                        value = '%s: %s' % (str(v.__class__), take_first_n_coll_elements(
+                            v, pydevd_resolver.MAX_ITEMS_TO_HANDLE))
+                        value = value.rstrip(')]}') + '...'
                     else:
                         value = '%s: %s' % (str(v.__class__), v)
                 else:

python/helpers/pydev/pydevd_plugins/extensions/types/pydevd_plugin_numpy_types.py

@@ -1,7 +1,17 @@
-from _pydevd_bundle.pydevd_extension_api import TypeResolveProvider
+from _pydevd_bundle.pydevd_constants import IS_PYCHARM
+from _pydevd_bundle.pydevd_extension_api import TypeResolveProvider, StrPresentationProvider
 from _pydevd_bundle.pydevd_resolver import defaultResolver, MAX_ITEMS_TO_HANDLE, TOO_LARGE_ATTR, TOO_LARGE_MSG
+from _pydevd_bundle.pydevd_utils import get_var_and_offset
 from .pydevd_helpers import find_mod_attr
 
+try:
+    from collections import OrderedDict
+except:
+    OrderedDict = dict
+
+
+DEFAULT_PRECISION = 5
+
 
 # =======================================================================================================================
 # NdArrayResolver
@@ -26,9 +36,16 @@ class NDArrayTypeResolveProvider(object):
             return False
         return obj.dtype.kind in 'biufc'
 
+    def round_if_possible(self, obj):
+        try:
+            return obj.round(DEFAULT_PRECISION)
+        except TypeError:
+            return obj
+
     def resolve(self, obj, attribute):
         if attribute == '__internals__':
-            return defaultResolver.get_dictionary(obj)
+            if not IS_PYCHARM:
+                return defaultResolver.get_dictionary(obj)
         if attribute == 'min':
             if self.is_numeric(obj):
                 return obj.min()
@@ -56,11 +73,16 @@ class NDArrayTypeResolveProvider(object):
                     setattr(container, TOO_LARGE_ATTR, TOO_LARGE_MSG)
                     break
             return container
+        if IS_PYCHARM and attribute == 'array':
+            container = NdArrayItemsContainer()
+            container.items = obj
+            return container
         return None
 
     def get_dictionary(self, obj):
         ret = dict()
-        ret['__internals__'] = defaultResolver.get_dictionary(obj)
+        if not IS_PYCHARM:
+            ret['__internals__'] = defaultResolver.get_dictionary(obj)
         if obj.size > 1024 * 1024:
             ret['min'] = 'ndarray too big, calculating min would slow down debugging'
             ret['max'] = 'ndarray too big, calculating max would slow down debugging'
@@ -74,11 +96,54 @@ class NDArrayTypeResolveProvider(object):
         ret['shape'] = obj.shape
         ret['dtype'] = obj.dtype
         ret['size'] = obj.size
-        ret['[0:%s] ' % (len(obj))] = list(obj[0:MAX_ITEMS_TO_HANDLE])
+        if IS_PYCHARM:
+            ret['array'] = NdArrayItemsContainer()
+        else:
+            ret['[0:%s] ' % (len(obj))] = list(obj[0:MAX_ITEMS_TO_HANDLE])
         return ret
 
 
+class NDArrayStrProvider(object):
+    def can_provide(self, type_object, type_name):
+        nd_array = find_mod_attr('numpy', 'ndarray')
+        return nd_array is not None and issubclass(type_object, nd_array)
+
+    def get_str(self, val):
+        return str(val[:MAX_ITEMS_TO_HANDLE])
+
+
+class NdArrayItemsContainerProvider(object):
+    def can_provide(self, type_object, type_name):
+        return issubclass(type_object, NdArrayItemsContainer)
+
+    def resolve(self, obj, attribute):
+        if attribute == '__len__':
+            return None
+        return obj.items[int(attribute)]
+
+    def get_dictionary(self, obj):
+        obj, offset = get_var_and_offset(obj)
+
+        l = len(obj.items)
+        d = OrderedDict()
+
+        format_str = '%0' + str(int(len(str(l)))) + 'd'
+
+        i = offset
+        for item in obj.items[offset:offset + MAX_ITEMS_TO_HANDLE]:
+            d[format_str % i] = item
+            i += 1
+
+            if i > MAX_ITEMS_TO_HANDLE + offset:
+                break
+        d['__len__'] = l
+        return d
+
+
 import sys
 
 if not sys.platform.startswith("java"):
     TypeResolveProvider.register(NDArrayTypeResolveProvider)
+    if IS_PYCHARM:
+        TypeResolveProvider.register(NdArrayItemsContainerProvider)
+        StrPresentationProvider.register(NDArrayStrProvider)

python/pydevSrc/com/jetbrains/python/debugger/PyDebugValue.java

@@ -19,13 +19,13 @@ import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-// todo: load long lists by parts
 // todo: null modifier for modify modules, class objects etc.
 public class PyDebugValue extends XNamedValue {
   private static final Logger LOG = Logger.getInstance("#com.jetbrains.python.pydev.PyDebugValue");
   private static final String DATA_FRAME = "DataFrame";
   private static final String SERIES = "Series";
   private static final Map<String, String> EVALUATOR_POSTFIXES = ImmutableMap.of("ndarray", "Array", DATA_FRAME, DATA_FRAME, SERIES, SERIES);
+  private static final int MAX_ITEMS_TO_HANDLE = 100;
   public static final int MAX_VALUE = 256;
   public static final int AVAILABLE_PROCESSORS = Runtime.getRuntime().availableProcessors();
 
@@ -45,6 +45,8 @@ public class PyDebugValue extends XNamedValue {
   private @Nullable PyVariableLocator myVariableLocator;
   private volatile @Nullable XValueNode myLastNode = null;
   private final boolean myErrorOnEval;
+  private int myOffset;
+  private int myCollectionLength = -1;
 
   public enum ValuesPolicy {
     SYNC, ASYNC, ON_DEMAND
@@ -99,6 +101,7 @@ public class PyDebugValue extends XNamedValue {
   public PyDebugValue(@NotNull PyDebugValue value, @NotNull String newName) {
     this(newName, value.getType(), value.getTypeQualifier(), value.getValue(), value.isContainer(), value.isReturnedVal(),
          value.isIPythonHidden(), value.isErrorOnEval(), value.getParent(), value.getFrameAccessor());
+    myOffset = value.getOffset();
     setLoadValuePolicy(value.getLoadValuePolicy());
     setTempName(value.getTempName());
   }
@@ -197,8 +200,11 @@ public class PyDebugValue extends XNamedValue {
       else if (isLen(myName)) {
         result.append('.').append(myName).append("()");
       }
-      else if (("ndarray".equals(myParent.getType()) || "matrix".equals(myParent.getType())) && myName.startsWith("[")) {
-        result.append(removeLeadingZeros(myName));
+      else if (("ndarray".equals(myParent.getType()) || "matrix".equals(myParent.getType())) && myName.equals("array")) {
+        // return the string representation of an ndarray
+      }
+      else if ("array".equals(myParent.getName()) && myParent.myParent != null && "ndarray".equals(myParent.myParent.getType())) {
+        result.append("[").append(removeLeadingZeros(myName)).append("]");
       }
       else {
         result.append('.').append(myName);
@@ -370,9 +376,18 @@ public class PyDebugValue extends XNamedValue {
     if (node.isObsolete()) return;
     ApplicationManager.getApplication().executeOnPooledThread(() -> {
       try {
-        final XValueChildrenList values = myFrameAccessor.loadVariable(this);
+        XValueChildrenList values = myFrameAccessor.loadVariable(this);
         if (!node.isObsolete()) {
-          node.addChildren(values, true);
+          updateLengthIfIsCollection(values);
+
+          if (isLargeCollection()) {
+            values = processLargeCollection(values);
+            node.addChildren(values, true);
+            updateOffset(node, values);
+          } else {
+            node.addChildren(values, true);
+          }
+
           getAsyncValues(myFrameAccessor, values);
         }
       }
@@ -500,4 +515,59 @@ public class PyDebugValue extends XNamedValue {
   public boolean isTemporary() {
     return myTempName != null;
   }
+
+  public int getOffset() {
+    return myOffset;
+  }
+
+  public void setOffset(int offset) {
+    myOffset = offset;
+  }
+
+  private boolean isLargeCollection() {
+    return myCollectionLength > MAX_ITEMS_TO_HANDLE;
+  }
+
+  private void updateLengthIfIsCollection(final XValueChildrenList values) {
+    if (myCollectionLength > 0 && values.size() == 0) return;
+
+    final int lastIndex = values.size() - 1;
+
+    // If there is the __len__ attribute it should always goes last.
+    if (values.size() > 0 && isLen(values.getName(lastIndex))) {
+      PyDebugValue len = (PyDebugValue)values.getValue(lastIndex);
+      try {
+        if (myCollectionLength == -1 && len.getValue() != null)
+          myCollectionLength = Integer.parseInt(len.getValue());
+      } catch (NumberFormatException ex) {
+        // Do nothing.
+      }
+    }
+  }
+
+  private XValueChildrenList processLargeCollection(final XValueChildrenList values) {
+    if (values.size() > 0 && isLargeCollection()) {
+      if (myOffset + Math.min(MAX_ITEMS_TO_HANDLE, values.size()) < myCollectionLength) {
+        XValueChildrenList newValues = new XValueChildrenList();
+        for(int i = 0; i < values.size() - 1; i++) {
+          newValues.add(values.getName(i), values.getValue(i));
+        }
+        return newValues;
+      }
+    }
+    return values;
+  }
+
+  private void updateOffset(final XCompositeNode node, final XValueChildrenList values) {
+    if (myContainer && isLargeCollection()) {
+      if (myOffset + Math.min(values.size(), MAX_ITEMS_TO_HANDLE) >= myCollectionLength) {
+        myOffset = myCollectionLength;
+      }
+      else {
+        node.tooManyChildren(myCollectionLength - Math.min(values.size(), MAX_ITEMS_TO_HANDLE) - myOffset);
+        myOffset += Math.min(values.size(), MAX_ITEMS_TO_HANDLE);
+      }
+    }
+  }
+
 }

python/pydevSrc/com/jetbrains/python/debugger/PyFullValueEvaluator.java

@@ -39,7 +39,7 @@ public class PyFullValueEvaluator extends XFullValueEvaluator {
 
     ApplicationManager.getApplication().executeOnPooledThread(() -> {
       try {
-        final PyDebugValue value = myDebugProcess.evaluate(expression, false, false);
+        final PyDebugValue value = myDebugProcess.evaluate(expression, false, true);
         if (value.getValue() == null) {
           throw new PyDebuggerException("Failed to Load Value");
         }

python/pydevSrc/com/jetbrains/python/debugger/pydev/GetVariableCommand.java

@@ -12,7 +12,7 @@ public class GetVariableCommand extends GetFrameCommand {
 
   public GetVariableCommand(final RemoteDebugger debugger, final String threadId, final String frameId, PyDebugValue var) {
     super(debugger, GET_VARIABLE, threadId, frameId);
-    myVariableName = composeName(var);
+    myVariableName = var.getOffset() == 0 ? composeName(var) : var.getOffset() + "\t" + composeName(var);
     myParent = var;
   }
 
@@ -26,6 +26,7 @@ public class GetVariableCommand extends GetFrameCommand {
   public static String composeName(final PyDebugValue var) {
     final StringBuilder sb = new StringBuilder();
     PyDebugValue p = var;
+
     while (p != null) {
       if (sb.length() > 0 ) {
         sb.insert(0, '\t');
@@ -41,6 +42,7 @@ public class GetVariableCommand extends GetFrameCommand {
       }
       p = p.getParent();
     }
+
     return sb.toString();
   }
 

python/src/com/jetbrains/python/console/PydevConsoleCommunication.java

@@ -551,7 +551,9 @@ public abstract class PydevConsoleCommunication extends AbstractConsoleCommunica
   public XValueChildrenList loadVariable(PyDebugValue var) throws PyDebuggerException {
     if (!isCommunicationClosed()) {
       try {
-        List<DebugValue> ret = getPythonConsoleBackendClient().getVariable(GetVariableCommand.composeName(var));
+        final String name = var.getOffset() == 0 ? GetVariableCommand.composeName(var)
+                                                 : var.getOffset() + "\t" + GetVariableCommand.composeName(var);
+        List<DebugValue> ret = getPythonConsoleBackendClient().getVariable(name);
         return parseVars(ret, var, this);
       }
       catch (CommunicationClosedException | PyConsoleProcessFinishedException e) {

python/testData/debug/test_large_collections.py

@@ -0,0 +1,16 @@
+from collections import deque
+
+L = list(range(1000))
+
+D = {}
+for i in range(1000):
+    D[i] = i
+
+S = set()
+
+for i in range(1000):
+    S.add(i)
+
+dq = deque(range(1000))
+
+len(dq)

python/testData/debug/test_large_numpy_arrays.py

@@ -0,0 +1,10 @@
+import numpy as np
+import pandas as pd
+
+nd = np.arange(1000)
+
+s = pd.Series(name='foo', data=nd)
+
+df = pd.DataFrame({'bar': nd})
+
+df.size

python/testSrc/com/jetbrains/env/python/PythonDebuggerTest.java

@@ -10,6 +10,7 @@ import com.intellij.testFramework.fixtures.IdeaProjectTestFixture;
 import com.intellij.xdebugger.XDebuggerTestUtil;
 import com.intellij.xdebugger.breakpoints.SuspendPolicy;
 import com.intellij.xdebugger.breakpoints.XBreakpoint;
+import com.intellij.xdebugger.frame.XValueChildrenList;
 import com.jetbrains.TestEnv;
 import com.jetbrains.env.PyEnvTestCase;
 import com.jetbrains.env.PyProcessWithConsoleTestTask;
@@ -1493,5 +1494,232 @@ public class PythonDebuggerTest extends PyEnvTestCase {
       }
     });
   }
+
+  @Test
+  public void testLargeCollectionsLoading() {
+    runPythonTest(new PyDebuggerTask("/debug", "test_large_collections.py") {
+      @Override
+      public void before() { toggleBreakpoint(getFilePath(getScriptName()), 15); }
+
+      @Override
+      public void testing() throws Exception {
+        waitForPause();
+        List<PyDebugValue> frameVariables = loadFrame();
+
+        // The effective maximum number of the debugger returns is MAX_ITEMS_TO_HANDLE
+        // plus the __len__ attribute.
+        final int effectiveMaxItemsNumber = MAX_ITEMS_TO_HANDLE + 1;
+
+        // Large list.
+        PyDebugValue L = findDebugValueByName(frameVariables, "L");
+
+        XValueChildrenList children = loadVariable(L);
+        int collectionLength = 1000;
+
+        assertEquals(effectiveMaxItemsNumber, children.size());
+        for (int i = 0; i < MAX_ITEMS_TO_HANDLE; i++) {
+          assertTrue(hasChildWithName(children, formatStr(i, collectionLength)));
+          assertTrue(hasChildWithValue(children, i));
+        }
+
+        L.setOffset(600);
+        children = loadVariable(L);
+        assertEquals(effectiveMaxItemsNumber, children.size());
+        for (int i = 600; i < MAX_ITEMS_TO_HANDLE; i++) {
+          assertTrue(hasChildWithName(children, formatStr(i, collectionLength)));
+          assertTrue(hasChildWithValue(children, i));
+        }
+
+        L.setOffset(900);
+        children = loadVariable(L);
+        assertEquals(101, children.size());
+        for (int i = 900; i < collectionLength; i++) {
+          assertTrue(hasChildWithName(children, formatStr(i, collectionLength)));
+          assertTrue(hasChildWithValue(children, i));
+        }
+
+        // Large dict.
+        PyDebugValue D = findDebugValueByName(frameVariables, "D");
+
+        children = loadVariable(D);
+        collectionLength = 1000;
+
+        assertEquals(effectiveMaxItemsNumber, children.size());
+        for (int i = 0; i < MAX_ITEMS_TO_HANDLE; i++) {
+          assertTrue(hasChildWithName(children, i));
+          assertTrue(hasChildWithValue(children, i));
+        }
+
+        D.setOffset(600);
+        children = loadVariable(D);
+        assertEquals(effectiveMaxItemsNumber, children.size());
+        for (int i = 600; i < MAX_ITEMS_TO_HANDLE; i++) {
+          assertTrue(hasChildWithName(children, i));
+          assertTrue(hasChildWithValue(children, i));
+        }
+
+        D.setOffset(900);
+        children = loadVariable(D);
+        assertEquals(101, children.size());
+        for (int i = 900; i < collectionLength; i++) {
+          assertTrue(hasChildWithName(children, i));
+          assertTrue(hasChildWithValue(children, i));
+        }
+
+        // Large set.
+        PyDebugValue S = findDebugValueByName(frameVariables, "S");
+
+        children = loadVariable(S);
+        collectionLength = 1000;
+
+        assertEquals(effectiveMaxItemsNumber, children.size());
+        for (int i = 0; i < MAX_ITEMS_TO_HANDLE; i++) {
+          assertTrue(hasChildWithValue(children, i));
+        }
+
+        S.setOffset(600);
+        children = loadVariable(S);
+        assertEquals(effectiveMaxItemsNumber, children.size());
+        for (int i = 600; i < MAX_ITEMS_TO_HANDLE; i++) {
+          assertTrue(hasChildWithValue(children, i));
+        }
+
+        S.setOffset(900);
+        children = loadVariable(S);
+        assertEquals(101, children.size());
+        for (int i = 900; i < collectionLength; i++) {
+          assertTrue(hasChildWithValue(children, i));
+        }
+
+        // Large deque.
+        PyDebugValue dq = findDebugValueByName(frameVariables, "dq");
+
+        children = loadVariable(dq);
+        collectionLength = 1000;
+
+        assertEquals(effectiveMaxItemsNumber + 1, children.size()); // one extra child for maxlen
+        for (int i = 1; i < MAX_ITEMS_TO_HANDLE; i++) {
+          assertTrue(hasChildWithName(children, formatStr(i, collectionLength)));
+          assertTrue(hasChildWithValue(children, i));
+        }
+
+        dq.setOffset(600);
+        children = loadVariable(dq);
+        assertEquals(effectiveMaxItemsNumber, children.size());
+        for (int i = 600; i < MAX_ITEMS_TO_HANDLE; i++) {
+          assertTrue(hasChildWithName(children, formatStr(i, collectionLength)));
+          assertTrue(hasChildWithValue(children, i));
+        }
+
+        dq.setOffset(900);
+        children = loadVariable(dq);
+        assertEquals(101, children.size());
+        for (int i = 900; i < collectionLength; i++) {
+          assertTrue(hasChildWithName(children, formatStr(i, collectionLength)));
+          assertTrue(hasChildWithValue(children, i));
+        }
+      }
+    });
+  }
+
+  @Test
+  public void testLargeNumpyArraysLoading() {
+    runPythonTest(new PyDebuggerTask("/debug", "test_large_numpy_arrays.py") {
+      @NotNull
+      @Override
+      public Set<String> getTags() {
+        return ImmutableSet.of("pandas");
+      }
+
+      @Override
+      public void before() { toggleBreakpoint(getFilePath(getScriptName()), 9); }
+
+      @Override
+      public void testing() throws Exception {
+        waitForPause();
+        List<PyDebugValue> frameVariables = loadFrame();
+
+        int collectionLength = 1000;
+
+        // NumPy array
+
+        PyDebugValue nd = findDebugValueByName(frameVariables, "nd");
+        XValueChildrenList children = loadVariable(nd);
+
+        assertEquals("min", children.getName(0));
+        assertEquals("max", children.getName(1));
+        assertEquals("shape", children.getName(2));
+        assertEquals("dtype", children.getName(3));
+        assertEquals("size", children.getName(4));
+        assertEquals("array", children.getName(5));
+
+        PyDebugValue array = (PyDebugValue) children.getValue(5);
+
+        children = loadVariable(array);
+        assertEquals(MAX_ITEMS_TO_HANDLE + 1, children.size());
+
+        for (int i = 0; i < MAX_ITEMS_TO_HANDLE; i++) {
+          assertTrue(hasChildWithName(children, formatStr(i, collectionLength)));
+        }
+
+        array.setOffset(950);
+        children = loadVariable(array);
+
+        assertEquals(51, children.size());
+
+        for (int i = 950; i < collectionLength; i++) {
+          assertTrue(hasChildWithName(children, formatStr(i, collectionLength)));
+        }
+
+        // Pandas series
+
+        PyDebugValue s = findDebugValueByName(frameVariables, "s");
+        children = loadVariable(s);
+        PyDebugValue values = (PyDebugValue) children.getValue(children.size() - 1);
+        children = loadVariable(values);
+        array = (PyDebugValue) children.getValue(children.size() - 1);
+        children = loadVariable(array);
+
+        assertEquals(MAX_ITEMS_TO_HANDLE + 1, children.size());
+
+        for (int i = 0; i < MAX_ITEMS_TO_HANDLE; i++) {
+          assertTrue(hasChildWithName(children, formatStr(i, collectionLength)));
+        }
+
+        array.setOffset(950);
+        children = loadVariable(array);
+
+        assertEquals(51, children.size());
+
+        for (int i = 950; i < collectionLength; i++) {
+          assertTrue(hasChildWithName(children, formatStr(i, collectionLength)));
+        }
+
+        // Pandas data frame
+
+        PyDebugValue df = findDebugValueByName(frameVariables, "df");
+        children = loadVariable(df);
+        values = (PyDebugValue) children.getValue(children.size() - 1);
+        children = loadVariable(values);
+        array = (PyDebugValue) children.getValue(children.size() - 1);
+        children = loadVariable(array);
+
+        assertEquals(MAX_ITEMS_TO_HANDLE + 1, children.size());
+
+        for (int i = 0; i < MAX_ITEMS_TO_HANDLE; i++) {
+          assertTrue(hasChildWithName(children, formatStr(i, collectionLength)));
+        }
+
+        array.setOffset(950);
+        children = loadVariable(array);
+
+        assertEquals(51, children.size());
+
+        for (int i = 950; i < collectionLength; i++) {
+          assertTrue(hasChildWithName(children, formatStr(i, collectionLength)));
+        }
+      }
+    });
+  }
 }
 

python/testSrc/com/jetbrains/env/python/debug/PyBaseDebuggerTask.java

@@ -64,6 +64,10 @@ public abstract class PyBaseDebuggerTask extends PyExecutionFixtureTestTask {
   protected boolean myProcessCanTerminate;
   protected ExecutionResult myExecutionResult;
   protected SuspendPolicy myDefaultSuspendPolicy = SuspendPolicy.THREAD;
+  /**
+   * The value must align with the one from the pydevd_resolver.py module.
+   */
+  protected static final int MAX_ITEMS_TO_HANDLE = 100;
 
   protected PyBaseDebuggerTask(@Nullable final String relativeTestDataPath) {
     super(relativeTestDataPath);
@@ -173,6 +177,10 @@ public abstract class PyBaseDebuggerTask extends PyExecutionFixtureTestTask {
     return convertToList(myDebugProcess.loadVariable(var));
   }
 
+  protected XValueChildrenList loadVariable(PyDebugValue var) throws PyDebuggerException {
+    return myDebugProcess.loadVariable(var);
+  }
+
   protected List<PyDebugValue> loadFrame() throws PyDebuggerException {
     return convertToList(myDebugProcess.loadFrame());
   }
@@ -410,6 +418,35 @@ public abstract class PyBaseDebuggerTask extends PyExecutionFixtureTestTask {
     this.shouldPrintOutput = shouldPrintOutput;
   }
 
+  public String formatStr(int x, int collectionLength) {
+    return String.format("%0" + Integer.toString(collectionLength).length() + "d", x);
+  }
+
+  public boolean hasChildWithName(XValueChildrenList children, String name) {
+    for (int i = 0; i < children.size(); i++)
+      // Dictionary key names are followed by the hash so we need to consider only
+      // the first word of a name. For lists this operation doesn't have any effect.
+      if (children.getName(i).split(" ")[0].equals(name)) return true;
+    return false;
+  }
+
+  public boolean hasChildWithName(XValueChildrenList children, int name) {
+    return hasChildWithName(children, Integer.toString(name));
+  }
+
+  public boolean hasChildWithValue(XValueChildrenList children, String value) {
+    for (int i = 0; i < children.size(); i++) {
+      PyDebugValue current = (PyDebugValue)children.getValue(i);
+      if (current.getValue().equals(value)) return true;
+    }
+    return false;
+  }
+
+  public boolean hasChildWithValue(XValueChildrenList children, int value) {
+    return hasChildWithValue(children, Integer.toString(value));
+  }
+
+
   @Override
   public void setUp(final String testName) throws Exception {
     if (myFixture == null) {