From 0212297426d8ec0028f226ad853210f873c6be42 Mon Sep 17 00:00:00 2001 From: Alexander Koshevoy Date: Mon, 9 Jul 2018 12:30:15 +0300 Subject: [PATCH] PY-18029 Move methods for building Python console protocol structures from pydevd_vars to pydevd_thrift --- .../_pydev_bundle/pydev_console_utils.py | 2 +- .../pydev/_pydevd_bundle/pydevd_thrift.py | 244 ++++++++++++++++++ .../pydev/_pydevd_bundle/pydevd_vars.py | 242 ----------------- 3 files changed, 245 insertions(+), 243 deletions(-) diff --git a/python/helpers/pydev/_pydev_bundle/pydev_console_utils.py b/python/helpers/pydev/_pydev_bundle/pydev_console_utils.py index 6bbb8e1a140f..bcd2e1e1cf5f 100644 --- a/python/helpers/pydev/_pydev_bundle/pydev_console_utils.py +++ b/python/helpers/pydev/_pydev_bundle/pydev_console_utils.py @@ -515,7 +515,7 @@ class BaseInterpreterInterface: name = attr.split("\t")[-1] array = pydevd_vars.eval_in_context(name, self.get_namespace(), self.get_namespace()) # return pydevd_vars.table_like_struct_to_xml(array, name, roffset, coffset, rows, cols, format) - return pydevd_vars.table_like_struct_to_thrift_struct(array, name, roffset, coffset, rows, cols, format) + return pydevd_thrift.table_like_struct_to_thrift_struct(array, name, roffset, coffset, rows, cols, format) def evaluate(self, expression): # returns `DebugValue` of evaluated expression diff --git a/python/helpers/pydev/_pydevd_bundle/pydevd_thrift.py b/python/helpers/pydev/_pydevd_bundle/pydevd_thrift.py index c0de1988b17f..d885974d4819 100644 --- a/python/helpers/pydev/_pydevd_bundle/pydevd_thrift.py +++ b/python/helpers/pydev/_pydevd_bundle/pydevd_thrift.py @@ -7,6 +7,7 @@ from _pydevd_bundle import pydevd_resolver from _pydevd_bundle.pydevd_constants import dict_iter_items, dict_keys, IS_PY3K, \ BUILTINS_MODULE_NAME, MAXIMUM_VARIABLE_REPRESENTATION_SIZE, RETURN_VALUES_DICT, LOAD_VALUES_POLICY, ValuesPolicy, DEFAULT_VALUES_DICT from _pydevd_bundle.pydevd_extension_api import TypeResolveProvider, StrPresentationProvider +from _pydevd_bundle.pydevd_vars import get_label, VariableError, array_default_format, MAXIMUM_ARRAY_SIZE from pydev_console.thrift_communication import console_thrift try: @@ -367,3 +368,246 @@ def var_to_struct(val, name, doTrim=True, additional_in_xml='', evaluate_full_va def var_to_str(val, doTrim=True, evaluate_full_value=True): struct = var_to_struct(val, '', doTrim, '', evaluate_full_value) return struct.value + + +# from pydevd_vars.py + +def array_to_thrift_struct(array, name, roffset, coffset, rows, cols, format): + # returns `GetArrayResponse` + + # array, xml, r, c, f = array_to_meta_xml(array, name, format) + array, array_chunk, r, c, f = array_to_meta_thrift_struct(array, name, format) + format = '%' + f + if rows == -1 and cols == -1: + rows = r + cols = c + + rows = min(rows, MAXIMUM_ARRAY_SIZE) + cols = min(cols, MAXIMUM_ARRAY_SIZE) + + # there is no obvious rule for slicing (at least 5 choices) + if len(array) == 1 and (rows > 1 or cols > 1): + array = array[0] + if array.size > len(array): + array = array[roffset:, coffset:] + rows = min(rows, len(array)) + cols = min(cols, len(array[0])) + if len(array) == 1: + array = array[0] + elif array.size == len(array): + if roffset == 0 and rows == 1: + array = array[coffset:] + cols = min(cols, len(array)) + elif coffset == 0 and cols == 1: + array = array[roffset:] + rows = min(rows, len(array)) + + def get_value(row, col): + value = array + if rows == 1 or cols == 1: + if rows == 1 and cols == 1: + value = array[0] + else: + value = array[(col if rows == 1 else row)] + if "ndarray" in str(type(value)): + value = value[0] + else: + value = array[row][col] + return value + # xml += array_data_to_xml(rows, cols, lambda r: (get_value(r, c) for c in range(cols))) + array_chunk.data = array_data_to_thrift_struct(rows, cols, lambda r: (get_value(r, c) for c in range(cols))) + return array_chunk + + +def array_to_meta_thrift_struct(array, name, format): + type = array.dtype.kind + slice = name + l = len(array.shape) + + # initial load, compute slice + if format == '%': + if l > 2: + slice += '[0]' * (l - 2) + for r in range(l - 2): + array = array[0] + if type == 'f': + format = '.5f' + elif type == 'i' or type == 'u': + format = 'd' + else: + format = 's' + else: + format = format.replace('%', '') + + l = len(array.shape) + reslice = "" + if l > 2: + raise Exception("%s has more than 2 dimensions." % slice) + elif l == 1: + # special case with 1D arrays arr[i, :] - row, but arr[:, i] - column with equal shape and ndim + # http://stackoverflow.com/questions/16837946/numpy-a-2-rows-1-column-file-loadtxt-returns-1row-2-columns + # explanation: http://stackoverflow.com/questions/15165170/how-do-i-maintain-row-column-orientation-of-vectors-in-numpy?rq=1 + # we use kind of a hack - get information about memory from C_CONTIGUOUS + is_row = array.flags['C_CONTIGUOUS'] + + if is_row: + rows = 1 + cols = len(array) + if cols < len(array): + reslice = '[0:%s]' % (cols) + array = array[0:cols] + else: + cols = 1 + rows = len(array) + if rows < len(array): + reslice = '[0:%s]' % (rows) + array = array[0:rows] + elif l == 2: + rows = array.shape[-2] + cols = array.shape[-1] + if cols < array.shape[-1] or rows < array.shape[-2]: + reslice = '[0:%s, 0:%s]' % (rows, cols) + array = array[0:rows, 0:cols] + + # avoid slice duplication + if not slice.endswith(reslice): + slice += reslice + + bounds = (0, 0) + if type in "biufc": + bounds = (array.min(), array.max()) + # return array, slice_to_xml(slice, rows, cols, format, type, bounds), rows, cols, format + array_chunk = console_thrift.GetArrayResponse() + array_chunk.slice = slice + array_chunk.rows = rows + array_chunk.cols = cols + array_chunk.format = format + array_chunk.type = type + array_chunk.bounds = bounds + # return array, slice_to_xml(slice, rows, cols, format, type, bounds), rows, cols, format + return array, array_chunk, rows, cols, format + + +def dataframe_to_thrift_struct(df, name, roffset, coffset, rows, cols, format): + """ + :type df: pandas.core.frame.DataFrame + :type name: str + :type coffset: int + :type roffset: int + :type rows: int + :type cols: int + :type format: str + + + """ + dim = len(df.axes) + num_rows = df.shape[0] + num_cols = df.shape[1] if dim > 1 else 1 + # xml = slice_to_xml(name, num_rows, num_cols, "", "", (0, 0)) + array_chunk = console_thrift.GetArrayResponse() + array_chunk.slice = name + array_chunk.rows = num_rows + array_chunk.cols = num_cols + array_chunk.format = "" + array_chunk.type = "" + array_chunk.bounds = (0, 0) + + if (rows, cols) == (-1, -1): + rows, cols = num_rows, num_cols + + rows = min(rows, MAXIMUM_ARRAY_SIZE) + cols = min(cols, MAXIMUM_ARRAY_SIZE, num_cols) + # need to precompute column bounds here before slicing! + col_bounds = [None] * cols + dtypes = [None] * cols + if dim > 1: + for col in range(cols): + dtype = df.dtypes.iloc[coffset + col].kind + dtypes[col] = dtype + if dtype in "biufc": + cvalues = df.iloc[:, coffset + col] + bounds = (cvalues.min(), cvalues.max()) + else: + bounds = (0, 0) + col_bounds[col] = bounds + else: + dtype = df.dtype.kind + dtypes[0] = dtype + col_bounds[0] = (df.min(), df.max()) if dtype in "biufc" else (0, 0) + + df = df.iloc[roffset: roffset + rows, coffset: coffset + cols] if dim > 1 else df.iloc[roffset: roffset + rows] + rows = df.shape[0] + cols = df.shape[1] if dim > 1 else 1 + format = format.replace('%', '') + + def col_to_format(c): + return format if dtypes[c] == 'f' and format else array_default_format(dtypes[c]) + + # xml += header_data_to_xml(rows, cols, dtypes, col_bounds, col_to_format, df, dim) + array_chunk.headers = header_data_to_thrift_struct(rows, cols, dtypes, col_bounds, col_to_format, df, dim) + # xml += array_data_to_xml(rows, cols, lambda r: (("%" + col_to_format(c)) % (df.iat[r, c] if dim > 1 else df.iat[r]) + # for c in range(cols))) + array_chunk.data = array_data_to_thrift_struct(rows, cols, lambda r: (("%" + col_to_format(c)) % (df.iat[r, c] if dim > 1 else df.iat[r]) + for c in range(cols))) + # return xml + return array_chunk + + +def array_data_to_thrift_struct(rows, cols, get_row): + array_data = console_thrift.ArrayData() + # xml = "\n" % (rows, cols) + array_data.rows = rows + array_data.cols = cols + # `ArrayData.data` + data = [] + for row in range(rows): + # xml += "\n" % to_string(row) + # for value in get_row(row): + # xml += var_to_xml(value, '') + data.append([var_to_str(value) for value in get_row(row)]) + + array_data.data = data + # return xml + return array_data + + +def header_data_to_thrift_struct(rows, cols, dtypes, col_bounds, col_to_format, df, dim): + # xml = "\n" % (rows, cols) + array_headers = console_thrift.ArrayHeaders() + col_headers = [] + for col in range(cols): + col_label = get_label(df.axes[1].values[col]) if dim > 1 else str(col) + bounds = col_bounds[col] + col_format = "%" + col_to_format(col) + col_header = console_thrift.ColHeader() + # col_header.index = col + col_header.label = col_label + col_header.type = dtypes[col] + col_header.format = col_to_format(col) + col_header.max = col_format % bounds[1] + col_header.min = col_format % bounds[0] + col_headers.append(col_header) + row_headers = [] + for row in range(rows): + row_header = console_thrift.RowHeader() + row_header.index = row + row_header.label = get_label(df.axes[0].values[row]) + row_headers.append(row_header) + # xml += "\n" % (str(row), get_label(df.axes[0].values[row])) + # xml += "\n" + array_headers.colHeaders = col_headers + array_headers.rowHeaders = row_headers + # return xml + return array_headers + + +TYPE_TO_THRIFT_STRUCT_CONVERTERS = {"ndarray": array_to_thrift_struct, "DataFrame": dataframe_to_thrift_struct, "Series": dataframe_to_thrift_struct} + + +def table_like_struct_to_thrift_struct(array, name, roffset, coffset, rows, cols, format): + # returns `GetArrayResponse` + _, type_name, _ = get_type(array) + if type_name in TYPE_TO_THRIFT_STRUCT_CONVERTERS: + return TYPE_TO_THRIFT_STRUCT_CONVERTERS[type_name](array, name, roffset, coffset, rows, cols, format) + else: + raise VariableError("type %s not supported" % type_name) diff --git a/python/helpers/pydev/_pydevd_bundle/pydevd_vars.py b/python/helpers/pydev/_pydevd_bundle/pydevd_vars.py index 15cbff88921c..e1a1213653f7 100644 --- a/python/helpers/pydev/_pydevd_bundle/pydevd_vars.py +++ b/python/helpers/pydev/_pydevd_bundle/pydevd_vars.py @@ -6,9 +6,7 @@ import pickle from _pydev_imps._pydev_saved_modules import thread from _pydevd_bundle.pydevd_constants import get_frame, get_thread_id, xrange from _pydevd_bundle.pydevd_custom_frames import get_custom_frame -from _pydevd_bundle.pydevd_thrift import var_to_str from _pydevd_bundle.pydevd_xml import ExceptionOnEvaluate, get_type, var_to_xml -from pydev_console.thrift_communication import console_thrift try: from StringIO import StringIO @@ -645,243 +643,3 @@ def table_like_struct_to_xml(array, name, roffset, coffset, rows, cols, format): else: raise VariableError("type %s not supported" % type_name) - -def array_to_thrift_struct(array, name, roffset, coffset, rows, cols, format): - # returns `GetArrayResponse` - - # array, xml, r, c, f = array_to_meta_xml(array, name, format) - array, array_chunk, r, c, f = array_to_meta_thrift_struct(array, name, format) - format = '%' + f - if rows == -1 and cols == -1: - rows = r - cols = c - - rows = min(rows, MAXIMUM_ARRAY_SIZE) - cols = min(cols, MAXIMUM_ARRAY_SIZE) - - # there is no obvious rule for slicing (at least 5 choices) - if len(array) == 1 and (rows > 1 or cols > 1): - array = array[0] - if array.size > len(array): - array = array[roffset:, coffset:] - rows = min(rows, len(array)) - cols = min(cols, len(array[0])) - if len(array) == 1: - array = array[0] - elif array.size == len(array): - if roffset == 0 and rows == 1: - array = array[coffset:] - cols = min(cols, len(array)) - elif coffset == 0 and cols == 1: - array = array[roffset:] - rows = min(rows, len(array)) - - def get_value(row, col): - value = array - if rows == 1 or cols == 1: - if rows == 1 and cols == 1: - value = array[0] - else: - value = array[(col if rows == 1 else row)] - if "ndarray" in str(type(value)): - value = value[0] - else: - value = array[row][col] - return value - # xml += array_data_to_xml(rows, cols, lambda r: (get_value(r, c) for c in range(cols))) - array_chunk.data = array_data_to_thrift_struct(rows, cols, lambda r: (get_value(r, c) for c in range(cols))) - return array_chunk - - -def array_to_meta_thrift_struct(array, name, format): - type = array.dtype.kind - slice = name - l = len(array.shape) - - # initial load, compute slice - if format == '%': - if l > 2: - slice += '[0]' * (l - 2) - for r in range(l - 2): - array = array[0] - if type == 'f': - format = '.5f' - elif type == 'i' or type == 'u': - format = 'd' - else: - format = 's' - else: - format = format.replace('%', '') - - l = len(array.shape) - reslice = "" - if l > 2: - raise Exception("%s has more than 2 dimensions." % slice) - elif l == 1: - # special case with 1D arrays arr[i, :] - row, but arr[:, i] - column with equal shape and ndim - # http://stackoverflow.com/questions/16837946/numpy-a-2-rows-1-column-file-loadtxt-returns-1row-2-columns - # explanation: http://stackoverflow.com/questions/15165170/how-do-i-maintain-row-column-orientation-of-vectors-in-numpy?rq=1 - # we use kind of a hack - get information about memory from C_CONTIGUOUS - is_row = array.flags['C_CONTIGUOUS'] - - if is_row: - rows = 1 - cols = len(array) - if cols < len(array): - reslice = '[0:%s]' % (cols) - array = array[0:cols] - else: - cols = 1 - rows = len(array) - if rows < len(array): - reslice = '[0:%s]' % (rows) - array = array[0:rows] - elif l == 2: - rows = array.shape[-2] - cols = array.shape[-1] - if cols < array.shape[-1] or rows < array.shape[-2]: - reslice = '[0:%s, 0:%s]' % (rows, cols) - array = array[0:rows, 0:cols] - - # avoid slice duplication - if not slice.endswith(reslice): - slice += reslice - - bounds = (0, 0) - if type in "biufc": - bounds = (array.min(), array.max()) - # return array, slice_to_xml(slice, rows, cols, format, type, bounds), rows, cols, format - array_chunk = console_thrift.GetArrayResponse() - array_chunk.slice = slice - array_chunk.rows = rows - array_chunk.cols = cols - array_chunk.format = format - array_chunk.type = type - array_chunk.bounds = bounds - # return array, slice_to_xml(slice, rows, cols, format, type, bounds), rows, cols, format - return array, array_chunk, rows, cols, format - - -def dataframe_to_thrift_struct(df, name, roffset, coffset, rows, cols, format): - """ - :type df: pandas.core.frame.DataFrame - :type name: str - :type coffset: int - :type roffset: int - :type rows: int - :type cols: int - :type format: str - - - """ - dim = len(df.axes) - num_rows = df.shape[0] - num_cols = df.shape[1] if dim > 1 else 1 - # xml = slice_to_xml(name, num_rows, num_cols, "", "", (0, 0)) - array_chunk = console_thrift.GetArrayResponse() - array_chunk.slice = name - array_chunk.rows = num_rows - array_chunk.cols = num_cols - array_chunk.format = "" - array_chunk.type = "" - array_chunk.bounds = (0, 0) - - if (rows, cols) == (-1, -1): - rows, cols = num_rows, num_cols - - rows = min(rows, MAXIMUM_ARRAY_SIZE) - cols = min(cols, MAXIMUM_ARRAY_SIZE, num_cols) - # need to precompute column bounds here before slicing! - col_bounds = [None] * cols - dtypes = [None] * cols - if dim > 1: - for col in range(cols): - dtype = df.dtypes.iloc[coffset + col].kind - dtypes[col] = dtype - if dtype in "biufc": - cvalues = df.iloc[:, coffset + col] - bounds = (cvalues.min(), cvalues.max()) - else: - bounds = (0, 0) - col_bounds[col] = bounds - else: - dtype = df.dtype.kind - dtypes[0] = dtype - col_bounds[0] = (df.min(), df.max()) if dtype in "biufc" else (0, 0) - - df = df.iloc[roffset: roffset + rows, coffset: coffset + cols] if dim > 1 else df.iloc[roffset: roffset + rows] - rows = df.shape[0] - cols = df.shape[1] if dim > 1 else 1 - format = format.replace('%', '') - - def col_to_format(c): - return format if dtypes[c] == 'f' and format else array_default_format(dtypes[c]) - - # xml += header_data_to_xml(rows, cols, dtypes, col_bounds, col_to_format, df, dim) - array_chunk.headers = header_data_to_thrift_struct(rows, cols, dtypes, col_bounds, col_to_format, df, dim) - # xml += array_data_to_xml(rows, cols, lambda r: (("%" + col_to_format(c)) % (df.iat[r, c] if dim > 1 else df.iat[r]) - # for c in range(cols))) - array_chunk.data = array_data_to_thrift_struct(rows, cols, lambda r: (("%" + col_to_format(c)) % (df.iat[r, c] if dim > 1 else df.iat[r]) - for c in range(cols))) - # return xml - return array_chunk - - -def array_data_to_thrift_struct(rows, cols, get_row): - array_data = console_thrift.ArrayData() - # xml = "\n" % (rows, cols) - array_data.rows = rows - array_data.cols = cols - # `ArrayData.data` - data = [] - for row in range(rows): - # xml += "\n" % to_string(row) - # for value in get_row(row): - # xml += var_to_xml(value, '') - data.append([var_to_str(value) for value in get_row(row)]) - - array_data.data = data - # return xml - return array_data - - -def header_data_to_thrift_struct(rows, cols, dtypes, col_bounds, col_to_format, df, dim): - # xml = "\n" % (rows, cols) - array_headers = console_thrift.ArrayHeaders() - col_headers = [] - for col in range(cols): - col_label = get_label(df.axes[1].values[col]) if dim > 1 else str(col) - bounds = col_bounds[col] - col_format = "%" + col_to_format(col) - col_header = console_thrift.ColHeader() - # col_header.index = col - col_header.label = col_label - col_header.type = dtypes[col] - col_header.format = col_to_format(col) - col_header.max = col_format % bounds[1] - col_header.min = col_format % bounds[0] - col_headers.append(col_header) - row_headers = [] - for row in range(rows): - row_header = console_thrift.RowHeader() - row_header.index = row - row_header.label = get_label(df.axes[0].values[row]) - row_headers.append(row_header) - # xml += "\n" % (str(row), get_label(df.axes[0].values[row])) - # xml += "\n" - array_headers.colHeaders = col_headers - array_headers.rowHeaders = row_headers - # return xml - return array_headers - - -TYPE_TO_THRIFT_STRUCT_CONVERTERS = {"ndarray": array_to_thrift_struct, "DataFrame": dataframe_to_thrift_struct, "Series": dataframe_to_thrift_struct} - - -def table_like_struct_to_thrift_struct(array, name, roffset, coffset, rows, cols, format): - # returns `GetArrayResponse` - _, type_name, _ = get_type(array) - if type_name in TYPE_TO_THRIFT_STRUCT_CONVERTERS: - return TYPE_TO_THRIFT_STRUCT_CONVERTERS[type_name](array, name, roffset, coffset, rows, cols, format) - else: - raise VariableError("type %s not supported" % type_name)