PY-18029 Move methods for building Python console protocol structures from pydevd_vars to pydevd_thrift

This commit is contained in:
Alexander Koshevoy
2018-08-22 23:16:40 +03:00
parent 338b9fc572
commit 0212297426
3 changed files with 245 additions and 243 deletions
@@ -515,7 +515,7 @@ class BaseInterpreterInterface:
name = attr.split("\t")[-1]
array = pydevd_vars.eval_in_context(name, self.get_namespace(), self.get_namespace())
# return pydevd_vars.table_like_struct_to_xml(array, name, roffset, coffset, rows, cols, format)
return pydevd_vars.table_like_struct_to_thrift_struct(array, name, roffset, coffset, rows, cols, format)
return pydevd_thrift.table_like_struct_to_thrift_struct(array, name, roffset, coffset, rows, cols, format)
def evaluate(self, expression):
# returns `DebugValue` of evaluated expression
@@ -7,6 +7,7 @@ from _pydevd_bundle import pydevd_resolver
from _pydevd_bundle.pydevd_constants import dict_iter_items, dict_keys, IS_PY3K, \
BUILTINS_MODULE_NAME, MAXIMUM_VARIABLE_REPRESENTATION_SIZE, RETURN_VALUES_DICT, LOAD_VALUES_POLICY, ValuesPolicy, DEFAULT_VALUES_DICT
from _pydevd_bundle.pydevd_extension_api import TypeResolveProvider, StrPresentationProvider
from _pydevd_bundle.pydevd_vars import get_label, VariableError, array_default_format, MAXIMUM_ARRAY_SIZE
from pydev_console.thrift_communication import console_thrift
try:
@@ -367,3 +368,246 @@ def var_to_struct(val, name, doTrim=True, additional_in_xml='', evaluate_full_va
def var_to_str(val, doTrim=True, evaluate_full_value=True):
struct = var_to_struct(val, '', doTrim, '', evaluate_full_value)
return struct.value
# from pydevd_vars.py
def array_to_thrift_struct(array, name, roffset, coffset, rows, cols, format):
# returns `GetArrayResponse`
# array, xml, r, c, f = array_to_meta_xml(array, name, format)
array, array_chunk, r, c, f = array_to_meta_thrift_struct(array, name, format)
format = '%' + f
if rows == -1 and cols == -1:
rows = r
cols = c
rows = min(rows, MAXIMUM_ARRAY_SIZE)
cols = min(cols, MAXIMUM_ARRAY_SIZE)
# there is no obvious rule for slicing (at least 5 choices)
if len(array) == 1 and (rows > 1 or cols > 1):
array = array[0]
if array.size > len(array):
array = array[roffset:, coffset:]
rows = min(rows, len(array))
cols = min(cols, len(array[0]))
if len(array) == 1:
array = array[0]
elif array.size == len(array):
if roffset == 0 and rows == 1:
array = array[coffset:]
cols = min(cols, len(array))
elif coffset == 0 and cols == 1:
array = array[roffset:]
rows = min(rows, len(array))
def get_value(row, col):
value = array
if rows == 1 or cols == 1:
if rows == 1 and cols == 1:
value = array[0]
else:
value = array[(col if rows == 1 else row)]
if "ndarray" in str(type(value)):
value = value[0]
else:
value = array[row][col]
return value
# xml += array_data_to_xml(rows, cols, lambda r: (get_value(r, c) for c in range(cols)))
array_chunk.data = array_data_to_thrift_struct(rows, cols, lambda r: (get_value(r, c) for c in range(cols)))
return array_chunk
def array_to_meta_thrift_struct(array, name, format):
type = array.dtype.kind
slice = name
l = len(array.shape)
# initial load, compute slice
if format == '%':
if l > 2:
slice += '[0]' * (l - 2)
for r in range(l - 2):
array = array[0]
if type == 'f':
format = '.5f'
elif type == 'i' or type == 'u':
format = 'd'
else:
format = 's'
else:
format = format.replace('%', '')
l = len(array.shape)
reslice = ""
if l > 2:
raise Exception("%s has more than 2 dimensions." % slice)
elif l == 1:
# special case with 1D arrays arr[i, :] - row, but arr[:, i] - column with equal shape and ndim
# http://stackoverflow.com/questions/16837946/numpy-a-2-rows-1-column-file-loadtxt-returns-1row-2-columns
# explanation: http://stackoverflow.com/questions/15165170/how-do-i-maintain-row-column-orientation-of-vectors-in-numpy?rq=1
# we use kind of a hack - get information about memory from C_CONTIGUOUS
is_row = array.flags['C_CONTIGUOUS']
if is_row:
rows = 1
cols = len(array)
if cols < len(array):
reslice = '[0:%s]' % (cols)
array = array[0:cols]
else:
cols = 1
rows = len(array)
if rows < len(array):
reslice = '[0:%s]' % (rows)
array = array[0:rows]
elif l == 2:
rows = array.shape[-2]
cols = array.shape[-1]
if cols < array.shape[-1] or rows < array.shape[-2]:
reslice = '[0:%s, 0:%s]' % (rows, cols)
array = array[0:rows, 0:cols]
# avoid slice duplication
if not slice.endswith(reslice):
slice += reslice
bounds = (0, 0)
if type in "biufc":
bounds = (array.min(), array.max())
# return array, slice_to_xml(slice, rows, cols, format, type, bounds), rows, cols, format
array_chunk = console_thrift.GetArrayResponse()
array_chunk.slice = slice
array_chunk.rows = rows
array_chunk.cols = cols
array_chunk.format = format
array_chunk.type = type
array_chunk.bounds = bounds
# return array, slice_to_xml(slice, rows, cols, format, type, bounds), rows, cols, format
return array, array_chunk, rows, cols, format
def dataframe_to_thrift_struct(df, name, roffset, coffset, rows, cols, format):
"""
:type df: pandas.core.frame.DataFrame
:type name: str
:type coffset: int
:type roffset: int
:type rows: int
:type cols: int
:type format: str
"""
dim = len(df.axes)
num_rows = df.shape[0]
num_cols = df.shape[1] if dim > 1 else 1
# xml = slice_to_xml(name, num_rows, num_cols, "", "", (0, 0))
array_chunk = console_thrift.GetArrayResponse()
array_chunk.slice = name
array_chunk.rows = num_rows
array_chunk.cols = num_cols
array_chunk.format = ""
array_chunk.type = ""
array_chunk.bounds = (0, 0)
if (rows, cols) == (-1, -1):
rows, cols = num_rows, num_cols
rows = min(rows, MAXIMUM_ARRAY_SIZE)
cols = min(cols, MAXIMUM_ARRAY_SIZE, num_cols)
# need to precompute column bounds here before slicing!
col_bounds = [None] * cols
dtypes = [None] * cols
if dim > 1:
for col in range(cols):
dtype = df.dtypes.iloc[coffset + col].kind
dtypes[col] = dtype
if dtype in "biufc":
cvalues = df.iloc[:, coffset + col]
bounds = (cvalues.min(), cvalues.max())
else:
bounds = (0, 0)
col_bounds[col] = bounds
else:
dtype = df.dtype.kind
dtypes[0] = dtype
col_bounds[0] = (df.min(), df.max()) if dtype in "biufc" else (0, 0)
df = df.iloc[roffset: roffset + rows, coffset: coffset + cols] if dim > 1 else df.iloc[roffset: roffset + rows]
rows = df.shape[0]
cols = df.shape[1] if dim > 1 else 1
format = format.replace('%', '')
def col_to_format(c):
return format if dtypes[c] == 'f' and format else array_default_format(dtypes[c])
# xml += header_data_to_xml(rows, cols, dtypes, col_bounds, col_to_format, df, dim)
array_chunk.headers = header_data_to_thrift_struct(rows, cols, dtypes, col_bounds, col_to_format, df, dim)
# xml += array_data_to_xml(rows, cols, lambda r: (("%" + col_to_format(c)) % (df.iat[r, c] if dim > 1 else df.iat[r])
# for c in range(cols)))
array_chunk.data = array_data_to_thrift_struct(rows, cols, lambda r: (("%" + col_to_format(c)) % (df.iat[r, c] if dim > 1 else df.iat[r])
for c in range(cols)))
# return xml
return array_chunk
def array_data_to_thrift_struct(rows, cols, get_row):
array_data = console_thrift.ArrayData()
# xml = "<arraydata rows=\"%s\" cols=\"%s\"/>\n" % (rows, cols)
array_data.rows = rows
array_data.cols = cols
# `ArrayData.data`
data = []
for row in range(rows):
# xml += "<row index=\"%s\"/>\n" % to_string(row)
# for value in get_row(row):
# xml += var_to_xml(value, '')
data.append([var_to_str(value) for value in get_row(row)])
array_data.data = data
# return xml
return array_data
def header_data_to_thrift_struct(rows, cols, dtypes, col_bounds, col_to_format, df, dim):
# xml = "<headerdata rows=\"%s\" cols=\"%s\">\n" % (rows, cols)
array_headers = console_thrift.ArrayHeaders()
col_headers = []
for col in range(cols):
col_label = get_label(df.axes[1].values[col]) if dim > 1 else str(col)
bounds = col_bounds[col]
col_format = "%" + col_to_format(col)
col_header = console_thrift.ColHeader()
# col_header.index = col
col_header.label = col_label
col_header.type = dtypes[col]
col_header.format = col_to_format(col)
col_header.max = col_format % bounds[1]
col_header.min = col_format % bounds[0]
col_headers.append(col_header)
row_headers = []
for row in range(rows):
row_header = console_thrift.RowHeader()
row_header.index = row
row_header.label = get_label(df.axes[0].values[row])
row_headers.append(row_header)
# xml += "<rowheader index=\"%s\" label = \"%s\"/>\n" % (str(row), get_label(df.axes[0].values[row]))
# xml += "</headerdata>\n"
array_headers.colHeaders = col_headers
array_headers.rowHeaders = row_headers
# return xml
return array_headers
TYPE_TO_THRIFT_STRUCT_CONVERTERS = {"ndarray": array_to_thrift_struct, "DataFrame": dataframe_to_thrift_struct, "Series": dataframe_to_thrift_struct}
def table_like_struct_to_thrift_struct(array, name, roffset, coffset, rows, cols, format):
# returns `GetArrayResponse`
_, type_name, _ = get_type(array)
if type_name in TYPE_TO_THRIFT_STRUCT_CONVERTERS:
return TYPE_TO_THRIFT_STRUCT_CONVERTERS[type_name](array, name, roffset, coffset, rows, cols, format)
else:
raise VariableError("type %s not supported" % type_name)
@@ -6,9 +6,7 @@ import pickle
from _pydev_imps._pydev_saved_modules import thread
from _pydevd_bundle.pydevd_constants import get_frame, get_thread_id, xrange
from _pydevd_bundle.pydevd_custom_frames import get_custom_frame
from _pydevd_bundle.pydevd_thrift import var_to_str
from _pydevd_bundle.pydevd_xml import ExceptionOnEvaluate, get_type, var_to_xml
from pydev_console.thrift_communication import console_thrift
try:
from StringIO import StringIO
@@ -645,243 +643,3 @@ def table_like_struct_to_xml(array, name, roffset, coffset, rows, cols, format):
else:
raise VariableError("type %s not supported" % type_name)
def array_to_thrift_struct(array, name, roffset, coffset, rows, cols, format):
# returns `GetArrayResponse`
# array, xml, r, c, f = array_to_meta_xml(array, name, format)
array, array_chunk, r, c, f = array_to_meta_thrift_struct(array, name, format)
format = '%' + f
if rows == -1 and cols == -1:
rows = r
cols = c
rows = min(rows, MAXIMUM_ARRAY_SIZE)
cols = min(cols, MAXIMUM_ARRAY_SIZE)
# there is no obvious rule for slicing (at least 5 choices)
if len(array) == 1 and (rows > 1 or cols > 1):
array = array[0]
if array.size > len(array):
array = array[roffset:, coffset:]
rows = min(rows, len(array))
cols = min(cols, len(array[0]))
if len(array) == 1:
array = array[0]
elif array.size == len(array):
if roffset == 0 and rows == 1:
array = array[coffset:]
cols = min(cols, len(array))
elif coffset == 0 and cols == 1:
array = array[roffset:]
rows = min(rows, len(array))
def get_value(row, col):
value = array
if rows == 1 or cols == 1:
if rows == 1 and cols == 1:
value = array[0]
else:
value = array[(col if rows == 1 else row)]
if "ndarray" in str(type(value)):
value = value[0]
else:
value = array[row][col]
return value
# xml += array_data_to_xml(rows, cols, lambda r: (get_value(r, c) for c in range(cols)))
array_chunk.data = array_data_to_thrift_struct(rows, cols, lambda r: (get_value(r, c) for c in range(cols)))
return array_chunk
def array_to_meta_thrift_struct(array, name, format):
type = array.dtype.kind
slice = name
l = len(array.shape)
# initial load, compute slice
if format == '%':
if l > 2:
slice += '[0]' * (l - 2)
for r in range(l - 2):
array = array[0]
if type == 'f':
format = '.5f'
elif type == 'i' or type == 'u':
format = 'd'
else:
format = 's'
else:
format = format.replace('%', '')
l = len(array.shape)
reslice = ""
if l > 2:
raise Exception("%s has more than 2 dimensions." % slice)
elif l == 1:
# special case with 1D arrays arr[i, :] - row, but arr[:, i] - column with equal shape and ndim
# http://stackoverflow.com/questions/16837946/numpy-a-2-rows-1-column-file-loadtxt-returns-1row-2-columns
# explanation: http://stackoverflow.com/questions/15165170/how-do-i-maintain-row-column-orientation-of-vectors-in-numpy?rq=1
# we use kind of a hack - get information about memory from C_CONTIGUOUS
is_row = array.flags['C_CONTIGUOUS']
if is_row:
rows = 1
cols = len(array)
if cols < len(array):
reslice = '[0:%s]' % (cols)
array = array[0:cols]
else:
cols = 1
rows = len(array)
if rows < len(array):
reslice = '[0:%s]' % (rows)
array = array[0:rows]
elif l == 2:
rows = array.shape[-2]
cols = array.shape[-1]
if cols < array.shape[-1] or rows < array.shape[-2]:
reslice = '[0:%s, 0:%s]' % (rows, cols)
array = array[0:rows, 0:cols]
# avoid slice duplication
if not slice.endswith(reslice):
slice += reslice
bounds = (0, 0)
if type in "biufc":
bounds = (array.min(), array.max())
# return array, slice_to_xml(slice, rows, cols, format, type, bounds), rows, cols, format
array_chunk = console_thrift.GetArrayResponse()
array_chunk.slice = slice
array_chunk.rows = rows
array_chunk.cols = cols
array_chunk.format = format
array_chunk.type = type
array_chunk.bounds = bounds
# return array, slice_to_xml(slice, rows, cols, format, type, bounds), rows, cols, format
return array, array_chunk, rows, cols, format
def dataframe_to_thrift_struct(df, name, roffset, coffset, rows, cols, format):
"""
:type df: pandas.core.frame.DataFrame
:type name: str
:type coffset: int
:type roffset: int
:type rows: int
:type cols: int
:type format: str
"""
dim = len(df.axes)
num_rows = df.shape[0]
num_cols = df.shape[1] if dim > 1 else 1
# xml = slice_to_xml(name, num_rows, num_cols, "", "", (0, 0))
array_chunk = console_thrift.GetArrayResponse()
array_chunk.slice = name
array_chunk.rows = num_rows
array_chunk.cols = num_cols
array_chunk.format = ""
array_chunk.type = ""
array_chunk.bounds = (0, 0)
if (rows, cols) == (-1, -1):
rows, cols = num_rows, num_cols
rows = min(rows, MAXIMUM_ARRAY_SIZE)
cols = min(cols, MAXIMUM_ARRAY_SIZE, num_cols)
# need to precompute column bounds here before slicing!
col_bounds = [None] * cols
dtypes = [None] * cols
if dim > 1:
for col in range(cols):
dtype = df.dtypes.iloc[coffset + col].kind
dtypes[col] = dtype
if dtype in "biufc":
cvalues = df.iloc[:, coffset + col]
bounds = (cvalues.min(), cvalues.max())
else:
bounds = (0, 0)
col_bounds[col] = bounds
else:
dtype = df.dtype.kind
dtypes[0] = dtype
col_bounds[0] = (df.min(), df.max()) if dtype in "biufc" else (0, 0)
df = df.iloc[roffset: roffset + rows, coffset: coffset + cols] if dim > 1 else df.iloc[roffset: roffset + rows]
rows = df.shape[0]
cols = df.shape[1] if dim > 1 else 1
format = format.replace('%', '')
def col_to_format(c):
return format if dtypes[c] == 'f' and format else array_default_format(dtypes[c])
# xml += header_data_to_xml(rows, cols, dtypes, col_bounds, col_to_format, df, dim)
array_chunk.headers = header_data_to_thrift_struct(rows, cols, dtypes, col_bounds, col_to_format, df, dim)
# xml += array_data_to_xml(rows, cols, lambda r: (("%" + col_to_format(c)) % (df.iat[r, c] if dim > 1 else df.iat[r])
# for c in range(cols)))
array_chunk.data = array_data_to_thrift_struct(rows, cols, lambda r: (("%" + col_to_format(c)) % (df.iat[r, c] if dim > 1 else df.iat[r])
for c in range(cols)))
# return xml
return array_chunk
def array_data_to_thrift_struct(rows, cols, get_row):
array_data = console_thrift.ArrayData()
# xml = "<arraydata rows=\"%s\" cols=\"%s\"/>\n" % (rows, cols)
array_data.rows = rows
array_data.cols = cols
# `ArrayData.data`
data = []
for row in range(rows):
# xml += "<row index=\"%s\"/>\n" % to_string(row)
# for value in get_row(row):
# xml += var_to_xml(value, '')
data.append([var_to_str(value) for value in get_row(row)])
array_data.data = data
# return xml
return array_data
def header_data_to_thrift_struct(rows, cols, dtypes, col_bounds, col_to_format, df, dim):
# xml = "<headerdata rows=\"%s\" cols=\"%s\">\n" % (rows, cols)
array_headers = console_thrift.ArrayHeaders()
col_headers = []
for col in range(cols):
col_label = get_label(df.axes[1].values[col]) if dim > 1 else str(col)
bounds = col_bounds[col]
col_format = "%" + col_to_format(col)
col_header = console_thrift.ColHeader()
# col_header.index = col
col_header.label = col_label
col_header.type = dtypes[col]
col_header.format = col_to_format(col)
col_header.max = col_format % bounds[1]
col_header.min = col_format % bounds[0]
col_headers.append(col_header)
row_headers = []
for row in range(rows):
row_header = console_thrift.RowHeader()
row_header.index = row
row_header.label = get_label(df.axes[0].values[row])
row_headers.append(row_header)
# xml += "<rowheader index=\"%s\" label = \"%s\"/>\n" % (str(row), get_label(df.axes[0].values[row]))
# xml += "</headerdata>\n"
array_headers.colHeaders = col_headers
array_headers.rowHeaders = row_headers
# return xml
return array_headers
TYPE_TO_THRIFT_STRUCT_CONVERTERS = {"ndarray": array_to_thrift_struct, "DataFrame": dataframe_to_thrift_struct, "Series": dataframe_to_thrift_struct}
def table_like_struct_to_thrift_struct(array, name, roffset, coffset, rows, cols, format):
# returns `GetArrayResponse`
_, type_name, _ = get_type(array)
if type_name in TYPE_TO_THRIFT_STRUCT_CONVERTERS:
return TYPE_TO_THRIFT_STRUCT_CONVERTERS[type_name](array, name, roffset, coffset, rows, cols, format)
else:
raise VariableError("type %s not supported" % type_name)