IDEA-CR-52512: PY-26541 Fix formatting in data view panel

GitOrigin-RevId: ddcb33ebe7ad16c46def53c0cfd40e3bbcd27c77
This commit is contained in:
Andrey Lisin
2019-09-27 09:32:40 +00:00
committed by intellij-monorepo-bot
parent 34ea8fb853
commit 4e5852d366
7 changed files with 137 additions and 50 deletions
@@ -1238,7 +1238,7 @@ class InternalGetArray(InternalThreadCommand):
try:
frame = pydevd_vars.find_frame(self.thread_id, self.frame_id)
var = pydevd_vars.eval_in_context(self.name, frame.f_globals, frame.f_locals)
xml = pydevd_vars.table_like_struct_to_xml(var, self.name, self.roffset, self.coffset, self.rows, self.cols, self.format )
xml = pydevd_vars.table_like_struct_to_xml(var, self.name, self.roffset, self.coffset, self.rows, self.cols, self.format)
cmd = dbg.cmd_factory.make_get_array_message(self.sequence, xml)
dbg.writer.add_command(cmd)
except:
@@ -76,6 +76,8 @@ IS_MACOS = sys.platform == 'darwin'
IS_PYTHON_STACKLESS = "stackless" in sys.version.lower()
CYTHON_SUPPORTED = False
NUMPY_NUMERIC_TYPES = "biufc"
try:
import platform
python_implementation = platform.python_implementation()
@@ -11,9 +11,10 @@ from _pydev_bundle import pydev_log
from _pydevd_bundle import pydevd_extension_utils
from _pydevd_bundle import pydevd_resolver
from _pydevd_bundle.pydevd_constants import dict_iter_items, dict_keys, IS_PY3K, \
BUILTINS_MODULE_NAME, MAXIMUM_VARIABLE_REPRESENTATION_SIZE, RETURN_VALUES_DICT, LOAD_VALUES_POLICY, ValuesPolicy, DEFAULT_VALUES_DICT
BUILTINS_MODULE_NAME, MAXIMUM_VARIABLE_REPRESENTATION_SIZE, RETURN_VALUES_DICT, LOAD_VALUES_POLICY, ValuesPolicy, DEFAULT_VALUES_DICT, \
NUMPY_NUMERIC_TYPES
from _pydevd_bundle.pydevd_extension_api import TypeResolveProvider, StrPresentationProvider
from _pydevd_bundle.pydevd_vars import get_label, array_default_format, MAXIMUM_ARRAY_SIZE
from _pydevd_bundle.pydevd_vars import get_label, array_default_format, is_able_to_format_number, MAXIMUM_ARRAY_SIZE
from pydev_console.protocol import DebugValue, GetArrayResponse, ArrayData, ArrayHeaders, ColHeader, RowHeader, \
UnsupportedArrayTypeException, ExceedingArrayDimensionsException
from _pydevd_bundle.pydevd_utils import take_first_n_coll_elements
@@ -280,7 +281,7 @@ def frame_vars_to_struct(frame_f_locals, hidden_ns=None):
return return_values + values
def var_to_struct(val, name, do_trim=True, evaluate_full_value=True):
def var_to_struct(val, name, format='%s', do_trim=True, evaluate_full_value=True):
""" single variable or dictionary to Thrift struct representation """
debug_value = DebugValue()
@@ -317,7 +318,7 @@ def var_to_struct(val, name, do_trim=True, evaluate_full_value=True):
else:
value = '%s: %s' % (str(v.__class__, v))
else:
value = str(v)
value = format % v
else:
value = str(v)
except:
@@ -362,8 +363,8 @@ def var_to_struct(val, name, do_trim=True, evaluate_full_value=True):
return debug_value
def var_to_str(val, do_trim=True, evaluate_full_value=True):
struct = var_to_struct(val, '', do_trim, evaluate_full_value)
def var_to_str(val, format, do_trim=True, evaluate_full_value=True):
struct = var_to_struct(val, '', format, do_trim, evaluate_full_value)
value = struct.value
return value if value is not None else ''
@@ -413,7 +414,7 @@ def array_to_thrift_struct(array, name, roffset, coffset, rows, cols, format):
value = array[row][col]
return value
array_chunk.data = array_data_to_thrift_struct(rows, cols, lambda r: (get_value(r, c) for c in range(cols)))
array_chunk.data = array_data_to_thrift_struct(rows, cols, lambda r: (get_value(r, c) for c in range(cols)), format)
return array_chunk
@@ -472,13 +473,13 @@ def array_to_meta_thrift_struct(array, name, format):
slice += reslice
bounds = (0, 0)
if type in "biufc":
if type in NUMPY_NUMERIC_TYPES:
bounds = (array.min(), array.max())
array_chunk = GetArrayResponse()
array_chunk.slice = slice
array_chunk.rows = rows
array_chunk.cols = cols
array_chunk.format = format
array_chunk.format = "%" + format
array_chunk.type = type
array_chunk.max = "%s" % bounds[1]
array_chunk.min = "%s" % bounds[0]
@@ -504,10 +505,23 @@ def dataframe_to_thrift_struct(df, name, roffset, coffset, rows, cols, format):
array_chunk.slice = name
array_chunk.rows = num_rows
array_chunk.cols = num_cols
array_chunk.format = ""
array_chunk.type = ""
array_chunk.max = "0"
array_chunk.min = "0"
format = format.replace("%", "")
if not format:
if num_rows > 0 and num_cols == 1: # series or data frame with one column
try:
kind = df.dtype.kind
except AttributeError:
try:
kind = df.dtypes[0].kind
except IndexError:
kind = "O"
format = array_default_format(kind)
else:
format = array_default_format("f")
array_chunk.format = "%" + format
if (rows, cols) == (-1, -1):
rows, cols = num_rows, num_cols
@@ -521,7 +535,7 @@ def dataframe_to_thrift_struct(df, name, roffset, coffset, rows, cols, format):
for col in range(cols):
dtype = df.dtypes.iloc[coffset + col].kind
dtypes[col] = dtype
if dtype in "biufc":
if dtype in NUMPY_NUMERIC_TYPES:
cvalues = df.iloc[:, coffset + col]
bounds = (cvalues.min(), cvalues.max())
else:
@@ -530,33 +544,32 @@ def dataframe_to_thrift_struct(df, name, roffset, coffset, rows, cols, format):
else:
dtype = df.dtype.kind
dtypes[0] = dtype
col_bounds[0] = (df.min(), df.max()) if dtype in "biufc" else (0, 0)
col_bounds[0] = (df.min(), df.max()) if dtype in NUMPY_NUMERIC_TYPES else (0, 0)
df = df.iloc[roffset: roffset + rows, coffset: coffset + cols] if dim > 1 else df.iloc[roffset: roffset + rows]
rows = df.shape[0]
cols = df.shape[1] if dim > 1 else 1
format = format.replace('%', '')
def col_to_format(c):
return format if dtypes[c] == 'f' and format else array_default_format(dtypes[c])
return format if dtypes[c] in NUMPY_NUMERIC_TYPES and format else array_default_format(dtypes[c])
iat = df.iat if dim == 1 or len(df.columns.unique()) == len(df.columns) else df.iloc
array_chunk.headers = header_data_to_thrift_struct(rows, cols, dtypes, col_bounds, col_to_format, df, dim)
array_chunk.data = array_data_to_thrift_struct(rows, cols,
lambda r: (("%" + col_to_format(c)) % (iat[r, c] if dim > 1 else iat[r])
for c in range(cols)))
for c in range(cols)), format)
return array_chunk
def array_data_to_thrift_struct(rows, cols, get_row):
def array_data_to_thrift_struct(rows, cols, get_row, format):
array_data = ArrayData()
array_data.rows = rows
array_data.cols = cols
# `ArrayData.data`
data = []
for row in range(rows):
data.append([var_to_str(value) for value in get_row(row)])
data.append([var_to_str(value, format) for value in get_row(row)])
array_data.data = data
return array_data
@@ -598,6 +611,7 @@ def table_like_struct_to_thrift_struct(array, name, roffset, coffset, rows, cols
The `array` might be either `numpy.ndarray`, `pandas.DataFrame` or `pandas.Series`.
"""
_, type_name, _ = get_type(array)
format = format if is_able_to_format_number(format) else '%'
if type_name in TYPE_TO_THRIFT_STRUCT_CONVERTERS:
return TYPE_TO_THRIFT_STRUCT_CONVERTERS[type_name](array, name, roffset, coffset, rows, cols, format)
else:
@@ -1,12 +1,12 @@
""" pydevd_vars deals with variables:
resolution/conversion to XML.
"""
import math
import pickle
from _pydevd_bundle.pydevd_constants import get_frame, get_current_thread_id, xrange
from _pydev_imps._pydev_saved_modules import thread
from _pydevd_bundle.pydevd_constants import get_frame, get_current_thread_id, xrange
from _pydev_bundle.pydev_imports import quote
from _pydevd_bundle.pydevd_constants import get_frame, get_current_thread_id, xrange, NUMPY_NUMERIC_TYPES
from _pydevd_bundle.pydevd_custom_frames import get_custom_frame
from _pydevd_bundle.pydevd_xml import ExceptionOnEvaluate, get_type, var_to_xml
@@ -516,7 +516,7 @@ def array_to_xml(array, name, roffset, coffset, rows, cols, format):
else:
value = array[row][col]
return value
xml += array_data_to_xml(rows, cols, lambda r: (get_value(r, c) for c in range(cols)))
xml += array_data_to_xml(rows, cols, lambda r: (get_value(r, c) for c in range(cols)), format)
return xml
@@ -579,7 +579,7 @@ def array_to_meta_xml(array, name, format):
slice += reslice
bounds = (0, 0)
if type in "biufc":
if type in NUMPY_NUMERIC_TYPES:
bounds = (array.min(), array.max())
return array, slice_to_xml(slice, rows, cols, format, type, bounds), rows, cols, format
@@ -612,7 +612,22 @@ def dataframe_to_xml(df, name, roffset, coffset, rows, cols, format):
dim = len(df.axes)
num_rows = df.shape[0]
num_cols = df.shape[1] if dim > 1 else 1
xml = slice_to_xml(name, num_rows, num_cols, "", "", (0, 0))
format = format.replace('%', '')
if not format:
if num_rows > 0 and num_cols == 1: # series or data frame with one column
try:
kind = df.dtype.kind
except AttributeError:
try:
kind = df.dtypes[0].kind
except IndexError:
kind = 'O'
format = array_default_format(kind)
else:
format = array_default_format('f')
xml = slice_to_xml(name, num_rows, num_cols, format, "", (0, 0))
if (rows, cols) == (-1, -1):
rows, cols = num_rows, num_cols
@@ -626,7 +641,7 @@ def dataframe_to_xml(df, name, roffset, coffset, rows, cols, format):
for col in range(cols):
dtype = df.dtypes.iloc[coffset + col].kind
dtypes[col] = dtype
if dtype in "biufc":
if dtype in NUMPY_NUMERIC_TYPES:
cvalues = df.iloc[:, coffset + col]
bounds = (cvalues.min(), cvalues.max())
else:
@@ -635,36 +650,35 @@ def dataframe_to_xml(df, name, roffset, coffset, rows, cols, format):
else:
dtype = df.dtype.kind
dtypes[0] = dtype
col_bounds[0] = (df.min(), df.max()) if dtype in "biufc" else (0, 0)
col_bounds[0] = (df.min(), df.max()) if dtype in NUMPY_NUMERIC_TYPES else (0, 0)
df = df.iloc[roffset: roffset + rows, coffset: coffset + cols] if dim > 1 else df.iloc[roffset: roffset + rows]
rows = df.shape[0]
cols = df.shape[1] if dim > 1 else 1
format = format.replace('%', '')
def col_to_format(c):
return format if dtypes[c] == 'f' and format else array_default_format(dtypes[c])
return format if dtypes[c] in NUMPY_NUMERIC_TYPES and format else array_default_format(dtypes[c])
iat = df.iat if dim == 1 or len(df.columns.unique()) == len(df.columns) else df.iloc
xml += header_data_to_xml(rows, cols, dtypes, col_bounds, col_to_format, df, dim)
xml += array_data_to_xml(rows, cols, lambda r: (("%" + col_to_format(c)) % (iat[r, c] if dim > 1 else iat[r])
for c in range(cols)))
for c in range(cols)), format)
return xml
def array_data_to_xml(rows, cols, get_row):
def array_data_to_xml(rows, cols, get_row, format):
xml = "<arraydata rows=\"%s\" cols=\"%s\"/>\n" % (rows, cols)
for row in range(rows):
xml += "<row index=\"%s\"/>\n" % to_string(row)
for value in get_row(row):
xml += var_to_xml(value, '')
xml += var_to_xml(value, '', format=format)
return xml
def slice_to_xml(slice, rows, cols, format, type, bounds):
return '<array slice=\"%s\" rows=\"%s\" cols=\"%s\" format=\"%s\" type=\"%s\" max=\"%s\" min=\"%s\"/>' % \
(slice, rows, cols, format, type, bounds[1], bounds[0])
(slice, rows, cols, quote(format), type, bounds[1], bounds[0])
def header_data_to_xml(rows, cols, dtypes, col_bounds, col_to_format, df, dim):
@@ -680,11 +694,21 @@ def header_data_to_xml(rows, cols, dtypes, col_bounds, col_to_format, df, dim):
xml += "</headerdata>\n"
return xml
def is_able_to_format_number(format):
try:
format % math.pi
except Exception:
return False
return True
TYPE_TO_XML_CONVERTERS = {"ndarray": array_to_xml, "DataFrame": dataframe_to_xml, "Series": dataframe_to_xml}
def table_like_struct_to_xml(array, name, roffset, coffset, rows, cols, format):
_, type_name, _ = get_type(array)
format = format if is_able_to_format_number(format) else '%'
if type_name in TYPE_TO_XML_CONVERTERS:
return "<xml>%s</xml>" % TYPE_TO_XML_CONVERTERS[type_name](array, name, roffset, coffset, rows, cols, format)
else:
@@ -290,7 +290,7 @@ def frame_vars_to_xml(frame_f_locals, hidden_ns=None):
return return_values_xml + xml
def var_to_xml(val, name, doTrim=True, additional_in_xml='', evaluate_full_value=True):
def var_to_xml(val, name, doTrim=True, additional_in_xml='', evaluate_full_value=True, format='%s'):
""" single variable or dictionary to xml representation """
try:
@@ -338,7 +338,7 @@ def var_to_xml(val, name, doTrim=True, additional_in_xml='', evaluate_full_value
except:
cName = str(v.__class__)
value = '%s: %s' % (cName, v)
value = ('%s: ' + format) % (cName, v)
else:
value = str(v)
except:
+1 -1
View File
@@ -1,7 +1,7 @@
import pandas as pd
import numpy as np
frame = pd.DataFrame(data=np.random.randint(0, high=10, size=(4, 2)), columns=['a', 'b'], index=pd.MultiIndex([['s', 'd'], [2, 3]], [[0, 0, 1, 1], [0, 1, 0, 1]]))
frame = pd.DataFrame(data=np.arange(8, dtype='f').reshape((4, 2)), columns=['a', 'b'], index=pd.MultiIndex([['s', 'd'], [2, 3]], [[0, 0, 1, 1], [0, 1, 0, 1]]))
series = frame.a
@@ -54,8 +54,8 @@ public class PythonDataViewerTest extends PyEnvTestCase {
doTest("df3", 7, 3, arrayChunk -> {
ArrayChunk.ColHeader header = arrayChunk.getColHeaders().get(2);
assertEquals("Sales", header.getLabel());
assertEquals(16, (int)Integer.valueOf(header.getMax()));
assertEquals(1, (int)Integer.valueOf(header.getMin()));
assertEquals(16, Float.valueOf(header.getMax()).intValue());
assertEquals(1, Float.valueOf(header.getMin()).intValue());
});
}
});
@@ -96,14 +96,47 @@ public class PythonDataViewerTest extends PyEnvTestCase {
ImmutableSet.of(7)) {
@Override
public void testing() throws Exception {
doTest("c", 10, 2, arrayChunk -> {
doTest("c", 10, 2, (varName, session) -> getChunk(varName, "%d", session), arrayChunk -> {
for (ArrayChunk.ColHeader header : arrayChunk.getColHeaders())
assertEquals("A", header.getLabel());
Object[][] data = arrayChunk.getData();
assertEquals("0", data[0][0].toString());
assertEquals("0", data[0][1].toString());
assertEquals("6", data[6][0].toString());
assertEquals("9", data[9][0].toString());
assertEquals("'0'", data[0][0].toString());
assertEquals("'0'", data[0][1].toString());
assertEquals("'6'", data[6][0].toString());
assertEquals("'9'", data[9][0].toString());
});
}
});
}
@Test
@Staging
public void testDataFrameFormatting() {
runPythonTest(new PyDataFrameDebuggerTask(getRelativeTestDataPath(), "test_dataframe.py", ImmutableSet.of(7)) {
@Override
public void testing() throws Exception {
doTest("df1", 3, 5, (varName, session) -> getChunk(varName, "%.2f", session), arrayChunk -> {
Object[][] data = arrayChunk.getData();
assertEquals("'1.10'", data[0][1].toString());
assertEquals("'1.20'", data[0][2].toString());
assertEquals("'1.22'", data[1][4].toString());
});
}
});
}
@Test
@Staging
public void testSeriesFormatting() {
runPythonTest(new PyDataFrameDebuggerTask(getRelativeTestDataPath(), "test_series.py", ImmutableSet.of(7)) {
@Override
public void testing() throws Exception {
doTest("series", 4, 1, (varName, session) -> getChunk(varName, "%03d", session), arrayChunk -> {
Object[][] data = arrayChunk.getData();
assertEquals("'000'", data[0][0].toString());
assertEquals("'002'", data[1][0].toString());
assertEquals("'004'", data[2][0].toString());
assertEquals("'006'", data[3][0].toString());
});
}
});
@@ -113,6 +146,11 @@ public class PythonDataViewerTest extends PyEnvTestCase {
private final Set<Integer> myLines;
@FunctionalInterface
private interface ChunkSupplier {
ArrayChunk supply(String varName, XDebugSession session) throws PyDebuggerException;
}
PyDataFrameDebuggerTask(@Nullable String relativeTestDataPath, String scriptName, Set<Integer> lines) {
super(relativeTestDataPath, scriptName);
myLines = lines;
@@ -125,13 +163,18 @@ public class PythonDataViewerTest extends PyEnvTestCase {
protected void doTest(String name, int expectedRows, int expectedColumns, @Nullable Consumer<ArrayChunk> test)
throws InterruptedException, PyDebuggerException {
waitForPause();
ArrayChunk arrayChunk = getDefaultChunk(name, mySession);
testShape(arrayChunk, expectedRows, expectedColumns);
if (test != null) {
test.consume(arrayChunk);
}
resume();
doTest(name, expectedRows, expectedColumns, PythonDataViewerTest::getDefaultChunk, test);
}
protected void doTest(String name, int expectedRows, int expectedColumns, @NotNull ChunkSupplier getChunk,
@Nullable Consumer<ArrayChunk> test) throws InterruptedException, PyDebuggerException {
waitForPause();
ArrayChunk arrayChunk = getChunk.supply(name, mySession);
testShape(arrayChunk, expectedRows, expectedColumns);
if (test != null) {
test.consume(arrayChunk);
}
resume();
}
@Override
@@ -149,8 +192,12 @@ public class PythonDataViewerTest extends PyEnvTestCase {
}
private static ArrayChunk getDefaultChunk(String varName, XDebugSession session) throws PyDebuggerException {
return getChunk(varName, "%.5f", session);
}
private static ArrayChunk getChunk(String varName, String format, XDebugSession session) throws PyDebuggerException {
PyDebugValue dbgVal = (PyDebugValue)XDebuggerTestUtil.evaluate(session, varName).first;
return dbgVal.getFrameAccessor().getArrayItems(dbgVal, 0, 0, -1, -1, ".%5f");
return dbgVal.getFrameAccessor().getArrayItems(dbgVal, 0, 0, -1, -1, format);
}
private static String getRelativeTestDataPath() {