From 4e5852d366823faf545db646cceb53289064deea Mon Sep 17 00:00:00 2001 From: Andrey Lisin Date: Fri, 13 Sep 2019 20:05:10 +0300 Subject: [PATCH] IDEA-CR-52512: PY-26541 Fix formatting in data view panel GitOrigin-RevId: ddcb33ebe7ad16c46def53c0cfd40e3bbcd27c77 --- .../pydev/_pydevd_bundle/pydevd_comm.py | 2 +- .../pydev/_pydevd_bundle/pydevd_constants.py | 2 + .../pydev/_pydevd_bundle/pydevd_thrift.py | 48 ++++++++---- .../pydev/_pydevd_bundle/pydevd_vars.py | 52 +++++++++---- .../pydev/_pydevd_bundle/pydevd_xml.py | 4 +- python/testData/debug/test_series.py | 2 +- .../env/python/PythonDataViewerTest.java | 77 +++++++++++++++---- 7 files changed, 137 insertions(+), 50 deletions(-) diff --git a/python/helpers/pydev/_pydevd_bundle/pydevd_comm.py b/python/helpers/pydev/_pydevd_bundle/pydevd_comm.py index 3d4ac2cb106d..7db5183cd5df 100644 --- a/python/helpers/pydev/_pydevd_bundle/pydevd_comm.py +++ b/python/helpers/pydev/_pydevd_bundle/pydevd_comm.py @@ -1238,7 +1238,7 @@ class InternalGetArray(InternalThreadCommand): try: frame = pydevd_vars.find_frame(self.thread_id, self.frame_id) var = pydevd_vars.eval_in_context(self.name, frame.f_globals, frame.f_locals) - xml = pydevd_vars.table_like_struct_to_xml(var, self.name, self.roffset, self.coffset, self.rows, self.cols, self.format ) + xml = pydevd_vars.table_like_struct_to_xml(var, self.name, self.roffset, self.coffset, self.rows, self.cols, self.format) cmd = dbg.cmd_factory.make_get_array_message(self.sequence, xml) dbg.writer.add_command(cmd) except: diff --git a/python/helpers/pydev/_pydevd_bundle/pydevd_constants.py b/python/helpers/pydev/_pydevd_bundle/pydevd_constants.py index 1d782fc8e3a1..bb5555bb7d30 100644 --- a/python/helpers/pydev/_pydevd_bundle/pydevd_constants.py +++ b/python/helpers/pydev/_pydevd_bundle/pydevd_constants.py @@ -76,6 +76,8 @@ IS_MACOS = sys.platform == 'darwin' IS_PYTHON_STACKLESS = "stackless" in sys.version.lower() CYTHON_SUPPORTED = False +NUMPY_NUMERIC_TYPES = "biufc" + try: import platform python_implementation = platform.python_implementation() diff --git a/python/helpers/pydev/_pydevd_bundle/pydevd_thrift.py b/python/helpers/pydev/_pydevd_bundle/pydevd_thrift.py index 684fa3b2ee1b..c1769a80ffce 100644 --- a/python/helpers/pydev/_pydevd_bundle/pydevd_thrift.py +++ b/python/helpers/pydev/_pydevd_bundle/pydevd_thrift.py @@ -11,9 +11,10 @@ from _pydev_bundle import pydev_log from _pydevd_bundle import pydevd_extension_utils from _pydevd_bundle import pydevd_resolver from _pydevd_bundle.pydevd_constants import dict_iter_items, dict_keys, IS_PY3K, \ - BUILTINS_MODULE_NAME, MAXIMUM_VARIABLE_REPRESENTATION_SIZE, RETURN_VALUES_DICT, LOAD_VALUES_POLICY, ValuesPolicy, DEFAULT_VALUES_DICT + BUILTINS_MODULE_NAME, MAXIMUM_VARIABLE_REPRESENTATION_SIZE, RETURN_VALUES_DICT, LOAD_VALUES_POLICY, ValuesPolicy, DEFAULT_VALUES_DICT, \ + NUMPY_NUMERIC_TYPES from _pydevd_bundle.pydevd_extension_api import TypeResolveProvider, StrPresentationProvider -from _pydevd_bundle.pydevd_vars import get_label, array_default_format, MAXIMUM_ARRAY_SIZE +from _pydevd_bundle.pydevd_vars import get_label, array_default_format, is_able_to_format_number, MAXIMUM_ARRAY_SIZE from pydev_console.protocol import DebugValue, GetArrayResponse, ArrayData, ArrayHeaders, ColHeader, RowHeader, \ UnsupportedArrayTypeException, ExceedingArrayDimensionsException from _pydevd_bundle.pydevd_utils import take_first_n_coll_elements @@ -280,7 +281,7 @@ def frame_vars_to_struct(frame_f_locals, hidden_ns=None): return return_values + values -def var_to_struct(val, name, do_trim=True, evaluate_full_value=True): +def var_to_struct(val, name, format='%s', do_trim=True, evaluate_full_value=True): """ single variable or dictionary to Thrift struct representation """ debug_value = DebugValue() @@ -317,7 +318,7 @@ def var_to_struct(val, name, do_trim=True, evaluate_full_value=True): else: value = '%s: %s' % (str(v.__class__, v)) else: - value = str(v) + value = format % v else: value = str(v) except: @@ -362,8 +363,8 @@ def var_to_struct(val, name, do_trim=True, evaluate_full_value=True): return debug_value -def var_to_str(val, do_trim=True, evaluate_full_value=True): - struct = var_to_struct(val, '', do_trim, evaluate_full_value) +def var_to_str(val, format, do_trim=True, evaluate_full_value=True): + struct = var_to_struct(val, '', format, do_trim, evaluate_full_value) value = struct.value return value if value is not None else '' @@ -413,7 +414,7 @@ def array_to_thrift_struct(array, name, roffset, coffset, rows, cols, format): value = array[row][col] return value - array_chunk.data = array_data_to_thrift_struct(rows, cols, lambda r: (get_value(r, c) for c in range(cols))) + array_chunk.data = array_data_to_thrift_struct(rows, cols, lambda r: (get_value(r, c) for c in range(cols)), format) return array_chunk @@ -472,13 +473,13 @@ def array_to_meta_thrift_struct(array, name, format): slice += reslice bounds = (0, 0) - if type in "biufc": + if type in NUMPY_NUMERIC_TYPES: bounds = (array.min(), array.max()) array_chunk = GetArrayResponse() array_chunk.slice = slice array_chunk.rows = rows array_chunk.cols = cols - array_chunk.format = format + array_chunk.format = "%" + format array_chunk.type = type array_chunk.max = "%s" % bounds[1] array_chunk.min = "%s" % bounds[0] @@ -504,10 +505,23 @@ def dataframe_to_thrift_struct(df, name, roffset, coffset, rows, cols, format): array_chunk.slice = name array_chunk.rows = num_rows array_chunk.cols = num_cols - array_chunk.format = "" array_chunk.type = "" array_chunk.max = "0" array_chunk.min = "0" + format = format.replace("%", "") + if not format: + if num_rows > 0 and num_cols == 1: # series or data frame with one column + try: + kind = df.dtype.kind + except AttributeError: + try: + kind = df.dtypes[0].kind + except IndexError: + kind = "O" + format = array_default_format(kind) + else: + format = array_default_format("f") + array_chunk.format = "%" + format if (rows, cols) == (-1, -1): rows, cols = num_rows, num_cols @@ -521,7 +535,7 @@ def dataframe_to_thrift_struct(df, name, roffset, coffset, rows, cols, format): for col in range(cols): dtype = df.dtypes.iloc[coffset + col].kind dtypes[col] = dtype - if dtype in "biufc": + if dtype in NUMPY_NUMERIC_TYPES: cvalues = df.iloc[:, coffset + col] bounds = (cvalues.min(), cvalues.max()) else: @@ -530,33 +544,32 @@ def dataframe_to_thrift_struct(df, name, roffset, coffset, rows, cols, format): else: dtype = df.dtype.kind dtypes[0] = dtype - col_bounds[0] = (df.min(), df.max()) if dtype in "biufc" else (0, 0) + col_bounds[0] = (df.min(), df.max()) if dtype in NUMPY_NUMERIC_TYPES else (0, 0) df = df.iloc[roffset: roffset + rows, coffset: coffset + cols] if dim > 1 else df.iloc[roffset: roffset + rows] rows = df.shape[0] cols = df.shape[1] if dim > 1 else 1 - format = format.replace('%', '') def col_to_format(c): - return format if dtypes[c] == 'f' and format else array_default_format(dtypes[c]) + return format if dtypes[c] in NUMPY_NUMERIC_TYPES and format else array_default_format(dtypes[c]) iat = df.iat if dim == 1 or len(df.columns.unique()) == len(df.columns) else df.iloc array_chunk.headers = header_data_to_thrift_struct(rows, cols, dtypes, col_bounds, col_to_format, df, dim) array_chunk.data = array_data_to_thrift_struct(rows, cols, lambda r: (("%" + col_to_format(c)) % (iat[r, c] if dim > 1 else iat[r]) - for c in range(cols))) + for c in range(cols)), format) return array_chunk -def array_data_to_thrift_struct(rows, cols, get_row): +def array_data_to_thrift_struct(rows, cols, get_row, format): array_data = ArrayData() array_data.rows = rows array_data.cols = cols # `ArrayData.data` data = [] for row in range(rows): - data.append([var_to_str(value) for value in get_row(row)]) + data.append([var_to_str(value, format) for value in get_row(row)]) array_data.data = data return array_data @@ -598,6 +611,7 @@ def table_like_struct_to_thrift_struct(array, name, roffset, coffset, rows, cols The `array` might be either `numpy.ndarray`, `pandas.DataFrame` or `pandas.Series`. """ _, type_name, _ = get_type(array) + format = format if is_able_to_format_number(format) else '%' if type_name in TYPE_TO_THRIFT_STRUCT_CONVERTERS: return TYPE_TO_THRIFT_STRUCT_CONVERTERS[type_name](array, name, roffset, coffset, rows, cols, format) else: diff --git a/python/helpers/pydev/_pydevd_bundle/pydevd_vars.py b/python/helpers/pydev/_pydevd_bundle/pydevd_vars.py index a28f8c2b52e5..97c68fccbdd4 100644 --- a/python/helpers/pydev/_pydevd_bundle/pydevd_vars.py +++ b/python/helpers/pydev/_pydevd_bundle/pydevd_vars.py @@ -1,12 +1,12 @@ """ pydevd_vars deals with variables: resolution/conversion to XML. """ +import math import pickle -from _pydevd_bundle.pydevd_constants import get_frame, get_current_thread_id, xrange - from _pydev_imps._pydev_saved_modules import thread -from _pydevd_bundle.pydevd_constants import get_frame, get_current_thread_id, xrange +from _pydev_bundle.pydev_imports import quote +from _pydevd_bundle.pydevd_constants import get_frame, get_current_thread_id, xrange, NUMPY_NUMERIC_TYPES from _pydevd_bundle.pydevd_custom_frames import get_custom_frame from _pydevd_bundle.pydevd_xml import ExceptionOnEvaluate, get_type, var_to_xml @@ -516,7 +516,7 @@ def array_to_xml(array, name, roffset, coffset, rows, cols, format): else: value = array[row][col] return value - xml += array_data_to_xml(rows, cols, lambda r: (get_value(r, c) for c in range(cols))) + xml += array_data_to_xml(rows, cols, lambda r: (get_value(r, c) for c in range(cols)), format) return xml @@ -579,7 +579,7 @@ def array_to_meta_xml(array, name, format): slice += reslice bounds = (0, 0) - if type in "biufc": + if type in NUMPY_NUMERIC_TYPES: bounds = (array.min(), array.max()) return array, slice_to_xml(slice, rows, cols, format, type, bounds), rows, cols, format @@ -612,7 +612,22 @@ def dataframe_to_xml(df, name, roffset, coffset, rows, cols, format): dim = len(df.axes) num_rows = df.shape[0] num_cols = df.shape[1] if dim > 1 else 1 - xml = slice_to_xml(name, num_rows, num_cols, "", "", (0, 0)) + format = format.replace('%', '') + + if not format: + if num_rows > 0 and num_cols == 1: # series or data frame with one column + try: + kind = df.dtype.kind + except AttributeError: + try: + kind = df.dtypes[0].kind + except IndexError: + kind = 'O' + format = array_default_format(kind) + else: + format = array_default_format('f') + + xml = slice_to_xml(name, num_rows, num_cols, format, "", (0, 0)) if (rows, cols) == (-1, -1): rows, cols = num_rows, num_cols @@ -626,7 +641,7 @@ def dataframe_to_xml(df, name, roffset, coffset, rows, cols, format): for col in range(cols): dtype = df.dtypes.iloc[coffset + col].kind dtypes[col] = dtype - if dtype in "biufc": + if dtype in NUMPY_NUMERIC_TYPES: cvalues = df.iloc[:, coffset + col] bounds = (cvalues.min(), cvalues.max()) else: @@ -635,36 +650,35 @@ def dataframe_to_xml(df, name, roffset, coffset, rows, cols, format): else: dtype = df.dtype.kind dtypes[0] = dtype - col_bounds[0] = (df.min(), df.max()) if dtype in "biufc" else (0, 0) + col_bounds[0] = (df.min(), df.max()) if dtype in NUMPY_NUMERIC_TYPES else (0, 0) df = df.iloc[roffset: roffset + rows, coffset: coffset + cols] if dim > 1 else df.iloc[roffset: roffset + rows] rows = df.shape[0] cols = df.shape[1] if dim > 1 else 1 - format = format.replace('%', '') def col_to_format(c): - return format if dtypes[c] == 'f' and format else array_default_format(dtypes[c]) + return format if dtypes[c] in NUMPY_NUMERIC_TYPES and format else array_default_format(dtypes[c]) iat = df.iat if dim == 1 or len(df.columns.unique()) == len(df.columns) else df.iloc xml += header_data_to_xml(rows, cols, dtypes, col_bounds, col_to_format, df, dim) xml += array_data_to_xml(rows, cols, lambda r: (("%" + col_to_format(c)) % (iat[r, c] if dim > 1 else iat[r]) - for c in range(cols))) + for c in range(cols)), format) return xml -def array_data_to_xml(rows, cols, get_row): +def array_data_to_xml(rows, cols, get_row, format): xml = "\n" % (rows, cols) for row in range(rows): xml += "\n" % to_string(row) for value in get_row(row): - xml += var_to_xml(value, '') + xml += var_to_xml(value, '', format=format) return xml def slice_to_xml(slice, rows, cols, format, type, bounds): return '' % \ - (slice, rows, cols, format, type, bounds[1], bounds[0]) + (slice, rows, cols, quote(format), type, bounds[1], bounds[0]) def header_data_to_xml(rows, cols, dtypes, col_bounds, col_to_format, df, dim): @@ -680,11 +694,21 @@ def header_data_to_xml(rows, cols, dtypes, col_bounds, col_to_format, df, dim): xml += "\n" return xml + +def is_able_to_format_number(format): + try: + format % math.pi + except Exception: + return False + return True + + TYPE_TO_XML_CONVERTERS = {"ndarray": array_to_xml, "DataFrame": dataframe_to_xml, "Series": dataframe_to_xml} def table_like_struct_to_xml(array, name, roffset, coffset, rows, cols, format): _, type_name, _ = get_type(array) + format = format if is_able_to_format_number(format) else '%' if type_name in TYPE_TO_XML_CONVERTERS: return "%s" % TYPE_TO_XML_CONVERTERS[type_name](array, name, roffset, coffset, rows, cols, format) else: diff --git a/python/helpers/pydev/_pydevd_bundle/pydevd_xml.py b/python/helpers/pydev/_pydevd_bundle/pydevd_xml.py index ad203178471e..2aa3e5fc4d16 100644 --- a/python/helpers/pydev/_pydevd_bundle/pydevd_xml.py +++ b/python/helpers/pydev/_pydevd_bundle/pydevd_xml.py @@ -290,7 +290,7 @@ def frame_vars_to_xml(frame_f_locals, hidden_ns=None): return return_values_xml + xml -def var_to_xml(val, name, doTrim=True, additional_in_xml='', evaluate_full_value=True): +def var_to_xml(val, name, doTrim=True, additional_in_xml='', evaluate_full_value=True, format='%s'): """ single variable or dictionary to xml representation """ try: @@ -338,7 +338,7 @@ def var_to_xml(val, name, doTrim=True, additional_in_xml='', evaluate_full_value except: cName = str(v.__class__) - value = '%s: %s' % (cName, v) + value = ('%s: ' + format) % (cName, v) else: value = str(v) except: diff --git a/python/testData/debug/test_series.py b/python/testData/debug/test_series.py index 151c1211927d..e99873889242 100644 --- a/python/testData/debug/test_series.py +++ b/python/testData/debug/test_series.py @@ -1,7 +1,7 @@ import pandas as pd import numpy as np -frame = pd.DataFrame(data=np.random.randint(0, high=10, size=(4, 2)), columns=['a', 'b'], index=pd.MultiIndex([['s', 'd'], [2, 3]], [[0, 0, 1, 1], [0, 1, 0, 1]])) +frame = pd.DataFrame(data=np.arange(8, dtype='f').reshape((4, 2)), columns=['a', 'b'], index=pd.MultiIndex([['s', 'd'], [2, 3]], [[0, 0, 1, 1], [0, 1, 0, 1]])) series = frame.a diff --git a/python/testSrc/com/jetbrains/env/python/PythonDataViewerTest.java b/python/testSrc/com/jetbrains/env/python/PythonDataViewerTest.java index 93529df759b7..6cf70a0431ea 100644 --- a/python/testSrc/com/jetbrains/env/python/PythonDataViewerTest.java +++ b/python/testSrc/com/jetbrains/env/python/PythonDataViewerTest.java @@ -54,8 +54,8 @@ public class PythonDataViewerTest extends PyEnvTestCase { doTest("df3", 7, 3, arrayChunk -> { ArrayChunk.ColHeader header = arrayChunk.getColHeaders().get(2); assertEquals("Sales", header.getLabel()); - assertEquals(16, (int)Integer.valueOf(header.getMax())); - assertEquals(1, (int)Integer.valueOf(header.getMin())); + assertEquals(16, Float.valueOf(header.getMax()).intValue()); + assertEquals(1, Float.valueOf(header.getMin()).intValue()); }); } }); @@ -96,14 +96,47 @@ public class PythonDataViewerTest extends PyEnvTestCase { ImmutableSet.of(7)) { @Override public void testing() throws Exception { - doTest("c", 10, 2, arrayChunk -> { + doTest("c", 10, 2, (varName, session) -> getChunk(varName, "%d", session), arrayChunk -> { for (ArrayChunk.ColHeader header : arrayChunk.getColHeaders()) assertEquals("A", header.getLabel()); Object[][] data = arrayChunk.getData(); - assertEquals("0", data[0][0].toString()); - assertEquals("0", data[0][1].toString()); - assertEquals("6", data[6][0].toString()); - assertEquals("9", data[9][0].toString()); + assertEquals("'0'", data[0][0].toString()); + assertEquals("'0'", data[0][1].toString()); + assertEquals("'6'", data[6][0].toString()); + assertEquals("'9'", data[9][0].toString()); + }); + } + }); + } + + @Test + @Staging + public void testDataFrameFormatting() { + runPythonTest(new PyDataFrameDebuggerTask(getRelativeTestDataPath(), "test_dataframe.py", ImmutableSet.of(7)) { + @Override + public void testing() throws Exception { + doTest("df1", 3, 5, (varName, session) -> getChunk(varName, "%.2f", session), arrayChunk -> { + Object[][] data = arrayChunk.getData(); + assertEquals("'1.10'", data[0][1].toString()); + assertEquals("'1.20'", data[0][2].toString()); + assertEquals("'1.22'", data[1][4].toString()); + }); + } + }); + } + + @Test + @Staging + public void testSeriesFormatting() { + runPythonTest(new PyDataFrameDebuggerTask(getRelativeTestDataPath(), "test_series.py", ImmutableSet.of(7)) { + @Override + public void testing() throws Exception { + doTest("series", 4, 1, (varName, session) -> getChunk(varName, "%03d", session), arrayChunk -> { + Object[][] data = arrayChunk.getData(); + assertEquals("'000'", data[0][0].toString()); + assertEquals("'002'", data[1][0].toString()); + assertEquals("'004'", data[2][0].toString()); + assertEquals("'006'", data[3][0].toString()); }); } }); @@ -113,6 +146,11 @@ public class PythonDataViewerTest extends PyEnvTestCase { private final Set myLines; + @FunctionalInterface + private interface ChunkSupplier { + ArrayChunk supply(String varName, XDebugSession session) throws PyDebuggerException; + } + PyDataFrameDebuggerTask(@Nullable String relativeTestDataPath, String scriptName, Set lines) { super(relativeTestDataPath, scriptName); myLines = lines; @@ -125,13 +163,18 @@ public class PythonDataViewerTest extends PyEnvTestCase { protected void doTest(String name, int expectedRows, int expectedColumns, @Nullable Consumer test) throws InterruptedException, PyDebuggerException { - waitForPause(); - ArrayChunk arrayChunk = getDefaultChunk(name, mySession); - testShape(arrayChunk, expectedRows, expectedColumns); - if (test != null) { - test.consume(arrayChunk); - } - resume(); + doTest(name, expectedRows, expectedColumns, PythonDataViewerTest::getDefaultChunk, test); + } + + protected void doTest(String name, int expectedRows, int expectedColumns, @NotNull ChunkSupplier getChunk, + @Nullable Consumer test) throws InterruptedException, PyDebuggerException { + waitForPause(); + ArrayChunk arrayChunk = getChunk.supply(name, mySession); + testShape(arrayChunk, expectedRows, expectedColumns); + if (test != null) { + test.consume(arrayChunk); + } + resume(); } @Override @@ -149,8 +192,12 @@ public class PythonDataViewerTest extends PyEnvTestCase { } private static ArrayChunk getDefaultChunk(String varName, XDebugSession session) throws PyDebuggerException { + return getChunk(varName, "%.5f", session); + } + + private static ArrayChunk getChunk(String varName, String format, XDebugSession session) throws PyDebuggerException { PyDebugValue dbgVal = (PyDebugValue)XDebuggerTestUtil.evaluate(session, varName).first; - return dbgVal.getFrameAccessor().getArrayItems(dbgVal, 0, 0, -1, -1, ".%5f"); + return dbgVal.getFrameAccessor().getArrayItems(dbgVal, 0, 0, -1, -1, format); } private static String getRelativeTestDataPath() {