From 1693fd94b9043bc370b340cd878db9440e40e394 Mon Sep 17 00:00:00 2001 From: "ekaterina.itsenko" Date: Tue, 11 Jun 2024 04:03:23 +0200 Subject: [PATCH] [pycharm] PY-38294 Add Sparse tensors support GitOrigin-RevId: 0b9b046ae1da37b5fd34be800fbdd6f241f7514d --- .../outputs/statistic/NotebookOutputKeyType.kt | 1 + python/helpers/pydev/_pydevd_bundle/pydevd_tables.py | 1 + python/helpers/pydev/_pydevd_bundle/pydevd_thrift.py | 6 ++++++ python/helpers/pydev/_pydevd_bundle/pydevd_vars.py | 6 ++++++ python/helpers/pydev/_pydevd_bundle/tables/pydevd_numpy.py | 7 ++++++- .../src/com/jetbrains/python/debugger/PyDebugValue.java | 1 + .../jetbrains/python/debugger/array/ArrayViewStrategy.java | 4 ++++ .../python/debugger/containerview/DataViewStrategy.java | 1 + .../containerview/PyViewNumericContainerAction.java | 6 +++++- 9 files changed, 31 insertions(+), 2 deletions(-) diff --git a/notebooks/visualization/src/org/jetbrains/plugins/notebooks/visualization/outputs/statistic/NotebookOutputKeyType.kt b/notebooks/visualization/src/org/jetbrains/plugins/notebooks/visualization/outputs/statistic/NotebookOutputKeyType.kt index cbf635925958..063eacf815f4 100644 --- a/notebooks/visualization/src/org/jetbrains/plugins/notebooks/visualization/outputs/statistic/NotebookOutputKeyType.kt +++ b/notebooks/visualization/src/org/jetbrains/plugins/notebooks/visualization/outputs/statistic/NotebookOutputKeyType.kt @@ -13,6 +13,7 @@ enum class NotebookOutputKeyType { NUMPY_ARRAY, EAGER_TENSOR, RESOURCE_VARIABLE, + SPARSE_TENSOR, TORCH_TENSOR, PANDAS_DATA_FRAME, PANDAS_SERIES, diff --git a/python/helpers/pydev/_pydevd_bundle/pydevd_tables.py b/python/helpers/pydev/_pydevd_bundle/pydevd_tables.py index 7b63fcd6ffc7..4285cdc5c1ce 100644 --- a/python/helpers/pydev/_pydevd_bundle/pydevd_tables.py +++ b/python/helpers/pydev/_pydevd_bundle/pydevd_tables.py @@ -75,6 +75,7 @@ def __get_table_provider(output): elif type_qualified_name in ['numpy.ndarray', 'tensorflow.python.framework.ops.EagerTensor', 'tensorflow.python.ops.resource_variable_ops.ResourceVariable', + 'tensorflow.python.framework.sparse_tensor.SparseTensor', 'torch.Tensor', 'builtins.dict']: import _pydevd_bundle.tables.pydevd_numpy as table_provider diff --git a/python/helpers/pydev/_pydevd_bundle/pydevd_thrift.py b/python/helpers/pydev/_pydevd_bundle/pydevd_thrift.py index 79f2fc1d25e9..c86379065a8e 100644 --- a/python/helpers/pydev/_pydevd_bundle/pydevd_thrift.py +++ b/python/helpers/pydev/_pydevd_bundle/pydevd_thrift.py @@ -412,6 +412,11 @@ def tensor_to_thrift_struct(tensor, name, roffset, coffset, rows, cols, format): return array_to_thrift_struct(tensor.numpy(), name, roffset, coffset, rows, cols, format) +def sparse_tensor_to_thrift_struct(tensor, name, roffset, coffset, rows, cols, format): + import tensorflow as tf + return tensor_to_thrift_struct(tf.sparse.to_dense(tf.sparse.reorder(tensor)), name, roffset, coffset, rows, cols, format) + + def array_to_meta_thrift_struct(array, name, format): type = array.dtype.kind slice = name @@ -609,6 +614,7 @@ TYPE_TO_THRIFT_STRUCT_CONVERTERS = { "ndarray": array_to_thrift_struct, "EagerTensor": tensor_to_thrift_struct, "ResourceVariable": tensor_to_thrift_struct, + "SparseTensor": sparse_tensor_to_thrift_struct, "Tensor": tensor_to_thrift_struct, "DataFrame": dataframe_to_thrift_struct, "Series": dataframe_to_thrift_struct, diff --git a/python/helpers/pydev/_pydevd_bundle/pydevd_vars.py b/python/helpers/pydev/_pydevd_bundle/pydevd_vars.py index e5a1366c9f08..ab9a5814bf9b 100644 --- a/python/helpers/pydev/_pydevd_bundle/pydevd_vars.py +++ b/python/helpers/pydev/_pydevd_bundle/pydevd_vars.py @@ -586,6 +586,11 @@ def tensor_to_xml(tensor, name, roffset, coffset, rows, cols, format): return array_to_xml(tensor.numpy(), name, roffset, coffset, rows, cols, format) +def sparse_tensor_to_xml(tensor, name, roffset, coffset, rows, cols, format): + import tensorflow as tf + return tensor_to_xml(tf.sparse.to_dense(tf.sparse.reorder(tensor)), name, roffset, coffset, rows, cols, format) + + class ExceedingArrayDimensionsException(Exception): pass @@ -811,6 +816,7 @@ TYPE_TO_XML_CONVERTERS = { "GeoSeries": dataframe_to_xml, "EagerTensor": tensor_to_xml, "ResourceVariable": tensor_to_xml, + "SparseTensor": sparse_tensor_to_xml, "Tensor": tensor_to_xml } diff --git a/python/helpers/pydev/_pydevd_bundle/tables/pydevd_numpy.py b/python/helpers/pydev/_pydevd_bundle/tables/pydevd_numpy.py index 05c6d5545584..65bb8293a3bb 100644 --- a/python/helpers/pydev/_pydevd_bundle/tables/pydevd_numpy.py +++ b/python/helpers/pydev/_pydevd_bundle/tables/pydevd_numpy.py @@ -220,7 +220,12 @@ def _create_table(command, start_index=None, end_index=None): np_array = command['data'] sort_keys = command['sort_keys'] else: - np_array = command + try: + import tensorflow as tf + if isinstance(command, tf.SparseTensor): + command = tf.sparse.to_dense(tf.sparse.reorder(command)) + finally: + np_array = command if is_pd: sorting_arr = _sort_df(pd.DataFrame(np_array), sort_keys) diff --git a/python/pydevSrc/src/com/jetbrains/python/debugger/PyDebugValue.java b/python/pydevSrc/src/com/jetbrains/python/debugger/PyDebugValue.java index aa967d4a719e..0e09a888beed 100644 --- a/python/pydevSrc/src/com/jetbrains/python/debugger/PyDebugValue.java +++ b/python/pydevSrc/src/com/jetbrains/python/debugger/PyDebugValue.java @@ -37,6 +37,7 @@ public class PyDebugValue extends XNamedValue { "ndarray", ARRAY, "EagerTensor", ARRAY, "ResourceVariable", ARRAY, + "SparseTensor", ARRAY, "Tensor", ARRAY, DATA_FRAME, DATA_FRAME, SERIES, SERIES, diff --git a/python/src/com/jetbrains/python/debugger/array/ArrayViewStrategy.java b/python/src/com/jetbrains/python/debugger/array/ArrayViewStrategy.java index 3fed80814a39..3fb997d63e7a 100644 --- a/python/src/com/jetbrains/python/debugger/array/ArrayViewStrategy.java +++ b/python/src/com/jetbrains/python/debugger/array/ArrayViewStrategy.java @@ -27,6 +27,10 @@ public class ArrayViewStrategy extends DataViewStrategy { return new ArrayViewStrategy("ResourceVariable"); } + public static @NotNull ArrayViewStrategy createInstanceForSparseTensor() { + return new ArrayViewStrategy("SparseTensor"); + } + public static @NotNull ArrayViewStrategy createInstanceForTensor() { return new ArrayViewStrategy("Tensor"); } diff --git a/python/src/com/jetbrains/python/debugger/containerview/DataViewStrategy.java b/python/src/com/jetbrains/python/debugger/containerview/DataViewStrategy.java index c2db0ca32a0c..15783d447428 100644 --- a/python/src/com/jetbrains/python/debugger/containerview/DataViewStrategy.java +++ b/python/src/com/jetbrains/python/debugger/containerview/DataViewStrategy.java @@ -21,6 +21,7 @@ public abstract class DataViewStrategy { ArrayViewStrategy.createInstanceForNumpyArray(), ArrayViewStrategy.createInstanceForEagerTensor(), ArrayViewStrategy.createInstanceForResourceVariable(), + ArrayViewStrategy.createInstanceForSparseTensor(), ArrayViewStrategy.createInstanceForTensor(), DataFrameViewStrategy.createInstanceForDataFrame(), DataFrameViewStrategy.createInstanceForGeoDataFrame(), diff --git a/python/src/com/jetbrains/python/debugger/containerview/PyViewNumericContainerAction.java b/python/src/com/jetbrains/python/debugger/containerview/PyViewNumericContainerAction.java index ec702764d90d..d7af0bf8ef26 100644 --- a/python/src/com/jetbrains/python/debugger/containerview/PyViewNumericContainerAction.java +++ b/python/src/com/jetbrains/python/debugger/containerview/PyViewNumericContainerAction.java @@ -64,7 +64,11 @@ public class PyViewNumericContainerAction extends XDebuggerTreeActionBase { } String nodeType = debugValue.getType(); - if ("ndarray".equals(nodeType) || "EagerTensor".equals(nodeType) || "ResourceVariable".equals(nodeType) || "Tensor".equals(nodeType)) { + if ("ndarray".equals(nodeType) || + "EagerTensor".equals(nodeType) || + "ResourceVariable".equals(nodeType) || + "SparseTensor".equals(nodeType) || + "Tensor".equals(nodeType)) { e.getPresentation().setText(PyBundle.message("debugger.numeric.view.as.array")); e.getPresentation().setVisible(true); }