From 0d53aa47dff5a9bbbb237ffa7fdc4f88aea85857 Mon Sep 17 00:00:00 2001 From: "Natalia.Murycheva" Date: Sat, 13 May 2023 20:44:29 +0000 Subject: [PATCH] DS-4878 Pandas-specific-quick-fix-replace-listdf.col.values-to-df.col.tolist Add new intention and a corresponding quick fix for the usage pd.Series.values property from pandas library. ^DS-4878 Fixed Merge-request: IJ-MR-106089 Merged-by: Natalia Murycheva GitOrigin-RevId: 0c8dc40b09ee2d95ecd8ded532f31f5ef4a7740f --- .../PyPandasSeriesToListInspection.html | 14 + .../resources/messages/PyPsiBundle.properties | 4 + python/src/META-INF/python-core-common.xml | 3 + .../PyPandasSeriesToListInspection.kt | 88 ++++++ .../dataframeGetattr.py | 5 + .../dataframeGetattr_after.py | 5 + .../dataframeGetitem.py | 12 + .../dataframeGetitem_after.py | 12 + .../pandas/__init__.py | 2 + .../pandas/frame.py | 18 ++ .../pandas/series.py | 292 ++++++++++++++++++ .../seriesSimple.py | 5 + .../seriesSimple_after.py | 5 + .../PyPandasSeriesToListQuickFixTest.kt | 29 ++ 14 files changed, 494 insertions(+) create mode 100644 python/python-psi-impl/resources/inspectionDescriptions/PyPandasSeriesToListInspection.html create mode 100644 python/src/com/jetbrains/python/inspections/PyPandasSeriesToListInspection.kt create mode 100644 python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/dataframeGetattr.py create mode 100644 python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/dataframeGetattr_after.py create mode 100644 python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/dataframeGetitem.py create mode 100644 python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/dataframeGetitem_after.py create mode 100644 python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/pandas/__init__.py create mode 100644 python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/pandas/frame.py create mode 100644 python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/pandas/series.py create mode 100644 python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/seriesSimple.py create mode 100644 python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/seriesSimple_after.py create mode 100644 python/testSrc/com/jetbrains/python/quickFixes/PyPandasSeriesToListQuickFixTest.kt diff --git a/python/python-psi-impl/resources/inspectionDescriptions/PyPandasSeriesToListInspection.html b/python/python-psi-impl/resources/inspectionDescriptions/PyPandasSeriesToListInspection.html new file mode 100644 index 000000000000..d321e3b38917 --- /dev/null +++ b/python/python-psi-impl/resources/inspectionDescriptions/PyPandasSeriesToListInspection.html @@ -0,0 +1,14 @@ + + +

Reports redundant list in list(Series.values) statement for pandas and polars libraries. + Such Series values extraction can be replaced with the to_list() function call.

+

Example:

+
+list(df['column'].values)
+
+

When the quick-fix is applied, the code changes to:

+
+df['column'].to_list()
+
+ + \ No newline at end of file diff --git a/python/python-psi-impl/resources/messages/PyPsiBundle.properties b/python/python-psi-impl/resources/messages/PyPsiBundle.properties index 36396345e88a..b2e65adece49 100644 --- a/python/python-psi-impl/resources/messages/PyPsiBundle.properties +++ b/python/python-psi-impl/resources/messages/PyPsiBundle.properties @@ -1245,3 +1245,7 @@ INSP.class.var.can.not.override.instance.variable=Cannot override instance varia INSP.class.var.can.not.be.used.in.annotations.for.function.parameters='ClassVar' cannot be used in annotations for function parameters INSP.class.var.can.not.be.used.in.annotation.for.function.return.value='ClassVar' cannot be used in annotation for a function return value INSP.class.var.can.not.include.type.variables='ClassVar' parameter cannot include type variables + +# Pandas-Specific inspections and quick fixes +INSP.pandas.series.values.replace.with.tolist=Method Series.to_list() is recommended +QFIX.pandas.series.values.replace.with.tolist=Replace list(Series.values) with Series.to_list() diff --git a/python/src/META-INF/python-core-common.xml b/python/src/META-INF/python-core-common.xml index 2c2a8c5cd643..74ba90e8ba77 100644 --- a/python/src/META-INF/python-core-common.xml +++ b/python/src/META-INF/python-core-common.xml @@ -194,6 +194,9 @@ + list(df.b.values) \ No newline at end of file diff --git a/python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/dataframeGetattr_after.py b/python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/dataframeGetattr_after.py new file mode 100644 index 000000000000..620d859165b9 --- /dev/null +++ b/python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/dataframeGetattr_after.py @@ -0,0 +1,5 @@ +import pandas as pd +# DataFrame columns case +df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6], "c": [7, 8, 9]}) + +df.b.to_list() \ No newline at end of file diff --git a/python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/dataframeGetitem.py b/python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/dataframeGetitem.py new file mode 100644 index 000000000000..ee17a810d951 --- /dev/null +++ b/python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/dataframeGetitem.py @@ -0,0 +1,12 @@ +import pandas as pd +# DataFrame columns case +df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6], "c": [7, 8, 9]}) + +list(df[['a', 'b']].values) +bb = ["a", "b", "c"] +list(df[bb].values) + +# with errors +list(df.['a'].values) + +list(df['a'].values) \ No newline at end of file diff --git a/python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/dataframeGetitem_after.py b/python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/dataframeGetitem_after.py new file mode 100644 index 000000000000..8a2b300103e2 --- /dev/null +++ b/python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/dataframeGetitem_after.py @@ -0,0 +1,12 @@ +import pandas as pd +# DataFrame columns case +df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6], "c": [7, 8, 9]}) + +list(df[['a', 'b']].values) +bb = ["a", "b", "c"] +list(df[bb].values) + +# with errors +list(df.['a'].values) + +df['a'].to_list() \ No newline at end of file diff --git a/python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/pandas/__init__.py b/python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/pandas/__init__.py new file mode 100644 index 000000000000..492330bb1e84 --- /dev/null +++ b/python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/pandas/__init__.py @@ -0,0 +1,2 @@ +from series import Series +from frame import DataFrame \ No newline at end of file diff --git a/python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/pandas/frame.py b/python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/pandas/frame.py new file mode 100644 index 000000000000..e30369171bc6 --- /dev/null +++ b/python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/pandas/frame.py @@ -0,0 +1,18 @@ +from series import Series + + +class NDFrame: + ... + + +class DataFrame: + def __getitem__(self, key): + if key == "1": + return None + if key == "2": + return Series() + if key == "3": + return NDFrame() + if key == "4": + return DataFrame() + diff --git a/python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/pandas/series.py b/python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/pandas/series.py new file mode 100644 index 000000000000..c053b71d1d5b --- /dev/null +++ b/python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/pandas/series.py @@ -0,0 +1,292 @@ +class property(object): + """ + Property attribute. + + fget + function to be used for getting an attribute value + fset + function to be used for setting an attribute value + fdel + function to be used for del'ing an attribute + doc + docstring + + Typical use is to define a managed attribute x: + + class C(object): + def getx(self): return self._x + def setx(self, value): self._x = value + def delx(self): del self._x + x = property(getx, setx, delx, "I'm the 'x' property.") + + Decorators make defining new properties or modifying existing ones easy: + + class C(object): + @property + def x(self): + "I am the 'x' property." + return self._x + @x.setter + def x(self, value): + self._x = value + @x.deleter + def x(self): + del self._x + """ + def deleter(self, *args, **kwargs): # real signature unknown + """ Descriptor to obtain a copy of the property with a different deleter. """ + pass + + def getter(self, *args, **kwargs): # real signature unknown + """ Descriptor to obtain a copy of the property with a different getter. """ + pass + + def setter(self, *args, **kwargs): # real signature unknown + """ Descriptor to obtain a copy of the property with a different setter. """ + pass + + def __delete__(self, *args, **kwargs): # real signature unknown + """ Delete an attribute of instance. """ + pass + + def __getattribute__(self, *args, **kwargs): # real signature unknown + """ Return getattr(self, name). """ + pass + + def __get__(self, *args, **kwargs): # real signature unknown + """ Return an attribute of instance, which is of type owner. """ + pass + + def __init__(self, fget=None, fset=None, fdel=None, doc=None): # known special case of property.__init__ + """ + Property attribute. + + fget + function to be used for getting an attribute value + fset + function to be used for setting an attribute value + fdel + function to be used for del'ing an attribute + doc + docstring + + Typical use is to define a managed attribute x: + + class C(object): + def getx(self): return self._x + def setx(self, value): self._x = value + def delx(self): del self._x + x = property(getx, setx, delx, "I'm the 'x' property.") + + Decorators make defining new properties or modifying existing ones easy: + + class C(object): + @property + def x(self): + "I am the 'x' property." + return self._x + @x.setter + def x(self, value): + self._x = value + @x.deleter + def x(self): + del self._x + # (copied from class doc) + """ + pass + + @staticmethod # known case of __new__ + def __new__(*args, **kwargs): # real signature unknown + """ Create and return a new object. See help(type) for accurate signature. """ + pass + + def __set__(self, *args, **kwargs): # real signature unknown + """ Set an attribute of instance to value. """ + pass + + fdel = property(lambda self: object(), lambda self, v: None, lambda self: None) # default + + fget = property(lambda self: object(), lambda self, v: None, lambda self: None) # default + + fset = property(lambda self: object(), lambda self, v: None, lambda self: None) # default + + __isabstractmethod__ = property(lambda self: object(), lambda self, v: None, lambda self: None) # default + + +class IndexOpsMixin(): + """ + Common ops mixin to support a unified interface / docs for Series / Index + """ + def tolist(self): + """ + Return a list of the values. + + These are each a scalar type, which is a Python scalar + (for str, int, float) or a pandas scalar + (for Timestamp/Timedelta/Interval/Period) + + Returns + ------- + list + + See Also + -------- + numpy.ndarray.tolist : Return the array as an a.ndim-levels deep + nested list of Python scalars. + """ + # return self._values.tolist() + ... + + to_list = tolist + + +class NDFrame: + ... + + +class Series(IndexOpsMixin, NDFrame): + """ + One-dimensional ndarray with axis labels (including time series). + + Labels need not be unique but must be a hashable type. The object + supports both integer- and label-based indexing and provides a host of + methods for performing operations involving the index. Statistical + methods from ndarray have been overridden to automatically exclude + missing data (currently represented as NaN). + + Operations between Series (+, -, /, \\*, \\*\\*) align values based on their + associated index values-- they need not be the same length. The result + index will be the sorted union of the two indexes. + + Parameters + ---------- + data : array-like, Iterable, dict, or scalar value + Contains data stored in Series. If data is a dict, argument order is + maintained. + index : array-like or Index (1d) + Values must be hashable and have the same length as `data`. + Non-unique index values are allowed. Will default to + RangeIndex (0, 1, 2, ..., n) if not provided. If data is dict-like + and index is None, then the keys in the data are used as the index. If the + index is not None, the resulting Series is reindexed with the index values. + dtype : str, numpy.dtype, or ExtensionDtype, optional + Data type for the output Series. If not specified, this will be + inferred from `data`. + See the :ref:`user guide ` for more usages. + name : str, optional + The name to give to the Series. + copy : bool, default False + Copy input data. Only affects Series or 1d ndarray input. See examples. + + Examples + -------- + Constructing Series from a dictionary with an Index specified + + # >>> d = {'a': 1, 'b': 2, 'c': 3} + # >>> ser = pd.Series(data=d, index=['a', 'b', 'c']) + # >>> ser + a 1 + b 2 + c 3 + dtype: int64 + + The keys of the dictionary match with the Index values, hence the Index + values have no effect. + + # >>> d = {'a': 1, 'b': 2, 'c': 3} + # >>> ser = pd.Series(data=d, index=['x', 'y', 'z']) + # >>> ser + x NaN + y NaN + z NaN + dtype: float64 + + Note that the Index is first build with the keys from the dictionary. + After this the Series is reindexed with the given Index values, hence we + get all NaN as a result. + + Constructing Series from a list with `copy=False`. + + # >>> r = [1, 2] + # >>> ser = pd.Series(r, copy=False) + # >>> ser.iloc[0] = 999 + # >>> r + [1, 2] + # >>> ser + 0 999 + 1 2 + dtype: int64 + + Due to input data type the Series has a `copy` of + the original data even though `copy=False`, so + the data is unchanged. + + Constructing Series from a 1d ndarray with `copy=False`. + + # >>> r = np.array([1, 2]) + # >>> ser = pd.Series(r, copy=False) + # >>> ser.iloc[0] = 999 + # >>> r + array([999, 2]) + # >>> ser + 0 999 + 1 2 + dtype: int64 + + Due to input data type the Series has a `view` on + the original data, so + the data is changed as well. + """ + def __init__( + self, + data=None, + index=None, + dtype = None, + name=None, + copy = False, + fastpath = False, + ): + ... + + @property + def values(self): + """ + Return Series as ndarray or ndarray-like depending on the dtype. + + .. warning:: + + We recommend using :attr:`Series.array` or + :meth:`Series.to_numpy`, depending on whether you need + a reference to the underlying data or a NumPy array. + + Returns + ------- + numpy.ndarray or ndarray-like + + See Also + -------- + Series.array : Reference to the underlying data. + Series.to_numpy : A NumPy array representing the underlying data. + + Examples + -------- + # >>> pd.Series([1, 2, 3]).values + array([1, 2, 3]) + + # >>> pd.Series(list('aabc')).values + array(['a', 'a', 'b', 'c'], dtype=object) + + # >>> pd.Series(list('aabc')).astype('category').values + ['a', 'a', 'b', 'c'] + Categories (3, object): ['a', 'b', 'c'] + + Timezone aware datetime data is converted to UTC: + + # >>> pd.Series(pd.date_range('20130101', periods=3, + ... tz='US/Eastern')).values + array(['2013-01-01T05:00:00.000000000', + '2013-01-02T05:00:00.000000000', + '2013-01-03T05:00:00.000000000'], dtype='datetime64[ns]') + """ + # return self._mgr.external_values() + ... \ No newline at end of file diff --git a/python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/seriesSimple.py b/python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/seriesSimple.py new file mode 100644 index 000000000000..18b5452fc17c --- /dev/null +++ b/python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/seriesSimple.py @@ -0,0 +1,5 @@ +import pandas as pd +# Series case +a = pd.Series([1, 2, 3]) + +list(a.values) \ No newline at end of file diff --git a/python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/seriesSimple_after.py b/python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/seriesSimple_after.py new file mode 100644 index 000000000000..2239b52026eb --- /dev/null +++ b/python/testData/quickFixes/PyPandasSeriesToListQuickFixTest/seriesSimple_after.py @@ -0,0 +1,5 @@ +import pandas as pd +# Series case +a = pd.Series([1, 2, 3]) + +a.to_list() \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/python/quickFixes/PyPandasSeriesToListQuickFixTest.kt b/python/testSrc/com/jetbrains/python/quickFixes/PyPandasSeriesToListQuickFixTest.kt new file mode 100644 index 000000000000..9d684def3f7f --- /dev/null +++ b/python/testSrc/com/jetbrains/python/quickFixes/PyPandasSeriesToListQuickFixTest.kt @@ -0,0 +1,29 @@ +// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.jetbrains.python.quickFixes + +import com.jetbrains.python.PyPsiBundle +import com.jetbrains.python.PyQuickFixTestCase +import com.jetbrains.python.inspections.PyPandasSeriesToListInspection + +class PyPandasSeriesToListQuickFixTest : PyQuickFixTestCase() { + private val quickFixName = PyPsiBundle.message("QFIX.pandas.series.values.replace.with.tolist") + + @Throws(Exception::class) + override fun setUp() { + super.setUp() + myFixture.copyDirectoryToProject("", "") + } + + fun testDataframeGetitem() { + doQuickFixTest(PyPandasSeriesToListInspection::class.java, quickFixName) + } + + fun testDataframeGetattr() { + doQuickFixTest(PyPandasSeriesToListInspection::class.java, quickFixName) + } + + + fun testSeriesSimple() { + doQuickFixTest(PyPandasSeriesToListInspection::class.java, quickFixName) + } +} \ No newline at end of file