PY-83156 PyInvalidCast: consider variance

(cherry picked from commit c0cdc880837bb5c7ed7043e065c1e08c9b839b40)

GitOrigin-RevId: c8395f5047e44d80f3962c7f582948d37ae91fad

python/python-psi-impl/resources/inspectionDescriptions/PyInvalidCastInspection.html

@@ -33,9 +33,10 @@ cast(list[object], int_list)  # ok
 <!-- tooltip end -->
 <p>The inspection relies on static type information; when a type is unknown, no warning is reported.</p>
 
-<p>Use the <b>Ignore variance of generic type arguments</b> option to control whether generic type arguments are taken
-  into account. When enabled (the default), parameterized types are compared by their base class only, so a cast such as
-  <code>cast(list[object], int_list)</code> is allowed. Disable it to also report casts that differ only in their
+<p>Use the <b>Ignore variance of generic type arguments</b> option to control whether the variance of generic type
+  arguments is taken into account. When enabled (the default), the variance is ignored: as long as the corresponding
+  type arguments are in the same inheritance hierarchy, the cast is allowed, so <code>cast(list[object], int_list)</code>
+  is not reported even though <code>list</code> is invariant. Disable it to also report casts that differ only in their
   (invariant) type arguments.</p>
 
 <p>Use the <b>Ignore TypedDict structure (treat a TypedDict as a dict)</b> option to control whether a <code>TypedDict</code>

python/python-psi-impl/resources/messages/PyPsiBundle.properties

@@ -1506,7 +1506,8 @@ packaging.could.not.parse.relation=Could not parse relation from: {0}
 
 # PyInvalidCastInspection
 INSP.NAME.invalid.cast=Type cast between unrelated types
-INSP.invalid.cast.message=Cast of type ''{0}'' to type ''{1}'' may be a mistake because they are not in the same inheritance hierarchy. If this was intentional, cast the expression to ''{2}'' first.
+INSP.invalid.cast.message=Cast of type `{0}` to type `{1}` may be a mistake because they are not in the same inheritance hierarchy. If this was intentional, cast the expression to `{2}` first.
+INSP.invalid.cast.variance.message=Cast of type `{0}` to type `{1}` may be a mistake because they differ in invariant type arguments
 INSP.invalid.cast.ignore.generic.variance=Ignore variance of generic type arguments
 INSP.invalid.cast.ignore.typed.dict.structure=Ignore TypedDict structure (treat a TypedDict as a dict)
 

python/python-psi-impl/src/com/jetbrains/python/inspections/PyInvalidCastInspection.kt

@@ -1,6 +1,7 @@
 package com.jetbrains.python.inspections
 
 import com.intellij.codeInspection.LocalInspectionToolSession
+import com.intellij.codeInspection.ProblemHighlightType
 import com.intellij.codeInspection.ProblemsHolder
 import com.intellij.codeInspection.options.OptPane
 import com.intellij.modcommand.ModPsiUpdater
@@ -19,7 +20,6 @@ import com.jetbrains.python.psi.PyFunction
 import com.jetbrains.python.psi.types.PyClassLikeType
 import com.jetbrains.python.psi.types.PyClassType
 import com.jetbrains.python.psi.types.PyClassTypeImpl
-import com.jetbrains.python.psi.types.PyCollectionType
 import com.jetbrains.python.psi.types.PyType
 import com.jetbrains.python.psi.types.PyTypeUtil.isOverlappingWith
 import com.jetbrains.python.psi.types.PyTypedDictType
@@ -61,37 +61,35 @@ class PyInvalidCastInspection : PyInspection() {
 
         if (targetType.isOverlappingWith(actualType, myTypeEvalContext)) return
 
-        // Relax the overlap check according to the enabled options:
-        //  - ignoring generic variance erases generic type arguments, so e.g. casting 'list[int]' to 'list[object]'
-        //    is treated as overlapping;
-        //  - ignoring TypedDict structure treats a TypedDict as a plain 'dict', so e.g. a 'dict[str, object]' may be
-        //    cast to a TypedDict.
-        if (ignoreGenericVariance || ignoreTypedDictStructure) {
-          var relaxedTarget = targetType
-          var relaxedActual = actualType
-          if (ignoreGenericVariance) {
-            relaxedTarget = relaxedTarget.eraseGenericParameters()
-            relaxedActual = relaxedActual.eraseGenericParameters()
-          }
-          if (ignoreTypedDictStructure) {
-            relaxedTarget = relaxedTarget.eraseTypedDictStructure()
-            relaxedActual = relaxedActual.eraseTypedDictStructure()
-          }
-          if (relaxedTarget.isOverlappingWith(relaxedActual, myTypeEvalContext)) return
+        // Treat a TypedDict as a plain 'dict' when the corresponding option is enabled, so that e.g. a
+        // 'dict[str, object]' may be cast to a TypedDict and vice versa.
+        if (ignoreTypedDictStructure &&
+            targetType.eraseTypedDictStructure().isOverlappingWith(actualType.eraseTypedDictStructure(), myTypeEvalContext)) {
+          return
         }
+
+        // The types become subtype-related once the variance of their generic arguments is ignored, so the mismatch
+        // is purely about variance (e.g. casting the invariant 'list[int]' to 'list[object]'). Such casts are only
+        // reported when the "ignore generic variance" option is disabled, and get a dedicated message.
+        val relatedIgnoringVariance = isOverlappingIgnoringVariance(targetType, actualType, myTypeEvalContext)
+        if (relatedIgnoringVariance && ignoreGenericVariance) return
+
         val fromName = PythonDocumentationProvider.getTypeName(actualType, myTypeEvalContext)
         val toName = PythonDocumentationProvider.getVerboseTypeName(targetType, myTypeEvalContext)
 
         val suggestedName = computeSuggestedIntermediateTypeName(targetType, actualType, myTypeEvalContext)
 
+        val messageKey = if (relatedIgnoringVariance) "INSP.invalid.cast.variance.message" else "INSP.invalid.cast.message"
+
         registerProblem(
           callExpression,
-          PyPsiBundle.message(
-            "INSP.invalid.cast.message",
+          PyPsiBundle.problemMessage(
+            messageKey,
             fromName,
             toName,
             suggestedName
           ),
+          ProblemHighlightType.GENERIC_ERROR_OR_WARNING,
           AddIntermediateCastQuickFix(suggestedName)
         )
       }
@@ -118,14 +116,30 @@ private class AddIntermediateCastQuickFix(private val typeText: String) : PsiUpd
 }
 
 /**
- * Erases generic type arguments so that parameterized types are compared by their base class only,
- * making the overlap check insensitive to the variance of generic parameters
- * (e.g. `list[int]` becomes plain `list`). Union members are erased element-wise.
+ * Checks whether two types are overlapping while ignoring the variance of generic type arguments: two
+ * parameterized types of related base classes are considered related as long as their corresponding type arguments
+ * are themselves overlapping, regardless of the direction required by the declared variance
+ * (e.g. `list\[int]` and `list\[object]` are related, but `list\[str]` and `list\[int]` are not). This makes the check
+ * insensitive to whether a generic parameter is invariant, covariant, or contravariant. Union members are
+ * distributed over.
  */
-private fun PyType?.eraseGenericParameters(): PyType? = when (this) {
-  is PyUnionType -> this.map { it.eraseGenericParameters() }
-  is PyCollectionType -> PyClassTypeImpl(this.pyClass, this.isDefinition)
-  else -> this
+private fun isOverlappingIgnoringVariance(t1: PyType?, t2: PyType?, context: TypeEvalContext): Boolean {
+  if (t1 is PyUnionType) return t1.members.any { isOverlappingIgnoringVariance(it, t2, context) }
+  if (t2 is PyUnionType) return t2.members.any { isOverlappingIgnoringVariance(t1, it, context) }
+
+  if (t1 is PyClassType && t2 is PyClassType && t1.isParameterized && t2.isParameterized) {
+    val base1 = PyClassTypeImpl(t1.pyClass, t1.isDefinition)
+    val base2 = PyClassTypeImpl(t2.pyClass, t2.isDefinition)
+    if (!base1.isOverlappingWith(base2, context)) return false
+
+    val args1 = t1.typeArguments
+    val args2 = t2.typeArguments
+    // Different arity means we can't align the arguments; the base-class relation is enough to consider them related.
+    if (args1.size != args2.size) return true
+    return args1.indices.all { isOverlappingIgnoringVariance(args1[it], args2[it], context) }
+  }
+
+  return t1.isOverlappingWith(t2, context)
 }
 
 /**

python/testSrc/com/jetbrains/python/inspections/PyInvalidCastInspectionTest.kt

@@ -1,6 +1,8 @@
 // Copyright 2000-2026 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package com.jetbrains.python.inspections
 
+import com.intellij.idea.TestFor
+import com.intellij.openapi.util.registry.Registry
 import com.jetbrains.python.PyPsiBundle
 import com.jetbrains.python.PythonFileType
 import com.jetbrains.python.fixtures.PyInspectionTestCase
@@ -14,12 +16,12 @@ class PyInvalidCastInspectionTest : PyInspectionTestCase() {
         from typing import cast
 
         <warning descr="Cast of type 'Literal[\"a\"]' to type 'int' may be a mistake because they are not in the same inheritance hierarchy. If this was intentional, cast the expression to 'object' first.">cast(int, "a")</warning>
+        <warning descr="Cast of type 'list[str]' to type 'list[int]' may be a mistake because they are not in the same inheritance hierarchy. If this was intentional, cast the expression to 'object' first.">cast(list[int], ["a"])</warning>
 
         cast(int, object())  # ok
         cast(object, 1)  # ok
 
         # variance of generic type arguments is ignored by default
-        cast(list[int], ["a"])  # ok
         lint = [1, 2, 3]
         cast(list[object], lint)  # ok
       """.trimIndent()
@@ -56,20 +58,25 @@ class PyInvalidCastInspectionTest : PyInspectionTestCase() {
     )
   }
 
-  /**
-   * test that the variance of invariant generics is ignored by default
-   */
-  fun `test generic variance ignored by default`() {
-    doTestByText(
+  @TestFor(issues = ["PY-83156"])
+  fun `test generic variance check enabled`() {
+    Registry.get("python.subtypechecks.respect.variance").setValue(true, testRootDisposable)
+
+    myFixture.configureByText(
+      PythonFileType.INSTANCE,
       """
         from typing import cast, Sequence
 
         lint = [1, 2, 3]
-        cast(list[object], lint)  # ok
+        <warning descr="Cast of type 'list[int]' to type 'list[object]' may be a mistake because they differ in invariant type arguments">cast(list[object], lint)</warning>
 
         cast(Sequence[object], lint)  # ok
       """.trimIndent()
     )
+    val inspection = PyInvalidCastInspection()
+    inspection.ignoreGenericVariance = false
+    myFixture.enableInspections(inspection)
+    myFixture.checkHighlighting(isWarning, isInfo, isWeakWarning)
   }
 
   /**

python/testSrc/com/jetbrains/python/types/PyOverloadTypeTest.kt

@@ -2,11 +2,9 @@
 package com.jetbrains.python.types
 
 import com.intellij.idea.TestFor
-import com.intellij.testFramework.TestLoggerFactory
 import com.intellij.testFramework.TestLoggerFactory.TestLoggerAssertionError
 import com.jetbrains.python.fixtures.PyCodeInsightTestCase
 import com.jetbrains.python.psi.LanguageLevel
-import org.junit.jupiter.api.Disabled
 import org.junit.jupiter.api.Nested
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.assertThrows