PY-20026 PyClassImpl#mroLinearize() properly handles duplicate base classes

The exception could happen in this case because we used the same cached
result of MRO linearization twice without defencive copying. Then later,
as a side effect of that, in mroMerge() we deleted one "head" from
several sequences simultaneously, hence the IndexOutOfBoundsException.

python/src/com/jetbrains/python/psi/impl/PyClassImpl.java

@@ -437,16 +437,17 @@ public class PyClassImpl extends PyBaseElementImpl<PyClassStub> implements PyCla
       }
       return computed.get();
     }
-    cache.put(type, Ref.<List<PyClassLikeType>>create());
+    cache.put(type, Ref.create());
     List<PyClassLikeType> result = null;
     try {
-      final List<PyClassLikeType> bases = type.getSuperClassTypes(context);
-      final List<List<PyClassLikeType>> lines = new ArrayList<List<PyClassLikeType>>();
+      final List<PyClassLikeType> bases = removeNotNullDuplicates(type.getSuperClassTypes(context));
+      final List<List<PyClassLikeType>> lines = new ArrayList<>();
       for (PyClassLikeType base : bases) {
         if (base != null) {
           final List<PyClassLikeType> baseClassMRO = mroLinearize(base, true, context, cache);
           if (!baseClassMRO.isEmpty()) {
-            lines.add(baseClassMRO);
+            // mroMerge() updates passed MRO lists internally
+            lines.add(new LinkedList<>(baseClassMRO));
           }
         }
       }
@@ -457,6 +458,7 @@ public class PyClassImpl extends PyBaseElementImpl<PyClassStub> implements PyCla
       if (addThisType) {
         result.add(0, type);
       }
+      result = Collections.unmodifiableList(result);
     }
     finally {
       cache.put(type, Ref.create(result));
@@ -464,6 +466,22 @@ public class PyClassImpl extends PyBaseElementImpl<PyClassStub> implements PyCla
     return result;
   }
 
+  @NotNull
+  private static <T> List<T> removeNotNullDuplicates(@NotNull List<T> list) {
+    final Set<T> distinct = new HashSet<>();
+    final List<T> result = new ArrayList<>();
+    for (T elem : list) {
+      if (elem != null) {
+        final boolean isUnique = distinct.add(elem);
+        if (!isUnique) {
+          continue;
+        }
+      }
+      result.add(elem);
+    }
+    return result;
+  }
+
   @Override
   @NotNull
   public PyFunction[] getMethods() {
@@ -1366,11 +1384,11 @@ public class PyClassImpl extends PyBaseElementImpl<PyClassStub> implements PyCla
     PyPsiUtils.assertValid(this);
     final PyType thisType = context.getType(this);
     if (thisType instanceof PyClassLikeType) {
-      final PyClassLikeType thisClassLikeType = (PyClassLikeType)thisType;
-      final List<PyClassLikeType> ancestorTypes =
-        mroLinearize(thisClassLikeType, false, context, new HashMap<PyClassLikeType, Ref<List<PyClassLikeType>>>());
+      final List<PyClassLikeType> ancestorTypes = mroLinearize((PyClassLikeType)thisType, false, context, new HashMap<>());
       if (isOverriddenMRO(ancestorTypes, context)) {
-        ancestorTypes.add(null);
+        final ArrayList<PyClassLikeType> withNull = new ArrayList<>(ancestorTypes);
+        withNull.add(null);
+        return withNull;
       }
       return ancestorTypes;
     }

python/testData/codeInsight/classMRO/DuplicatedBaseClasses.py

@@ -0,0 +1,6 @@
+class Base(object):
+    pass
+
+
+class MyClass(Base, Base):
+    pass

python/testSrc/com/jetbrains/python/codeInsight/PyClassMROTest.java

@@ -110,6 +110,11 @@ public class PyClassMROTest extends PyTestCase {
     assertOrderedEquals(classNames, Arrays.asList(mro));
   }
 
+  // PY-20026
+  public void testDuplicatedBaseClasses() {
+    assertMRO(getClass("MyClass"), "Base", "object");
+  }
+
   @NotNull
   public PyClass getClass(@NotNull String name) {
     myFixture.configureByFile(getPath(getTestName(false)));