Merge branch 'python-fixes'

python/pluginSrc/com/jetbrains/python/psi/impl/PyJavaClassType.java

@@ -132,4 +132,8 @@ public class PyJavaClassType implements PyClassLikeType {
   public boolean isValid() {
     return myClass.isValid();
   }
+
+  public PsiClass getPsiClass() {
+    return myClass;
+  }
 }

python/pluginSrc/com/jetbrains/python/psi/impl/PyJavaSuperMethodsSearchExecutor.java

@@ -9,6 +9,8 @@ import com.intellij.util.containers.ContainerUtil;
 import com.jetbrains.python.psi.PyClass;
 import com.jetbrains.python.psi.PyFunction;
 import com.jetbrains.python.psi.search.PySuperMethodsSearch;
+import com.jetbrains.python.psi.types.PyClassLikeType;
+import com.jetbrains.python.psi.types.TypeEvalContext;
 import org.jetbrains.annotations.NotNull;
 
 /**
@@ -19,10 +21,9 @@ public class PyJavaSuperMethodsSearchExecutor implements QueryExecutor<PsiElemen
     PyFunction func = queryParameters.getDerivedMethod();
     PyClass containingClass = func.getContainingClass();
     if (containingClass != null) {
-      PsiElement[] superClassElements = containingClass.getSuperClassElements();
-      for(PsiElement element: superClassElements) {
-        if (element instanceof PsiClass) {
-          PsiClass psiClass = (PsiClass) element;
+      for (PyClassLikeType type : containingClass.getSuperClassTypes(TypeEvalContext.codeInsightFallback())) {
+        if (type instanceof PyJavaClassType) {
+          final PsiClass psiClass = ((PyJavaClassType)type).getPsiClass();
           PsiMethod[] methods = psiClass.findMethodsByName(func.getName(), true);
           // the Python method actually does override/implement all of Java super methods with the same name
           if (!ContainerUtil.process(methods, consumer)) return false;

python/psi-api/src/com/jetbrains/python/PythonDialectsTokenSetContributor.java

@@ -2,19 +2,61 @@ package com.jetbrains.python;
 
 import com.intellij.openapi.extensions.ExtensionPointName;
 import com.intellij.psi.tree.TokenSet;
+import org.jetbrains.annotations.NotNull;
 
 /**
+ * Contributes element types of various kinds specific for a particular Python dialect.
+ *
  * @author vlan
  */
 public interface PythonDialectsTokenSetContributor {
   ExtensionPointName<PythonDialectsTokenSetContributor> EP_NAME = ExtensionPointName.create("Pythonid.dialectsTokenSetContributor");
 
+  /**
+   * Returns element types that are subclasses of {@link com.jetbrains.python.psi.PyStatement}.
+   */
+  @NotNull
   TokenSet getStatementTokens();
+
+  /**
+   * Returns element types that are subclasses of {@link com.jetbrains.python.psi.PyExpression}.
+   */
+  @NotNull
   TokenSet getExpressionTokens();
+
+  /**
+   * Returns element types that are subclasses of {@link com.jetbrains.python.psi.NameDefiner}.
+   */
+  @NotNull
   TokenSet getNameDefinerTokens();
+
+  /**
+   * Returns element types that are language keywords.
+   */
+  @NotNull
   TokenSet getKeywordTokens();
+
+  /**
+   * Returns element types that are subclasses of {@link com.jetbrains.python.psi.PyParameter}.
+   */
+  @NotNull
   TokenSet getParameterTokens();
+
+  /**
+   * Returns element types that are subclasses of {@link com.jetbrains.python.psi.PyFunction}.
+   */
+  @NotNull
   TokenSet getFunctionDeclarationTokens();
+
+  /**
+   * Returns element types that can be used as unbalanced braces recovery tokens in the lexer.
+   */
+  @NotNull
   TokenSet getUnbalancedBracesRecoveryTokens();
+
+  /**
+   * Returns element types that are subclasses of {@link com.jetbrains.python.psi.PyReferenceExpression}.
+   */
+  @NotNull
   TokenSet getReferenceExpressionTokens();
 }

python/psi-api/src/com/jetbrains/python/PythonDialectsTokenSetContributorBase.java

@@ -0,0 +1,57 @@
+package com.jetbrains.python;
+
+import com.intellij.psi.tree.TokenSet;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * @author vlan
+ */
+public abstract class PythonDialectsTokenSetContributorBase implements PythonDialectsTokenSetContributor {
+  @NotNull
+  @Override
+  public TokenSet getStatementTokens() {
+    return TokenSet.EMPTY;
+  }
+
+  @NotNull
+  @Override
+  public TokenSet getExpressionTokens() {
+    return TokenSet.EMPTY;
+  }
+
+  @NotNull
+  @Override
+  public TokenSet getNameDefinerTokens() {
+    return TokenSet.EMPTY;
+  }
+
+  @NotNull
+  @Override
+  public TokenSet getKeywordTokens() {
+    return TokenSet.EMPTY;
+  }
+
+  @NotNull
+  @Override
+  public TokenSet getParameterTokens() {
+    return TokenSet.EMPTY;
+  }
+
+  @NotNull
+  @Override
+  public TokenSet getFunctionDeclarationTokens() {
+    return TokenSet.EMPTY;
+  }
+
+  @NotNull
+  @Override
+  public TokenSet getUnbalancedBracesRecoveryTokens() {
+    return TokenSet.EMPTY;
+  }
+
+  @NotNull
+  @Override
+  public TokenSet getReferenceExpressionTokens() {
+    return TokenSet.EMPTY;
+  }
+}

python/psi-api/src/com/jetbrains/python/psi/PyClass.java

@@ -36,21 +36,65 @@ public interface PyClass extends PsiNameIdentifierOwner, PyStatement, NameDefine
   @NotNull
   PyStatementList getStatementList();
 
-  @Nullable
-  PyArgumentList getSuperClassExpressionList();
-
+  /**
+   * Returns types of all ancestors from the hierarchy.
+   */
   @NotNull
-  PyExpression[] getSuperClassExpressions();
+  List<PyClassLikeType> getAncestorTypes(@NotNull TypeEvalContext context);
 
+  /**
+   * Returns only those ancestors from the hierarchy, that are resolved to PyClass PSI elements.
+   *
+   * @see #getAncestorTypes(TypeEvalContext) for the full list of ancestors.
+   */
   @NotNull
-  PsiElement[] getSuperClassElements();
+  List<PyClass> getAncestorClasses(@NotNull TypeEvalContext context);
 
+  /**
+   * Returns only those ancestors from the hierarchy, that are resolved to PyClass PSI elements, using the default type evaluation context.
+   *
+   * @see #getAncestorClasses(TypeEvalContext) if a more detailed TypeEvalContext is available.
+   */
+  @NotNull
+  List<PyClass> getAncestorClasses();
+
+  /**
+   * Returns types of expressions in the super classes list.
+   *
+   * If no super classes are specified, returns the type of the implicit super class for old- and new-style classes.
+   *
+   * @see #getAncestorTypes(TypeEvalContext) for the full list of ancestors.
+   */
   @NotNull
   List<PyClassLikeType> getSuperClassTypes(@NotNull TypeEvalContext context);
 
+  /**
+   * Returns only those super classes for expressions from the super classes list, that are resolved to PyClass PSI elements.
+   *
+   * If no super classes are specified, returns the implicit super class for old- and new-style classes.
+   *
+   * @see #getSuperClassTypes(TypeEvalContext) for the full list of super classes.
+   * @see #getAncestorTypes(TypeEvalContext) for the full list of ancestors.
+   */
   @NotNull
   PyClass[] getSuperClasses();
 
+  /**
+   * Returns a PSI element for the super classes list.
+   *
+   * Operates at the AST level.
+   */
+  @Nullable
+  PyArgumentList getSuperClassExpressionList();
+
+  /**
+   * Returns PSI elements for the expressions in the super classes list.
+   *
+   * Operates at the AST level.
+   */
+  @NotNull
+  PyExpression[] getSuperClassExpressions();
+
   @NotNull
   PyFunction[] getMethods();
 
@@ -111,18 +155,6 @@ public interface PyClass extends PsiNameIdentifierOwner, PyStatement, NameDefine
    */
   boolean isNewStyleClass();
 
-  @NotNull
-  List<PyClass> getAncestorClasses();
-
-  @NotNull
-  List<PyClass> getAncestorClasses(@NotNull TypeEvalContext context);
-
-  /**
-   * Get a list of all ancestor types.
-   */
-  @NotNull
-  List<PyClassLikeType> getAncestorTypes(@NotNull TypeEvalContext context);
-
   /**
    * Scan properties in order of definition, until processor returns true for one of them.
    * @param processor to check properties

python/src/com/jetbrains/python/PythonDialectsTokenSetProvider.java

@@ -4,6 +4,8 @@ import com.intellij.openapi.extensions.Extensions;
 import com.intellij.psi.tree.TokenSet;
 
 /**
+ * Provides element types of various kinds for known Python dialects.
+ *
  * @author vlan
  */
 public class PythonDialectsTokenSetProvider {
@@ -47,34 +49,58 @@ public class PythonDialectsTokenSetProvider {
     myReferenceExpressionTokens = referenceExpressions;
   }
 
+  /**
+   * Returns all element types of Python dialects that are subclasses of {@link com.jetbrains.python.psi.PyStatement}.
+   */
   public TokenSet getStatementTokens() {
     return myStatementTokens;
   }
 
+  /**
+   * Returns all element types of Python dialects that are subclasses of {@link com.jetbrains.python.psi.PyExpression}.
+   */
   public TokenSet getExpressionTokens() {
     return myExpressionTokens;
   }
 
+  /**
+   * Returns all element types of Python dialects that are subclasses of {@link com.jetbrains.python.psi.NameDefiner}.
+   */
   public TokenSet getNameDefinerTokens() {
     return myNameDefinerTokens;
   }
 
+  /**
+   * Returns all element types of Python dialects that are language keywords.
+   */
   public TokenSet getKeywordTokens() {
     return myKeywordTokens;
   }
 
+  /**
+   * Returns all element types of Python dialects that are subclasses of {@link com.jetbrains.python.psi.PyParameter}.
+   */
   public TokenSet getParameterTokens() {
     return myParameterTokens;
   }
 
+  /**
+   * Returns all element types of Python dialects that are subclasses of {@link com.jetbrains.python.psi.PyFunction}.
+   */
   public TokenSet getFunctionDeclarationTokens() {
     return myFunctionDeclarationTokens;
   }
 
+  /**
+   * Returns all element types of Python dialects that can be used as unbalanced braces recovery tokens in the lexer.
+   */
   public TokenSet getUnbalancedBracesRecoveryTokens() {
     return myUnbalancedBracesRecoveryTokens;
   }
 
+  /**
+   * Returns all element types of Python dialects that are subclasses of {@link com.jetbrains.python.psi.PyReferenceExpression}.
+   */
   public TokenSet getReferenceExpressionTokens() {
     return myReferenceExpressionTokens;
   }

python/src/com/jetbrains/python/PythonTokenSetContributor.java

@@ -1,13 +1,16 @@
 package com.jetbrains.python;
 
 import com.intellij.psi.tree.TokenSet;
+import org.jetbrains.annotations.NotNull;
+
 import static com.jetbrains.python.PyElementTypes.*;
 import static com.jetbrains.python.PyTokenTypes.*;
 
 /**
  * @author vlan
  */
-public class PythonTokenSetContributor implements PythonDialectsTokenSetContributor {
+public class PythonTokenSetContributor extends PythonDialectsTokenSetContributorBase {
+  @NotNull
   @Override
   public TokenSet getStatementTokens() {
     return TokenSet.create(EXPRESSION_STATEMENT, ASSIGNMENT_STATEMENT, AUG_ASSIGNMENT_STATEMENT, ASSERT_STATEMENT,
@@ -17,6 +20,7 @@ public class PythonTokenSetContributor implements PythonDialectsTokenSetContribu
                            WHILE_STATEMENT, NONLOCAL_STATEMENT, CLASS_DECLARATION, FUNCTION_DECLARATION);
   }
 
+  @NotNull
   @Override
   public TokenSet getExpressionTokens() {
     return TokenSet.create(EMPTY_EXPRESSION, REFERENCE_EXPRESSION, INTEGER_LITERAL_EXPRESSION, FLOAT_LITERAL_EXPRESSION,
@@ -29,6 +33,7 @@ public class PythonTokenSetContributor implements PythonDialectsTokenSetContribu
                            SET_LITERAL_EXPRESSION, SET_COMP_EXPRESSION, DICT_COMP_EXPRESSION, STAR_EXPRESSION);
   }
 
+  @NotNull
   @Override
   public TokenSet getNameDefinerTokens() {
     // FROM_IMPORT_STATEMENT is not exactly a NameDefiner but needed anyway in mypackage/__init__.py, 'from mypackage.foo import bar' makes
@@ -41,6 +46,7 @@ public class PythonTokenSetContributor implements PythonDialectsTokenSetContribu
                            FROM_IMPORT_STATEMENT);
   }
 
+  @NotNull
   @Override
   public TokenSet getKeywordTokens() {
     return TokenSet.create(
@@ -55,22 +61,26 @@ public class PythonTokenSetContributor implements PythonDialectsTokenSetContribu
       NONE_KEYWORD, TRUE_KEYWORD, FALSE_KEYWORD, NONLOCAL_KEYWORD, DEBUG_KEYWORD);
   }
 
+  @NotNull
   @Override
   public TokenSet getParameterTokens() {
     return TokenSet.create(NAMED_PARAMETER, TUPLE_PARAMETER, SINGLE_STAR_PARAMETER);
   }
 
+  @NotNull
   @Override
   public TokenSet getFunctionDeclarationTokens() {
     return TokenSet.create(FUNCTION_DECLARATION);
   }
 
+  @NotNull
   @Override
   public TokenSet getUnbalancedBracesRecoveryTokens() {
     return TokenSet.create(DEF_KEYWORD, CLASS_KEYWORD, RETURN_KEYWORD, WITH_KEYWORD, WHILE_KEYWORD, BREAK_KEYWORD, CONTINUE_KEYWORD,
                            RAISE_KEYWORD, TRY_KEYWORD, EXCEPT_KEYWORD, FINALLY_KEYWORD);
   }
 
+  @NotNull
   @Override
   public TokenSet getReferenceExpressionTokens() {
     return TokenSet.create(REFERENCE_EXPRESSION);

python/src/com/jetbrains/python/documentation/doctest/PyDocstringTokenSetContributor.java

@@ -1,19 +1,22 @@
 package com.jetbrains.python.documentation.doctest;
 
 import com.intellij.psi.tree.TokenSet;
-import com.jetbrains.python.PythonTokenSetContributor;
+import com.jetbrains.python.PythonDialectsTokenSetContributorBase;
+import org.jetbrains.annotations.NotNull;
 
 /**
  * User : ktisha
  */
-public class PyDocstringTokenSetContributor extends PythonTokenSetContributor {
+public class PyDocstringTokenSetContributor extends PythonDialectsTokenSetContributorBase {
   public static final TokenSet DOCSTRING_REFERENCE_EXPRESSIONS = TokenSet.create(PyDocstringTokenTypes.DOC_REFERENCE);
 
+  @NotNull
   @Override
   public TokenSet getExpressionTokens() {
     return DOCSTRING_REFERENCE_EXPRESSIONS;
   }
 
+  @NotNull
   @Override
   public TokenSet getReferenceExpressionTokens() {
     return DOCSTRING_REFERENCE_EXPRESSIONS;

python/src/com/jetbrains/python/inspections/PyOldStyleClassesInspection.java

@@ -2,11 +2,12 @@ package com.jetbrains.python.inspections;
 
 import com.intellij.codeInspection.LocalInspectionToolSession;
 import com.intellij.codeInspection.ProblemsHolder;
-import com.intellij.psi.PsiElement;
 import com.intellij.psi.PsiElementVisitor;
 import com.intellij.psi.util.PsiTreeUtil;
 import com.jetbrains.python.PyBundle;
 import com.jetbrains.python.psi.*;
+import com.jetbrains.python.psi.types.PyClassLikeType;
+import com.jetbrains.python.psi.types.TypeEvalContext;
 import org.jetbrains.annotations.Nls;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
@@ -57,12 +58,21 @@ public class PyOldStyleClassesInspection extends PyInspection {
     public void visitPyCallExpression(final PyCallExpression node) {
       PyClass klass = PsiTreeUtil.getParentOfType(node, PyClass.class);
       if (klass != null && !klass.isNewStyleClass()) {
-        PyExpression[] superClassExprs = klass.getSuperClassExpressions();
-        PsiElement[] superClasses = klass.getSuperClassElements();
-        if (superClasses.length != superClassExprs.length) return;
+        if (hasUnknownAncestors(klass, myTypeEvalContext)) {
+          return;
+        }
         if (PyUtil.isSuperCall(node))
           registerProblem(node.getCallee(), "Old-style class contains call for super method");
       }
     }
   }
+
+  private static boolean hasUnknownAncestors(@NotNull PyClass cls, @NotNull TypeEvalContext context) {
+    for (PyClassLikeType type : cls.getAncestorTypes(context)) {
+      if (type == null) {
+        return true;
+      }
+    }
+    return false;
+  }
 }

python/src/com/jetbrains/python/psi/impl/PyClassImpl.java

@@ -25,7 +25,6 @@ import com.jetbrains.python.codeInsight.controlflow.ScopeOwner;
 import com.jetbrains.python.codeInsight.dataflow.scope.ScopeUtil;
 import com.jetbrains.python.documentation.DocStringUtil;
 import com.jetbrains.python.psi.*;
-import com.jetbrains.python.psi.resolve.PyResolveContext;
 import com.jetbrains.python.psi.resolve.PyResolveUtil;
 import com.jetbrains.python.psi.resolve.QualifiedNameFinder;
 import com.jetbrains.python.psi.stubs.PropertyStubStorage;
@@ -159,31 +158,6 @@ public class PyClassImpl extends PyPresentableElementImpl<PyClassStub> implement
     return PyExpression.EMPTY_ARRAY;
   }
 
-  @NotNull
-  public PsiElement[] getSuperClassElements() {
-    final PyExpression[] superExpressions = getSuperClassExpressions();
-    List<PsiElement> superClasses = new ArrayList<PsiElement>();
-    for (PyExpression expr : superExpressions) {
-      if (expr instanceof PyKeywordArgument) {
-        continue;
-      }
-      superClasses.add(classElementFromExpression(expr));
-    }
-    return PsiUtilCore.toPsiElementArray(superClasses);
-  }
-
-  @Nullable
-  public static PsiElement classElementFromExpression(@NotNull PyExpression expression) {
-    expression = unfoldClass(expression);
-    if (expression instanceof PyReferenceExpression) {
-      final PsiPolyVariantReference ref = ((PyReferenceExpression)expression).getReference(PyResolveContext.noProperties());
-      if (ref != null) {
-        return ref.resolve();
-      }
-    }
-    return null;
-  }
-
   @NotNull
   public static PyExpression unfoldClass(@NotNull PyExpression expression) {
     if (expression instanceof PyCallExpression) {