From a2b76b7735cd2d46a881464278294e6cb8fd8c36 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Tue, 17 Sep 2013 19:26:10 +0400 Subject: [PATCH 1/4] Docs and the base class for Python token set contributors --- .../PythonDialectsTokenSetContributor.java | 42 ++++++++++++++ ...PythonDialectsTokenSetContributorBase.java | 57 +++++++++++++++++++ .../PythonDialectsTokenSetProvider.java | 26 +++++++++ .../python/PythonTokenSetContributor.java | 12 +++- .../PyDocstringTokenSetContributor.java | 7 ++- 5 files changed, 141 insertions(+), 3 deletions(-) create mode 100644 python/psi-api/src/com/jetbrains/python/PythonDialectsTokenSetContributorBase.java diff --git a/python/psi-api/src/com/jetbrains/python/PythonDialectsTokenSetContributor.java b/python/psi-api/src/com/jetbrains/python/PythonDialectsTokenSetContributor.java index 798ec70e421f..0600c6849f47 100644 --- a/python/psi-api/src/com/jetbrains/python/PythonDialectsTokenSetContributor.java +++ b/python/psi-api/src/com/jetbrains/python/PythonDialectsTokenSetContributor.java @@ -2,19 +2,61 @@ package com.jetbrains.python; import com.intellij.openapi.extensions.ExtensionPointName; import com.intellij.psi.tree.TokenSet; +import org.jetbrains.annotations.NotNull; /** + * Contributes element types of various kinds specific for a particular Python dialect. + * * @author vlan */ public interface PythonDialectsTokenSetContributor { ExtensionPointName EP_NAME = ExtensionPointName.create("Pythonid.dialectsTokenSetContributor"); + /** + * Returns element types that are subclasses of {@link com.jetbrains.python.psi.PyStatement}. + */ + @NotNull TokenSet getStatementTokens(); + + /** + * Returns element types that are subclasses of {@link com.jetbrains.python.psi.PyExpression}. + */ + @NotNull TokenSet getExpressionTokens(); + + /** + * Returns element types that are subclasses of {@link com.jetbrains.python.psi.NameDefiner}. + */ + @NotNull TokenSet getNameDefinerTokens(); + + /** + * Returns element types that are language keywords. + */ + @NotNull TokenSet getKeywordTokens(); + + /** + * Returns element types that are subclasses of {@link com.jetbrains.python.psi.PyParameter}. + */ + @NotNull TokenSet getParameterTokens(); + + /** + * Returns element types that are subclasses of {@link com.jetbrains.python.psi.PyFunction}. + */ + @NotNull TokenSet getFunctionDeclarationTokens(); + + /** + * Returns element types that can be used as unbalanced braces recovery tokens in the lexer. + */ + @NotNull TokenSet getUnbalancedBracesRecoveryTokens(); + + /** + * Returns element types that are subclasses of {@link com.jetbrains.python.psi.PyReferenceExpression}. + */ + @NotNull TokenSet getReferenceExpressionTokens(); } diff --git a/python/psi-api/src/com/jetbrains/python/PythonDialectsTokenSetContributorBase.java b/python/psi-api/src/com/jetbrains/python/PythonDialectsTokenSetContributorBase.java new file mode 100644 index 000000000000..5fd0d47e953b --- /dev/null +++ b/python/psi-api/src/com/jetbrains/python/PythonDialectsTokenSetContributorBase.java @@ -0,0 +1,57 @@ +package com.jetbrains.python; + +import com.intellij.psi.tree.TokenSet; +import org.jetbrains.annotations.NotNull; + +/** + * @author vlan + */ +public abstract class PythonDialectsTokenSetContributorBase implements PythonDialectsTokenSetContributor { + @NotNull + @Override + public TokenSet getStatementTokens() { + return TokenSet.EMPTY; + } + + @NotNull + @Override + public TokenSet getExpressionTokens() { + return TokenSet.EMPTY; + } + + @NotNull + @Override + public TokenSet getNameDefinerTokens() { + return TokenSet.EMPTY; + } + + @NotNull + @Override + public TokenSet getKeywordTokens() { + return TokenSet.EMPTY; + } + + @NotNull + @Override + public TokenSet getParameterTokens() { + return TokenSet.EMPTY; + } + + @NotNull + @Override + public TokenSet getFunctionDeclarationTokens() { + return TokenSet.EMPTY; + } + + @NotNull + @Override + public TokenSet getUnbalancedBracesRecoveryTokens() { + return TokenSet.EMPTY; + } + + @NotNull + @Override + public TokenSet getReferenceExpressionTokens() { + return TokenSet.EMPTY; + } +} diff --git a/python/src/com/jetbrains/python/PythonDialectsTokenSetProvider.java b/python/src/com/jetbrains/python/PythonDialectsTokenSetProvider.java index 1e0da69a7807..34bb3263dc36 100644 --- a/python/src/com/jetbrains/python/PythonDialectsTokenSetProvider.java +++ b/python/src/com/jetbrains/python/PythonDialectsTokenSetProvider.java @@ -4,6 +4,8 @@ import com.intellij.openapi.extensions.Extensions; import com.intellij.psi.tree.TokenSet; /** + * Provides element types of various kinds for known Python dialects. + * * @author vlan */ public class PythonDialectsTokenSetProvider { @@ -47,34 +49,58 @@ public class PythonDialectsTokenSetProvider { myReferenceExpressionTokens = referenceExpressions; } + /** + * Returns all element types of Python dialects that are subclasses of {@link com.jetbrains.python.psi.PyStatement}. + */ public TokenSet getStatementTokens() { return myStatementTokens; } + /** + * Returns all element types of Python dialects that are subclasses of {@link com.jetbrains.python.psi.PyExpression}. + */ public TokenSet getExpressionTokens() { return myExpressionTokens; } + /** + * Returns all element types of Python dialects that are subclasses of {@link com.jetbrains.python.psi.NameDefiner}. + */ public TokenSet getNameDefinerTokens() { return myNameDefinerTokens; } + /** + * Returns all element types of Python dialects that are language keywords. + */ public TokenSet getKeywordTokens() { return myKeywordTokens; } + /** + * Returns all element types of Python dialects that are subclasses of {@link com.jetbrains.python.psi.PyParameter}. + */ public TokenSet getParameterTokens() { return myParameterTokens; } + /** + * Returns all element types of Python dialects that are subclasses of {@link com.jetbrains.python.psi.PyFunction}. + */ public TokenSet getFunctionDeclarationTokens() { return myFunctionDeclarationTokens; } + /** + * Returns all element types of Python dialects that can be used as unbalanced braces recovery tokens in the lexer. + */ public TokenSet getUnbalancedBracesRecoveryTokens() { return myUnbalancedBracesRecoveryTokens; } + /** + * Returns all element types of Python dialects that are subclasses of {@link com.jetbrains.python.psi.PyReferenceExpression}. + */ public TokenSet getReferenceExpressionTokens() { return myReferenceExpressionTokens; } diff --git a/python/src/com/jetbrains/python/PythonTokenSetContributor.java b/python/src/com/jetbrains/python/PythonTokenSetContributor.java index 878043dcb078..fc121647bec9 100644 --- a/python/src/com/jetbrains/python/PythonTokenSetContributor.java +++ b/python/src/com/jetbrains/python/PythonTokenSetContributor.java @@ -1,13 +1,16 @@ package com.jetbrains.python; import com.intellij.psi.tree.TokenSet; +import org.jetbrains.annotations.NotNull; + import static com.jetbrains.python.PyElementTypes.*; import static com.jetbrains.python.PyTokenTypes.*; /** * @author vlan */ -public class PythonTokenSetContributor implements PythonDialectsTokenSetContributor { +public class PythonTokenSetContributor extends PythonDialectsTokenSetContributorBase { + @NotNull @Override public TokenSet getStatementTokens() { return TokenSet.create(EXPRESSION_STATEMENT, ASSIGNMENT_STATEMENT, AUG_ASSIGNMENT_STATEMENT, ASSERT_STATEMENT, @@ -17,6 +20,7 @@ public class PythonTokenSetContributor implements PythonDialectsTokenSetContribu WHILE_STATEMENT, NONLOCAL_STATEMENT, CLASS_DECLARATION, FUNCTION_DECLARATION); } + @NotNull @Override public TokenSet getExpressionTokens() { return TokenSet.create(EMPTY_EXPRESSION, REFERENCE_EXPRESSION, INTEGER_LITERAL_EXPRESSION, FLOAT_LITERAL_EXPRESSION, @@ -29,6 +33,7 @@ public class PythonTokenSetContributor implements PythonDialectsTokenSetContribu SET_LITERAL_EXPRESSION, SET_COMP_EXPRESSION, DICT_COMP_EXPRESSION, STAR_EXPRESSION); } + @NotNull @Override public TokenSet getNameDefinerTokens() { // FROM_IMPORT_STATEMENT is not exactly a NameDefiner but needed anyway in mypackage/__init__.py, 'from mypackage.foo import bar' makes @@ -41,6 +46,7 @@ public class PythonTokenSetContributor implements PythonDialectsTokenSetContribu FROM_IMPORT_STATEMENT); } + @NotNull @Override public TokenSet getKeywordTokens() { return TokenSet.create( @@ -55,22 +61,26 @@ public class PythonTokenSetContributor implements PythonDialectsTokenSetContribu NONE_KEYWORD, TRUE_KEYWORD, FALSE_KEYWORD, NONLOCAL_KEYWORD, DEBUG_KEYWORD); } + @NotNull @Override public TokenSet getParameterTokens() { return TokenSet.create(NAMED_PARAMETER, TUPLE_PARAMETER, SINGLE_STAR_PARAMETER); } + @NotNull @Override public TokenSet getFunctionDeclarationTokens() { return TokenSet.create(FUNCTION_DECLARATION); } + @NotNull @Override public TokenSet getUnbalancedBracesRecoveryTokens() { return TokenSet.create(DEF_KEYWORD, CLASS_KEYWORD, RETURN_KEYWORD, WITH_KEYWORD, WHILE_KEYWORD, BREAK_KEYWORD, CONTINUE_KEYWORD, RAISE_KEYWORD, TRY_KEYWORD, EXCEPT_KEYWORD, FINALLY_KEYWORD); } + @NotNull @Override public TokenSet getReferenceExpressionTokens() { return TokenSet.create(REFERENCE_EXPRESSION); diff --git a/python/src/com/jetbrains/python/documentation/doctest/PyDocstringTokenSetContributor.java b/python/src/com/jetbrains/python/documentation/doctest/PyDocstringTokenSetContributor.java index 7059c6af8531..92d3113a7721 100644 --- a/python/src/com/jetbrains/python/documentation/doctest/PyDocstringTokenSetContributor.java +++ b/python/src/com/jetbrains/python/documentation/doctest/PyDocstringTokenSetContributor.java @@ -1,19 +1,22 @@ package com.jetbrains.python.documentation.doctest; import com.intellij.psi.tree.TokenSet; -import com.jetbrains.python.PythonTokenSetContributor; +import com.jetbrains.python.PythonDialectsTokenSetContributorBase; +import org.jetbrains.annotations.NotNull; /** * User : ktisha */ -public class PyDocstringTokenSetContributor extends PythonTokenSetContributor { +public class PyDocstringTokenSetContributor extends PythonDialectsTokenSetContributorBase { public static final TokenSet DOCSTRING_REFERENCE_EXPRESSIONS = TokenSet.create(PyDocstringTokenTypes.DOC_REFERENCE); + @NotNull @Override public TokenSet getExpressionTokens() { return DOCSTRING_REFERENCE_EXPRESSIONS; } + @NotNull @Override public TokenSet getReferenceExpressionTokens() { return DOCSTRING_REFERENCE_EXPRESSIONS; From 9ba8bb64879685536700756bf8eec556600ee647 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Tue, 17 Sep 2013 20:25:55 +0400 Subject: [PATCH 2/4] Docs for super classes and ancestors-related methods of PyClass --- .../src/com/jetbrains/python/psi/PyClass.java | 74 ++++++++++++++----- 1 file changed, 57 insertions(+), 17 deletions(-) diff --git a/python/psi-api/src/com/jetbrains/python/psi/PyClass.java b/python/psi-api/src/com/jetbrains/python/psi/PyClass.java index bdc69149a8d5..1e69ee5b02f5 100644 --- a/python/psi-api/src/com/jetbrains/python/psi/PyClass.java +++ b/python/psi-api/src/com/jetbrains/python/psi/PyClass.java @@ -36,21 +36,73 @@ public interface PyClass extends PsiNameIdentifierOwner, PyStatement, NameDefine @NotNull PyStatementList getStatementList(); - @Nullable - PyArgumentList getSuperClassExpressionList(); - + /** + * Returns types of all ancestors from the hierarchy. + */ @NotNull - PyExpression[] getSuperClassExpressions(); + List getAncestorTypes(@NotNull TypeEvalContext context); + /** + * Returns only those ancestors from the hierarchy, that are resolved to PyClass PSI elements. + * + * @see #getAncestorTypes(TypeEvalContext) for the full list of ancestors. + */ @NotNull - PsiElement[] getSuperClassElements(); + List getAncestorClasses(@NotNull TypeEvalContext context); + /** + * Returns types of all ancestors from the hierarchy using the default type evaluation context. + * + * @see #getAncestorClasses(TypeEvalContext) if a more detailed TypeEvalContext is available. + */ + @NotNull + List getAncestorClasses(); + + /** + * Returns types of expressions in the super classes list. + * + * If no super classes are specified, returns the type of the implicit super class for old- and new-style classes. + * + * @see #getAncestorTypes(TypeEvalContext) for the full list of ancestors. + */ @NotNull List getSuperClassTypes(@NotNull TypeEvalContext context); + /** + * Returns only those super classes for expressions from the super classes list, that are resolved to PyClass PSI elements. + * + * If no super classes are specified, returns the implicit super class for old- and new-style classes. + * + * @see #getSuperClassTypes(TypeEvalContext) for the full list of super classes. + * @see #getAncestorTypes(TypeEvalContext) for the full list of ancestors. + */ @NotNull PyClass[] getSuperClasses(); + /** + * Returns a PSI element for the super classes list. + * + * Operates at the AST level. + */ + @Nullable + PyArgumentList getSuperClassExpressionList(); + + /** + * Returns PSI elements for the expressions in the super classes list. + * + * Operates at the AST level. + */ + @NotNull + PyExpression[] getSuperClassExpressions(); + + /** + * Returns resolved elements representing super classes from the super classes list. + * + * Operates at the AST level. + */ + @NotNull + PsiElement[] getSuperClassElements(); + @NotNull PyFunction[] getMethods(); @@ -111,18 +163,6 @@ public interface PyClass extends PsiNameIdentifierOwner, PyStatement, NameDefine */ boolean isNewStyleClass(); - @NotNull - List getAncestorClasses(); - - @NotNull - List getAncestorClasses(@NotNull TypeEvalContext context); - - /** - * Get a list of all ancestor types. - */ - @NotNull - List getAncestorTypes(@NotNull TypeEvalContext context); - /** * Scan properties in order of definition, until processor returns true for one of them. * @param processor to check properties From 09fdfb698677c1188997b0fcce409df05e1c60f5 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Tue, 17 Sep 2013 20:59:01 +0400 Subject: [PATCH 3/4] Removed redundant PyClass.getSuperClassElements() method --- .../python/psi/impl/PyJavaClassType.java | 4 +++ .../PyJavaSuperMethodsSearchExecutor.java | 9 ++++--- .../src/com/jetbrains/python/psi/PyClass.java | 8 ------ .../PyOldStyleClassesInspection.java | 18 ++++++++++--- .../python/psi/impl/PyClassImpl.java | 26 ------------------- 5 files changed, 23 insertions(+), 42 deletions(-) diff --git a/python/pluginSrc/com/jetbrains/python/psi/impl/PyJavaClassType.java b/python/pluginSrc/com/jetbrains/python/psi/impl/PyJavaClassType.java index 18a8a80c0ec2..5a6105b10b2d 100644 --- a/python/pluginSrc/com/jetbrains/python/psi/impl/PyJavaClassType.java +++ b/python/pluginSrc/com/jetbrains/python/psi/impl/PyJavaClassType.java @@ -132,4 +132,8 @@ public class PyJavaClassType implements PyClassLikeType { public boolean isValid() { return myClass.isValid(); } + + public PsiClass getPsiClass() { + return myClass; + } } diff --git a/python/pluginSrc/com/jetbrains/python/psi/impl/PyJavaSuperMethodsSearchExecutor.java b/python/pluginSrc/com/jetbrains/python/psi/impl/PyJavaSuperMethodsSearchExecutor.java index 5c99c28df78a..93f8cce3d72f 100644 --- a/python/pluginSrc/com/jetbrains/python/psi/impl/PyJavaSuperMethodsSearchExecutor.java +++ b/python/pluginSrc/com/jetbrains/python/psi/impl/PyJavaSuperMethodsSearchExecutor.java @@ -9,6 +9,8 @@ import com.intellij.util.containers.ContainerUtil; import com.jetbrains.python.psi.PyClass; import com.jetbrains.python.psi.PyFunction; import com.jetbrains.python.psi.search.PySuperMethodsSearch; +import com.jetbrains.python.psi.types.PyClassLikeType; +import com.jetbrains.python.psi.types.TypeEvalContext; import org.jetbrains.annotations.NotNull; /** @@ -19,10 +21,9 @@ public class PyJavaSuperMethodsSearchExecutor implements QueryExecutor implement return PyExpression.EMPTY_ARRAY; } - @NotNull - public PsiElement[] getSuperClassElements() { - final PyExpression[] superExpressions = getSuperClassExpressions(); - List superClasses = new ArrayList(); - for (PyExpression expr : superExpressions) { - if (expr instanceof PyKeywordArgument) { - continue; - } - superClasses.add(classElementFromExpression(expr)); - } - return PsiUtilCore.toPsiElementArray(superClasses); - } - - @Nullable - public static PsiElement classElementFromExpression(@NotNull PyExpression expression) { - expression = unfoldClass(expression); - if (expression instanceof PyReferenceExpression) { - final PsiPolyVariantReference ref = ((PyReferenceExpression)expression).getReference(PyResolveContext.noProperties()); - if (ref != null) { - return ref.resolve(); - } - } - return null; - } - @NotNull public static PyExpression unfoldClass(@NotNull PyExpression expression) { if (expression instanceof PyCallExpression) { From e2dc3c839cc6a3cc20607ee97b6e36af3effe734 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Tue, 17 Sep 2013 21:06:39 +0400 Subject: [PATCH 4/4] Fixed docstring --- python/psi-api/src/com/jetbrains/python/psi/PyClass.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/psi-api/src/com/jetbrains/python/psi/PyClass.java b/python/psi-api/src/com/jetbrains/python/psi/PyClass.java index 32529cfd3f86..80ac706fa490 100644 --- a/python/psi-api/src/com/jetbrains/python/psi/PyClass.java +++ b/python/psi-api/src/com/jetbrains/python/psi/PyClass.java @@ -51,7 +51,7 @@ public interface PyClass extends PsiNameIdentifierOwner, PyStatement, NameDefine List getAncestorClasses(@NotNull TypeEvalContext context); /** - * Returns types of all ancestors from the hierarchy using the default type evaluation context. + * Returns only those ancestors from the hierarchy, that are resolved to PyClass PSI elements, using the default type evaluation context. * * @see #getAncestorClasses(TypeEvalContext) if a more detailed TypeEvalContext is available. */