diff --git a/python/psi-api/src/com/jetbrains/python/PythonDialectsTokenSetContributor.java b/python/psi-api/src/com/jetbrains/python/PythonDialectsTokenSetContributor.java index 5a7ef32790ba..798ec70e421f 100644 --- a/python/psi-api/src/com/jetbrains/python/PythonDialectsTokenSetContributor.java +++ b/python/psi-api/src/com/jetbrains/python/PythonDialectsTokenSetContributor.java @@ -15,4 +15,6 @@ public interface PythonDialectsTokenSetContributor { TokenSet getKeywordTokens(); TokenSet getParameterTokens(); TokenSet getFunctionDeclarationTokens(); + TokenSet getUnbalancedBracesRecoveryTokens(); + TokenSet getReferenceExpressionTokens(); } diff --git a/python/psi-api/src/com/jetbrains/python/psi/NameDefiner.java b/python/psi-api/src/com/jetbrains/python/psi/NameDefiner.java index f5537c7d0c44..870ca00a8a62 100644 --- a/python/psi-api/src/com/jetbrains/python/psi/NameDefiner.java +++ b/python/psi-api/src/com/jetbrains/python/psi/NameDefiner.java @@ -5,10 +5,10 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; /** - * PSI element that (re)defnies names in following namespace, e.g. as assignment statement does. + * PSI element that (re)defines names in following namespace, e.g. as assignment statement does. * * NOTE: When declaring additional elements as name definers, please also adjust the token set via - * {@link PythonDialectsTokenSetContributor#getNameDefinerTokens()}. + * {@link com.jetbrains.python.PythonDialectsTokenSetContributor#getNameDefinerTokens()}. * * @author dcheryasov */ diff --git a/python/psi-api/src/com/jetbrains/python/psi/PyImportElement.java b/python/psi-api/src/com/jetbrains/python/psi/PyImportElement.java index d53f5f2f93a9..b0ba10eced94 100644 --- a/python/psi-api/src/com/jetbrains/python/psi/PyImportElement.java +++ b/python/psi-api/src/com/jetbrains/python/psi/PyImportElement.java @@ -9,7 +9,7 @@ import org.jetbrains.annotations.Nullable; /** * @author yole */ -public interface PyImportElement extends PyElement, NameDefiner, StubBasedPsiElement { +public interface PyImportElement extends PyElement, PyImportedNameDefiner, StubBasedPsiElement { @Nullable PyReferenceExpression getImportReferenceExpression(); diff --git a/python/psi-api/src/com/jetbrains/python/psi/PyImportedNameDefiner.java b/python/psi-api/src/com/jetbrains/python/psi/PyImportedNameDefiner.java new file mode 100644 index 000000000000..8081d0fd38d9 --- /dev/null +++ b/python/psi-api/src/com/jetbrains/python/psi/PyImportedNameDefiner.java @@ -0,0 +1,9 @@ +package com.jetbrains.python.psi; + +/** + * Name definer that defines names imported somehow from other modules. + * + * @author vlan + */ +public interface PyImportedNameDefiner extends NameDefiner { +} diff --git a/python/psi-api/src/com/jetbrains/python/psi/PyStarImportElement.java b/python/psi-api/src/com/jetbrains/python/psi/PyStarImportElement.java index 1f91048da631..1a00a21c17d9 100644 --- a/python/psi-api/src/com/jetbrains/python/psi/PyStarImportElement.java +++ b/python/psi-api/src/com/jetbrains/python/psi/PyStarImportElement.java @@ -5,5 +5,5 @@ package com.jetbrains.python.psi; * User: dcheryasov * Date: Jul 28, 2008 */ -public interface PyStarImportElement extends PyElement, NameDefiner { +public interface PyStarImportElement extends PyElement, PyImportedNameDefiner { } diff --git a/python/src/com/jetbrains/python/PyElementTypes.java b/python/src/com/jetbrains/python/PyElementTypes.java index a76d47d1184e..7b58d74ac685 100644 --- a/python/src/com/jetbrains/python/PyElementTypes.java +++ b/python/src/com/jetbrains/python/PyElementTypes.java @@ -72,8 +72,6 @@ public interface PyElementTypes { PyElementType EMPTY_EXPRESSION = new PyElementType("EMPTY_EXPRESSION", PyEmptyExpressionImpl.class); PyElementType REFERENCE_EXPRESSION = new PyElementType("REFERENCE_EXPRESSION", PyReferenceExpressionImpl.class); - TokenSet REFERENCE_EXPRESSION_SET = TokenSet.create(REFERENCE_EXPRESSION); - PyStubElementType TARGET_EXPRESSION = new PyTargetExpressionElementType(); PyElementType INTEGER_LITERAL_EXPRESSION = new PyElementType("INTEGER_LITERAL_EXPRESSION", PyNumericLiteralExpressionImpl.class); PyElementType FLOAT_LITERAL_EXPRESSION = new PyElementType("FLOAT_LITERAL_EXPRESSION", PyNumericLiteralExpressionImpl.class); diff --git a/python/src/com/jetbrains/python/PythonDialectsTokenSetProvider.java b/python/src/com/jetbrains/python/PythonDialectsTokenSetProvider.java index 73375523ad99..1e0da69a7807 100644 --- a/python/src/com/jetbrains/python/PythonDialectsTokenSetProvider.java +++ b/python/src/com/jetbrains/python/PythonDialectsTokenSetProvider.java @@ -15,6 +15,8 @@ public class PythonDialectsTokenSetProvider { private final TokenSet myKeywordTokens; private final TokenSet myParameterTokens; private final TokenSet myFunctionDeclarationTokens; + private final TokenSet myUnbalancedBracesRecoveryTokens; + private final TokenSet myReferenceExpressionTokens; private PythonDialectsTokenSetProvider() { TokenSet stmts = TokenSet.EMPTY; @@ -23,6 +25,8 @@ public class PythonDialectsTokenSetProvider { TokenSet keywords = TokenSet.EMPTY; TokenSet parameters = TokenSet.EMPTY; TokenSet functionDeclarations = TokenSet.EMPTY; + TokenSet recoveryTokens = TokenSet.EMPTY; + TokenSet referenceExpressions = TokenSet.EMPTY; for(PythonDialectsTokenSetContributor contributor: Extensions.getExtensions(PythonDialectsTokenSetContributor.EP_NAME)) { stmts = TokenSet.orSet(stmts, contributor.getStatementTokens()); exprs = TokenSet.orSet(exprs, contributor.getExpressionTokens()); @@ -30,6 +34,8 @@ public class PythonDialectsTokenSetProvider { keywords = TokenSet.orSet(keywords, contributor.getKeywordTokens()); parameters = TokenSet.orSet(parameters, contributor.getParameterTokens()); functionDeclarations = TokenSet.orSet(functionDeclarations, contributor.getFunctionDeclarationTokens()); + recoveryTokens = TokenSet.orSet(recoveryTokens, contributor.getUnbalancedBracesRecoveryTokens()); + referenceExpressions = TokenSet.orSet(referenceExpressions, contributor.getReferenceExpressionTokens()); } myStatementTokens = stmts; myExpressionTokens = exprs; @@ -37,6 +43,8 @@ public class PythonDialectsTokenSetProvider { myKeywordTokens = keywords; myParameterTokens = parameters; myFunctionDeclarationTokens = functionDeclarations; + myUnbalancedBracesRecoveryTokens = recoveryTokens; + myReferenceExpressionTokens = referenceExpressions; } public TokenSet getStatementTokens() { @@ -62,4 +70,12 @@ public class PythonDialectsTokenSetProvider { public TokenSet getFunctionDeclarationTokens() { return myFunctionDeclarationTokens; } + + public TokenSet getUnbalancedBracesRecoveryTokens() { + return myUnbalancedBracesRecoveryTokens; + } + + public TokenSet getReferenceExpressionTokens() { + return myReferenceExpressionTokens; + } } diff --git a/python/src/com/jetbrains/python/PythonTokenSetContributor.java b/python/src/com/jetbrains/python/PythonTokenSetContributor.java index 4e1165a77868..878043dcb078 100644 --- a/python/src/com/jetbrains/python/PythonTokenSetContributor.java +++ b/python/src/com/jetbrains/python/PythonTokenSetContributor.java @@ -64,4 +64,15 @@ public class PythonTokenSetContributor implements PythonDialectsTokenSetContribu public TokenSet getFunctionDeclarationTokens() { return TokenSet.create(FUNCTION_DECLARATION); } + + @Override + public TokenSet getUnbalancedBracesRecoveryTokens() { + return TokenSet.create(DEF_KEYWORD, CLASS_KEYWORD, RETURN_KEYWORD, WITH_KEYWORD, WHILE_KEYWORD, BREAK_KEYWORD, CONTINUE_KEYWORD, + RAISE_KEYWORD, TRY_KEYWORD, EXCEPT_KEYWORD, FINALLY_KEYWORD); + } + + @Override + public TokenSet getReferenceExpressionTokens() { + return TokenSet.create(REFERENCE_EXPRESSION); + } } diff --git a/python/src/com/jetbrains/python/codeInsight/dataflow/scope/Scope.java b/python/src/com/jetbrains/python/codeInsight/dataflow/scope/Scope.java index d4aeb3f752c8..4882c509e157 100644 --- a/python/src/com/jetbrains/python/codeInsight/dataflow/scope/Scope.java +++ b/python/src/com/jetbrains/python/codeInsight/dataflow/scope/Scope.java @@ -3,7 +3,7 @@ package com.jetbrains.python.codeInsight.dataflow.scope; import com.intellij.codeInsight.dataflow.DFALimitExceededException; import com.intellij.psi.PsiElement; import com.intellij.psi.PsiNamedElement; -import com.jetbrains.python.psi.NameDefiner; +import com.jetbrains.python.psi.PyImportedNameDefiner; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -27,7 +27,7 @@ public interface Scope { boolean containsDeclaration(String name); @NotNull - List getNameDefiners(); + List getImportedNameDefiners(); @Nullable PsiNamedElement getNamedElement(String name); diff --git a/python/src/com/jetbrains/python/codeInsight/dataflow/scope/impl/ScopeImpl.java b/python/src/com/jetbrains/python/codeInsight/dataflow/scope/impl/ScopeImpl.java index 68ce31d527be..ab86fac9b9bb 100644 --- a/python/src/com/jetbrains/python/codeInsight/dataflow/scope/impl/ScopeImpl.java +++ b/python/src/com/jetbrains/python/codeInsight/dataflow/scope/impl/ScopeImpl.java @@ -6,7 +6,6 @@ import com.intellij.codeInsight.dataflow.map.DFAMap; import com.intellij.codeInsight.dataflow.map.DFAMapEngine; import com.intellij.psi.PsiElement; import com.intellij.psi.PsiNamedElement; -import com.jetbrains.cython.psi.CythonIncludeStatement; import com.jetbrains.python.codeInsight.controlflow.ControlFlowCache; import com.jetbrains.python.codeInsight.controlflow.ScopeOwner; import com.jetbrains.python.codeInsight.dataflow.PyReachingDefsDfaInstance; @@ -31,7 +30,7 @@ public class ScopeImpl implements Scope { private volatile List myNestedScopes; private final ScopeOwner myFlowOwner; private volatile Map myNamedElements; - private volatile List myNameDefiners; // declarations which declare unknown set of names, such as 'from ... import *' + private volatile List myImportedNameDefiners; // Declarations which declare unknown set of imported names private volatile Set myAugAssignments; public ScopeImpl(final ScopeOwner flowOwner) { @@ -97,7 +96,7 @@ public class ScopeImpl implements Scope { } public boolean containsDeclaration(final String name) { - if (myNamedElements == null || myNameDefiners == null) { + if (myNamedElements == null || myImportedNameDefiners == null) { collectDeclarations(); } if (isNonlocal(name)) { @@ -109,7 +108,7 @@ public class ScopeImpl implements Scope { if (isAugAssignment(name)) { return true; } - for (NameDefiner definer : getNameDefiners()) { + for (NameDefiner definer : getImportedNameDefiners()) { if (definer.getElementNamed(name) != null) { return true; } @@ -119,11 +118,11 @@ public class ScopeImpl implements Scope { @NotNull @Override - public List getNameDefiners() { - if (myNameDefiners == null) { + public List getImportedNameDefiners() { + if (myImportedNameDefiners == null) { collectDeclarations(); } - return myNameDefiners; + return myImportedNameDefiners; } @Nullable @@ -158,7 +157,7 @@ public class ScopeImpl implements Scope { private void collectDeclarations() { final Map namedElements = new HashMap(); - final List nameDefiners = new ArrayList(); + final List importedNameDefiners = new ArrayList(); final List nestedScopes = new ArrayList(); final Set globals = new HashSet(); final Set nonlocals = new HashSet(); @@ -214,9 +213,8 @@ public class ScopeImpl implements Scope { if (node instanceof PsiNamedElement && !(node instanceof PyKeywordArgument)) { namedElements.put(node.getName(), (PsiNamedElement)node); } - // TODO: Cython-specific code - if (node instanceof PyStarImportElement || node instanceof PyImportElement || node instanceof CythonIncludeStatement) { - nameDefiners.add((NameDefiner)node); + if (node instanceof PyImportedNameDefiner) { + importedNameDefiners.add((PyImportedNameDefiner)node); } if (node instanceof ScopeOwner) { final Scope scope = ControlFlowCache.getScope((ScopeOwner)node); @@ -228,7 +226,7 @@ public class ScopeImpl implements Scope { } }); - Collections.sort(nameDefiners, new Comparator() { + Collections.sort(importedNameDefiners, new Comparator() { @Override public int compare(NameDefiner d1, NameDefiner d2) { return getPriority(d2) - getPriority(d1); @@ -243,7 +241,7 @@ public class ScopeImpl implements Scope { }); myNamedElements = namedElements; - myNameDefiners = nameDefiners; + myImportedNameDefiners = importedNameDefiners; myNestedScopes = nestedScopes; myGlobals = globals; myNonlocals = nonlocals; diff --git a/python/src/com/jetbrains/python/console/parsing/PyConsoleParsingContext.java b/python/src/com/jetbrains/python/console/parsing/PyConsoleParsingContext.java index bd2ffb22d7e4..ba6eaae70527 100644 --- a/python/src/com/jetbrains/python/console/parsing/PyConsoleParsingContext.java +++ b/python/src/com/jetbrains/python/console/parsing/PyConsoleParsingContext.java @@ -138,7 +138,7 @@ public class PyConsoleParsingContext extends ParsingContext { if (myBuilder.getTokenType() == PyTokenTypes.IDENTIFIER) { myBuilder.advanceLexer(); - command.done(PyElementTypes.REFERENCE_EXPRESSION); + command.done(getReferenceType()); } else { expr.drop(); diff --git a/python/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java b/python/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java index 4ad93f840490..2cac358435f3 100644 --- a/python/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java +++ b/python/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java @@ -5,8 +5,8 @@ import com.intellij.lexer.FlexLexer; import com.intellij.lexer.MergingLexerAdapter; import com.intellij.psi.tree.IElementType; import com.intellij.psi.tree.TokenSet; -import com.jetbrains.cython.parser.CythonTokenTypes; import com.jetbrains.python.PyTokenTypes; +import com.jetbrains.python.PythonDialectsTokenSetProvider; import gnu.trove.TIntStack; import org.jetbrains.annotations.Nullable; @@ -21,13 +21,7 @@ public class PythonIndentingProcessor extends MergingLexerAdapter { private int myCurrentNewLineIndent = 0; private static final boolean DUMP_TOKENS = false; - private static final TokenSet RECOVERY_TOKENS = - TokenSet.create(PyTokenTypes.DEF_KEYWORD, PyTokenTypes.CLASS_KEYWORD, PyTokenTypes.RETURN_KEYWORD, PyTokenTypes.WITH_KEYWORD, - PyTokenTypes.WHILE_KEYWORD, PyTokenTypes.BREAK_KEYWORD, PyTokenTypes.CONTINUE_KEYWORD, PyTokenTypes.RAISE_KEYWORD, - PyTokenTypes.TRY_KEYWORD, PyTokenTypes.EXCEPT_KEYWORD, PyTokenTypes.FINALLY_KEYWORD, - // TODO: Cython dependency. Collect these tokens via the PythonDialectsTokenSetProvider (requires setting up the - // "Pythonid.dialectsTokenSetContributor" extension point in all the tests. - CythonTokenTypes.CDEF_KEYWORD, CythonTokenTypes.CPDEF_KEYWORD, CythonTokenTypes.CTYPEDEF_KEYWORD); + private final TokenSet RECOVERY_TOKENS = PythonDialectsTokenSetProvider.INSTANCE.getUnbalancedBracesRecoveryTokens(); public PythonIndentingProcessor(FlexLexer lexer, TokenSet tokens) { super(new FlexAdapter(lexer), tokens); diff --git a/python/src/com/jetbrains/python/psi/impl/PyDecoratorImpl.java b/python/src/com/jetbrains/python/psi/impl/PyDecoratorImpl.java index 05bee8401122..efa10ccd8128 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyDecoratorImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyDecoratorImpl.java @@ -7,6 +7,7 @@ import com.intellij.psi.util.PsiTreeUtil; import com.intellij.util.IncorrectOperationException; import com.jetbrains.python.PyElementTypes; import com.jetbrains.python.PyTokenTypes; +import com.jetbrains.python.PythonDialectsTokenSetProvider; import com.jetbrains.python.psi.*; import com.jetbrains.python.psi.resolve.PyResolveContext; import com.jetbrains.python.psi.resolve.PyResolveUtil; @@ -47,7 +48,7 @@ public class PyDecoratorImpl extends StubBasedPsiElementBase im } public boolean isBuiltin() { - ASTNode node = getNode().findChildByType(PyElementTypes.REFERENCE_EXPRESSION); + ASTNode node = getNode().findChildByType(PythonDialectsTokenSetProvider.INSTANCE.getReferenceExpressionTokens()); if (node != null) { PyReferenceExpression ref = (PyReferenceExpression)node.getPsi(); PsiElement target = ref.getReference().resolve(); diff --git a/python/src/com/jetbrains/python/psi/impl/PyFromImportStatementImpl.java b/python/src/com/jetbrains/python/psi/impl/PyFromImportStatementImpl.java index f887dece66af..b43511d0d442 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyFromImportStatementImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyFromImportStatementImpl.java @@ -4,11 +4,11 @@ import com.intellij.lang.ASTNode; import com.intellij.psi.*; import com.intellij.psi.scope.PsiScopeProcessor; import com.intellij.psi.stubs.IStubElementType; -import com.intellij.psi.tree.TokenSet; import com.intellij.util.ArrayFactory; import com.jetbrains.python.PyElementTypes; import com.jetbrains.python.PyNames; import com.jetbrains.python.PyTokenTypes; +import com.jetbrains.python.PythonDialectsTokenSetProvider; import com.jetbrains.python.psi.*; import com.jetbrains.python.psi.resolve.ResolveImportUtil; import com.jetbrains.python.psi.stubs.PyFromImportStatementStub; @@ -50,7 +50,7 @@ public class PyFromImportStatementImpl extends PyBaseElementImpl @Nullable public PyReferenceExpression getImportReferenceExpression() { - final ASTNode node = getNode().findChildByType(PyElementTypes.REFERENCE_EXPRESSION); + final ASTNode node = getNode().findChildByType(PythonDialectsTokenSetProvider.INSTANCE.getReferenceExpressionTokens()); return node == null ? null : (PyReferenceExpression) node.getPsi(); } diff --git a/python/src/com/jetbrains/python/psi/impl/PyNamedParameterImpl.java b/python/src/com/jetbrains/python/psi/impl/PyNamedParameterImpl.java index f9f175265e56..05cd3ccc91db 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyNamedParameterImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyNamedParameterImpl.java @@ -12,7 +12,6 @@ import com.intellij.psi.util.PsiTreeUtil; import com.intellij.util.IncorrectOperationException; import com.intellij.util.PlatformIcons; import com.intellij.util.Processor; -import com.jetbrains.cython.psi.CythonClass; import com.jetbrains.python.PyElementTypes; import com.jetbrains.python.PyNames; import com.jetbrains.python.PyTokenTypes; @@ -310,9 +309,6 @@ public class PyNamedParameterImpl extends PyPresentableElementImpl