From 9a1c7fe63259892838954574ae35162d0f454045 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Wed, 28 Aug 2013 16:47:50 +0400 Subject: [PATCH 01/11] Updated javadoc --- python/psi-api/src/com/jetbrains/python/psi/NameDefiner.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/psi-api/src/com/jetbrains/python/psi/NameDefiner.java b/python/psi-api/src/com/jetbrains/python/psi/NameDefiner.java index f5537c7d0c44..870ca00a8a62 100644 --- a/python/psi-api/src/com/jetbrains/python/psi/NameDefiner.java +++ b/python/psi-api/src/com/jetbrains/python/psi/NameDefiner.java @@ -5,10 +5,10 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; /** - * PSI element that (re)defnies names in following namespace, e.g. as assignment statement does. + * PSI element that (re)defines names in following namespace, e.g. as assignment statement does. * * NOTE: When declaring additional elements as name definers, please also adjust the token set via - * {@link PythonDialectsTokenSetContributor#getNameDefinerTokens()}. + * {@link com.jetbrains.python.PythonDialectsTokenSetContributor#getNameDefinerTokens()}. * * @author dcheryasov */ From 789d18607e4c912cc05db18f077c40fcd57fa3ff Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Wed, 28 Aug 2013 16:58:28 +0400 Subject: [PATCH 02/11] Name definers collected in ScopeImpl made PyImportedNameDefiner instances --- .../jetbrains/python/psi/PyImportElement.java | 2 +- .../python/psi/PyImportedNameDefiner.java | 9 +++++++ .../python/psi/PyStarImportElement.java | 2 +- .../codeInsight/dataflow/scope/Scope.java | 4 ++-- .../dataflow/scope/impl/ScopeImpl.java | 24 +++++++++---------- .../python/psi/resolve/PyResolveUtil.java | 2 +- 6 files changed, 25 insertions(+), 18 deletions(-) create mode 100644 python/psi-api/src/com/jetbrains/python/psi/PyImportedNameDefiner.java diff --git a/python/psi-api/src/com/jetbrains/python/psi/PyImportElement.java b/python/psi-api/src/com/jetbrains/python/psi/PyImportElement.java index d53f5f2f93a9..b0ba10eced94 100644 --- a/python/psi-api/src/com/jetbrains/python/psi/PyImportElement.java +++ b/python/psi-api/src/com/jetbrains/python/psi/PyImportElement.java @@ -9,7 +9,7 @@ import org.jetbrains.annotations.Nullable; /** * @author yole */ -public interface PyImportElement extends PyElement, NameDefiner, StubBasedPsiElement { +public interface PyImportElement extends PyElement, PyImportedNameDefiner, StubBasedPsiElement { @Nullable PyReferenceExpression getImportReferenceExpression(); diff --git a/python/psi-api/src/com/jetbrains/python/psi/PyImportedNameDefiner.java b/python/psi-api/src/com/jetbrains/python/psi/PyImportedNameDefiner.java new file mode 100644 index 000000000000..8081d0fd38d9 --- /dev/null +++ b/python/psi-api/src/com/jetbrains/python/psi/PyImportedNameDefiner.java @@ -0,0 +1,9 @@ +package com.jetbrains.python.psi; + +/** + * Name definer that defines names imported somehow from other modules. + * + * @author vlan + */ +public interface PyImportedNameDefiner extends NameDefiner { +} diff --git a/python/psi-api/src/com/jetbrains/python/psi/PyStarImportElement.java b/python/psi-api/src/com/jetbrains/python/psi/PyStarImportElement.java index 1f91048da631..1a00a21c17d9 100644 --- a/python/psi-api/src/com/jetbrains/python/psi/PyStarImportElement.java +++ b/python/psi-api/src/com/jetbrains/python/psi/PyStarImportElement.java @@ -5,5 +5,5 @@ package com.jetbrains.python.psi; * User: dcheryasov * Date: Jul 28, 2008 */ -public interface PyStarImportElement extends PyElement, NameDefiner { +public interface PyStarImportElement extends PyElement, PyImportedNameDefiner { } diff --git a/python/src/com/jetbrains/python/codeInsight/dataflow/scope/Scope.java b/python/src/com/jetbrains/python/codeInsight/dataflow/scope/Scope.java index d4aeb3f752c8..4882c509e157 100644 --- a/python/src/com/jetbrains/python/codeInsight/dataflow/scope/Scope.java +++ b/python/src/com/jetbrains/python/codeInsight/dataflow/scope/Scope.java @@ -3,7 +3,7 @@ package com.jetbrains.python.codeInsight.dataflow.scope; import com.intellij.codeInsight.dataflow.DFALimitExceededException; import com.intellij.psi.PsiElement; import com.intellij.psi.PsiNamedElement; -import com.jetbrains.python.psi.NameDefiner; +import com.jetbrains.python.psi.PyImportedNameDefiner; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -27,7 +27,7 @@ public interface Scope { boolean containsDeclaration(String name); @NotNull - List getNameDefiners(); + List getImportedNameDefiners(); @Nullable PsiNamedElement getNamedElement(String name); diff --git a/python/src/com/jetbrains/python/codeInsight/dataflow/scope/impl/ScopeImpl.java b/python/src/com/jetbrains/python/codeInsight/dataflow/scope/impl/ScopeImpl.java index 68ce31d527be..ab86fac9b9bb 100644 --- a/python/src/com/jetbrains/python/codeInsight/dataflow/scope/impl/ScopeImpl.java +++ b/python/src/com/jetbrains/python/codeInsight/dataflow/scope/impl/ScopeImpl.java @@ -6,7 +6,6 @@ import com.intellij.codeInsight.dataflow.map.DFAMap; import com.intellij.codeInsight.dataflow.map.DFAMapEngine; import com.intellij.psi.PsiElement; import com.intellij.psi.PsiNamedElement; -import com.jetbrains.cython.psi.CythonIncludeStatement; import com.jetbrains.python.codeInsight.controlflow.ControlFlowCache; import com.jetbrains.python.codeInsight.controlflow.ScopeOwner; import com.jetbrains.python.codeInsight.dataflow.PyReachingDefsDfaInstance; @@ -31,7 +30,7 @@ public class ScopeImpl implements Scope { private volatile List myNestedScopes; private final ScopeOwner myFlowOwner; private volatile Map myNamedElements; - private volatile List myNameDefiners; // declarations which declare unknown set of names, such as 'from ... import *' + private volatile List myImportedNameDefiners; // Declarations which declare unknown set of imported names private volatile Set myAugAssignments; public ScopeImpl(final ScopeOwner flowOwner) { @@ -97,7 +96,7 @@ public class ScopeImpl implements Scope { } public boolean containsDeclaration(final String name) { - if (myNamedElements == null || myNameDefiners == null) { + if (myNamedElements == null || myImportedNameDefiners == null) { collectDeclarations(); } if (isNonlocal(name)) { @@ -109,7 +108,7 @@ public class ScopeImpl implements Scope { if (isAugAssignment(name)) { return true; } - for (NameDefiner definer : getNameDefiners()) { + for (NameDefiner definer : getImportedNameDefiners()) { if (definer.getElementNamed(name) != null) { return true; } @@ -119,11 +118,11 @@ public class ScopeImpl implements Scope { @NotNull @Override - public List getNameDefiners() { - if (myNameDefiners == null) { + public List getImportedNameDefiners() { + if (myImportedNameDefiners == null) { collectDeclarations(); } - return myNameDefiners; + return myImportedNameDefiners; } @Nullable @@ -158,7 +157,7 @@ public class ScopeImpl implements Scope { private void collectDeclarations() { final Map namedElements = new HashMap(); - final List nameDefiners = new ArrayList(); + final List importedNameDefiners = new ArrayList(); final List nestedScopes = new ArrayList(); final Set globals = new HashSet(); final Set nonlocals = new HashSet(); @@ -214,9 +213,8 @@ public class ScopeImpl implements Scope { if (node instanceof PsiNamedElement && !(node instanceof PyKeywordArgument)) { namedElements.put(node.getName(), (PsiNamedElement)node); } - // TODO: Cython-specific code - if (node instanceof PyStarImportElement || node instanceof PyImportElement || node instanceof CythonIncludeStatement) { - nameDefiners.add((NameDefiner)node); + if (node instanceof PyImportedNameDefiner) { + importedNameDefiners.add((PyImportedNameDefiner)node); } if (node instanceof ScopeOwner) { final Scope scope = ControlFlowCache.getScope((ScopeOwner)node); @@ -228,7 +226,7 @@ public class ScopeImpl implements Scope { } }); - Collections.sort(nameDefiners, new Comparator() { + Collections.sort(importedNameDefiners, new Comparator() { @Override public int compare(NameDefiner d1, NameDefiner d2) { return getPriority(d2) - getPriority(d1); @@ -243,7 +241,7 @@ public class ScopeImpl implements Scope { }); myNamedElements = namedElements; - myNameDefiners = nameDefiners; + myImportedNameDefiners = importedNameDefiners; myNestedScopes = nestedScopes; myGlobals = globals; myNonlocals = nonlocals; diff --git a/python/src/com/jetbrains/python/psi/resolve/PyResolveUtil.java b/python/src/com/jetbrains/python/psi/resolve/PyResolveUtil.java index c39173f7a531..3d490f57d71f 100644 --- a/python/src/com/jetbrains/python/psi/resolve/PyResolveUtil.java +++ b/python/src/com/jetbrains/python/psi/resolve/PyResolveUtil.java @@ -125,7 +125,7 @@ public class PyResolveUtil { } } } - for (NameDefiner definer : scope.getNameDefiners()) { + for (NameDefiner definer : scope.getImportedNameDefiners()) { if (!processor.execute(definer, ResolveState.initial())) { found = true; break; From 9b0c92d278545692da3d444e4708f420e42695e4 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Wed, 28 Aug 2013 17:03:01 +0400 Subject: [PATCH 03/11] Low-rate import resolve results generalized as PyImportedNameDefiners --- .../python/psi/impl/references/PyReferenceImpl.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/python/src/com/jetbrains/python/psi/impl/references/PyReferenceImpl.java b/python/src/com/jetbrains/python/psi/impl/references/PyReferenceImpl.java index d6597cdd84fe..66107c7ffe13 100644 --- a/python/src/com/jetbrains/python/psi/impl/references/PyReferenceImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/references/PyReferenceImpl.java @@ -16,8 +16,6 @@ import com.intellij.psi.util.PsiTreeUtil; import com.intellij.util.IncorrectOperationException; import com.intellij.util.PlatformIcons; import com.intellij.util.ProcessingContext; -import com.jetbrains.cython.CythonLanguageDialect; -import com.jetbrains.cython.psi.CythonIncludeStatement; import com.jetbrains.cython.psi.CythonNamedElement; import com.jetbrains.cython.psi.CythonVariable; import com.jetbrains.python.PyNames; @@ -373,10 +371,7 @@ public class PyReferenceImpl implements PsiReferenceEx, PsiPolyVariantReference public static int getRate(PsiElement elt) { int rate; - if (CythonLanguageDialect.isInsideCythonFile(elt) && elt instanceof CythonIncludeStatement) { - rate = RatedResolveResult.RATE_LOW; - } - else if (elt instanceof PyImportElement || elt instanceof PyStarImportElement || elt instanceof PyReferenceExpression) { + if (elt instanceof PyImportedNameDefiner || elt instanceof PyReferenceExpression) { rate = RatedResolveResult.RATE_LOW; } else if (elt instanceof PyFile) { From 2069d45aebc9eb76caafd707f7eb8bf6f228f0bf Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Wed, 28 Aug 2013 17:27:17 +0400 Subject: [PATCH 04/11] Removed unnecessary Cython check Cython-related resolve logic has been already extracted into CythonReferenceResolveProvider. --- .../python/psi/impl/references/PyReferenceImpl.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/python/src/com/jetbrains/python/psi/impl/references/PyReferenceImpl.java b/python/src/com/jetbrains/python/psi/impl/references/PyReferenceImpl.java index 66107c7ffe13..91409ed4d160 100644 --- a/python/src/com/jetbrains/python/psi/impl/references/PyReferenceImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/references/PyReferenceImpl.java @@ -16,7 +16,6 @@ import com.intellij.psi.util.PsiTreeUtil; import com.intellij.util.IncorrectOperationException; import com.intellij.util.PlatformIcons; import com.intellij.util.ProcessingContext; -import com.jetbrains.cython.psi.CythonNamedElement; import com.jetbrains.cython.psi.CythonVariable; import com.jetbrains.python.PyNames; import com.jetbrains.python.codeInsight.controlflow.ControlFlowCache; @@ -277,7 +276,7 @@ public class PyReferenceImpl implements PsiReferenceEx, PsiPolyVariantReference definers = processor.getDefiners(); } } - else if (!isCythonLevel(myElement)) { + else { uexpr = null; } } @@ -327,10 +326,6 @@ public class PyReferenceImpl implements PsiReferenceEx, PsiPolyVariantReference return ret; } - private static boolean isCythonLevel(@Nullable PsiElement element) { - return PsiTreeUtil.getParentOfType(element, CythonNamedElement.class) != null; - } - private PsiElement findResolveRoof(String referencedName, PsiElement realContext) { if (PyUtil.isClassPrivateName(referencedName)) { // a class-private name; limited by either class or this file From d09a7bc76fc12ba70549f56a849873e0164f4da0 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Wed, 28 Aug 2013 18:23:41 +0400 Subject: [PATCH 05/11] Don't use resolvesToSameLocal() shortcut for Cython elements We rely on resolve() for Cython unqualified names. --- .../jetbrains/python/psi/impl/references/PyReferenceImpl.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/python/src/com/jetbrains/python/psi/impl/references/PyReferenceImpl.java b/python/src/com/jetbrains/python/psi/impl/references/PyReferenceImpl.java index 91409ed4d160..25561a0d8288 100644 --- a/python/src/com/jetbrains/python/psi/impl/references/PyReferenceImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/references/PyReferenceImpl.java @@ -16,7 +16,6 @@ import com.intellij.psi.util.PsiTreeUtil; import com.intellij.util.IncorrectOperationException; import com.intellij.util.PlatformIcons; import com.intellij.util.ProcessingContext; -import com.jetbrains.cython.psi.CythonVariable; import com.jetbrains.python.PyNames; import com.jetbrains.python.codeInsight.controlflow.ControlFlowCache; import com.jetbrains.python.codeInsight.controlflow.ReadWriteInstruction; @@ -433,8 +432,7 @@ public class PyReferenceImpl implements PsiReferenceEx, PsiPolyVariantReference if (!haveQualifiers(element)) { final ScopeOwner ourScopeOwner = ScopeUtil.getScopeOwner(getElement()); final ScopeOwner theirScopeOwner = ScopeUtil.getScopeOwner(element); - // TODO: Cython-dependent code without CythonLanguageDialect.isInsideCythonFile() check - if (element instanceof PyParameter || element instanceof PyTargetExpression || element instanceof CythonVariable) { + if (element instanceof PyParameter || element instanceof PyTargetExpression) { // Check if the reference is in the same or inner scope of the element scope, not shadowed by an intermediate declaration if (resolvesToSameLocal(element, elementName, ourScopeOwner, theirScopeOwner)) { return true; From c0f5b5afe311f238b3493fb1955978406d09b72e Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Wed, 28 Aug 2013 18:33:30 +0400 Subject: [PATCH 06/11] No need to check for CythonStructType in canQualifyAnImplicitName() It is invoked only for unknown types. --- .../python/psi/impl/references/PyQualifiedReference.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/python/src/com/jetbrains/python/psi/impl/references/PyQualifiedReference.java b/python/src/com/jetbrains/python/psi/impl/references/PyQualifiedReference.java index 68285124ba00..8e860e05fcfd 100644 --- a/python/src/com/jetbrains/python/psi/impl/references/PyQualifiedReference.java +++ b/python/src/com/jetbrains/python/psi/impl/references/PyQualifiedReference.java @@ -18,7 +18,6 @@ import com.intellij.util.ArrayUtil; import com.intellij.util.PlatformIcons; import com.intellij.util.ProcessingContext; import com.intellij.util.indexing.FileBasedIndex; -import com.jetbrains.cython.types.CythonStructType; import com.jetbrains.python.PyNames; import com.jetbrains.python.codeInsight.controlflow.ScopeOwner; import com.jetbrains.python.psi.*; @@ -199,10 +198,6 @@ public class PyQualifiedReference extends PyReferenceImpl { } } } - // TODO: Cython-specific code without CythonLanguageDialect.isInsideCythonFile() check - if (qualType instanceof CythonStructType) { - return false; - } return true; } From 77a675e29ec4579442e77536407e924cc6e7d286 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Wed, 28 Aug 2013 18:49:04 +0400 Subject: [PATCH 07/11] Moved 'cppclass' in PyNamedParameterImpl.isSelf() to CythonNamedParameter --- .../com/jetbrains/python/psi/impl/PyNamedParameterImpl.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/python/src/com/jetbrains/python/psi/impl/PyNamedParameterImpl.java b/python/src/com/jetbrains/python/psi/impl/PyNamedParameterImpl.java index f9f175265e56..05cd3ccc91db 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyNamedParameterImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyNamedParameterImpl.java @@ -12,7 +12,6 @@ import com.intellij.psi.util.PsiTreeUtil; import com.intellij.util.IncorrectOperationException; import com.intellij.util.PlatformIcons; import com.intellij.util.Processor; -import com.jetbrains.cython.psi.CythonClass; import com.jetbrains.python.PyElementTypes; import com.jetbrains.python.PyNames; import com.jetbrains.python.PyTokenTypes; @@ -310,9 +309,6 @@ public class PyNamedParameterImpl extends PyPresentableElementImpl Date: Wed, 28 Aug 2013 19:55:01 +0400 Subject: [PATCH 08/11] Added Cython lexer test --- python/testSrc/com/jetbrains/python/PythonAllTestsSuite.java | 1 + 1 file changed, 1 insertion(+) diff --git a/python/testSrc/com/jetbrains/python/PythonAllTestsSuite.java b/python/testSrc/com/jetbrains/python/PythonAllTestsSuite.java index d9cb855f6ac9..1cab27895f8a 100644 --- a/python/testSrc/com/jetbrains/python/PythonAllTestsSuite.java +++ b/python/testSrc/com/jetbrains/python/PythonAllTestsSuite.java @@ -25,6 +25,7 @@ public class PythonAllTestsSuite { public static final Class[] tests = { PythonLexerTest.class, PyStringLiteralLexerTest.class, + CythonLexerTest.class, PyStringFormatParserTest.class, PyEncodingTest.class, PythonParsingTest.class, From 8fec120fa48992a7712900b4067bc100c506e2e8 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Wed, 28 Aug 2013 20:29:05 +0400 Subject: [PATCH 09/11] Get unbalanced braces recovery tokens via PythonDialectsTokenSetProvider --- .../PythonDialectsTokenSetContributor.java | 1 + .../python/PythonDialectsTokenSetProvider.java | 8 ++++++++ .../python/PythonTokenSetContributor.java | 6 ++++++ .../python/lexer/PythonIndentingProcessor.java | 10 ++-------- .../com/jetbrains/python/PythonParsingTest.java | 2 +- .../python/fixtures/PyLexerTestCase.java | 16 ++++++++++++++-- 6 files changed, 32 insertions(+), 11 deletions(-) diff --git a/python/psi-api/src/com/jetbrains/python/PythonDialectsTokenSetContributor.java b/python/psi-api/src/com/jetbrains/python/PythonDialectsTokenSetContributor.java index 5a7ef32790ba..ce5f560b56d9 100644 --- a/python/psi-api/src/com/jetbrains/python/PythonDialectsTokenSetContributor.java +++ b/python/psi-api/src/com/jetbrains/python/PythonDialectsTokenSetContributor.java @@ -15,4 +15,5 @@ public interface PythonDialectsTokenSetContributor { TokenSet getKeywordTokens(); TokenSet getParameterTokens(); TokenSet getFunctionDeclarationTokens(); + TokenSet getUnbalancedBracesRecoveryTokens(); } diff --git a/python/src/com/jetbrains/python/PythonDialectsTokenSetProvider.java b/python/src/com/jetbrains/python/PythonDialectsTokenSetProvider.java index 73375523ad99..469f6206a549 100644 --- a/python/src/com/jetbrains/python/PythonDialectsTokenSetProvider.java +++ b/python/src/com/jetbrains/python/PythonDialectsTokenSetProvider.java @@ -15,6 +15,7 @@ public class PythonDialectsTokenSetProvider { private final TokenSet myKeywordTokens; private final TokenSet myParameterTokens; private final TokenSet myFunctionDeclarationTokens; + private final TokenSet myUnbalancedBracesRecoveryTokens; private PythonDialectsTokenSetProvider() { TokenSet stmts = TokenSet.EMPTY; @@ -23,6 +24,7 @@ public class PythonDialectsTokenSetProvider { TokenSet keywords = TokenSet.EMPTY; TokenSet parameters = TokenSet.EMPTY; TokenSet functionDeclarations = TokenSet.EMPTY; + TokenSet recoveryTokens = TokenSet.EMPTY; for(PythonDialectsTokenSetContributor contributor: Extensions.getExtensions(PythonDialectsTokenSetContributor.EP_NAME)) { stmts = TokenSet.orSet(stmts, contributor.getStatementTokens()); exprs = TokenSet.orSet(exprs, contributor.getExpressionTokens()); @@ -30,6 +32,7 @@ public class PythonDialectsTokenSetProvider { keywords = TokenSet.orSet(keywords, contributor.getKeywordTokens()); parameters = TokenSet.orSet(parameters, contributor.getParameterTokens()); functionDeclarations = TokenSet.orSet(functionDeclarations, contributor.getFunctionDeclarationTokens()); + recoveryTokens = TokenSet.orSet(recoveryTokens, contributor.getUnbalancedBracesRecoveryTokens()); } myStatementTokens = stmts; myExpressionTokens = exprs; @@ -37,6 +40,7 @@ public class PythonDialectsTokenSetProvider { myKeywordTokens = keywords; myParameterTokens = parameters; myFunctionDeclarationTokens = functionDeclarations; + myUnbalancedBracesRecoveryTokens = recoveryTokens; } public TokenSet getStatementTokens() { @@ -62,4 +66,8 @@ public class PythonDialectsTokenSetProvider { public TokenSet getFunctionDeclarationTokens() { return myFunctionDeclarationTokens; } + + public TokenSet getUnbalancedBracesRecoveryTokens() { + return myUnbalancedBracesRecoveryTokens; + } } diff --git a/python/src/com/jetbrains/python/PythonTokenSetContributor.java b/python/src/com/jetbrains/python/PythonTokenSetContributor.java index 4e1165a77868..e47eec8bfa07 100644 --- a/python/src/com/jetbrains/python/PythonTokenSetContributor.java +++ b/python/src/com/jetbrains/python/PythonTokenSetContributor.java @@ -64,4 +64,10 @@ public class PythonTokenSetContributor implements PythonDialectsTokenSetContribu public TokenSet getFunctionDeclarationTokens() { return TokenSet.create(FUNCTION_DECLARATION); } + + @Override + public TokenSet getUnbalancedBracesRecoveryTokens() { + return TokenSet.create(DEF_KEYWORD, CLASS_KEYWORD, RETURN_KEYWORD, WITH_KEYWORD, WHILE_KEYWORD, BREAK_KEYWORD, CONTINUE_KEYWORD, + RAISE_KEYWORD, TRY_KEYWORD, EXCEPT_KEYWORD, FINALLY_KEYWORD); + } } diff --git a/python/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java b/python/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java index 4ad93f840490..2cac358435f3 100644 --- a/python/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java +++ b/python/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java @@ -5,8 +5,8 @@ import com.intellij.lexer.FlexLexer; import com.intellij.lexer.MergingLexerAdapter; import com.intellij.psi.tree.IElementType; import com.intellij.psi.tree.TokenSet; -import com.jetbrains.cython.parser.CythonTokenTypes; import com.jetbrains.python.PyTokenTypes; +import com.jetbrains.python.PythonDialectsTokenSetProvider; import gnu.trove.TIntStack; import org.jetbrains.annotations.Nullable; @@ -21,13 +21,7 @@ public class PythonIndentingProcessor extends MergingLexerAdapter { private int myCurrentNewLineIndent = 0; private static final boolean DUMP_TOKENS = false; - private static final TokenSet RECOVERY_TOKENS = - TokenSet.create(PyTokenTypes.DEF_KEYWORD, PyTokenTypes.CLASS_KEYWORD, PyTokenTypes.RETURN_KEYWORD, PyTokenTypes.WITH_KEYWORD, - PyTokenTypes.WHILE_KEYWORD, PyTokenTypes.BREAK_KEYWORD, PyTokenTypes.CONTINUE_KEYWORD, PyTokenTypes.RAISE_KEYWORD, - PyTokenTypes.TRY_KEYWORD, PyTokenTypes.EXCEPT_KEYWORD, PyTokenTypes.FINALLY_KEYWORD, - // TODO: Cython dependency. Collect these tokens via the PythonDialectsTokenSetProvider (requires setting up the - // "Pythonid.dialectsTokenSetContributor" extension point in all the tests. - CythonTokenTypes.CDEF_KEYWORD, CythonTokenTypes.CPDEF_KEYWORD, CythonTokenTypes.CTYPEDEF_KEYWORD); + private final TokenSet RECOVERY_TOKENS = PythonDialectsTokenSetProvider.INSTANCE.getUnbalancedBracesRecoveryTokens(); public PythonIndentingProcessor(FlexLexer lexer, TokenSet tokens) { super(new FlexAdapter(lexer), tokens); diff --git a/python/testSrc/com/jetbrains/python/PythonParsingTest.java b/python/testSrc/com/jetbrains/python/PythonParsingTest.java index 53687856cf7a..75da8f3d51c5 100644 --- a/python/testSrc/com/jetbrains/python/PythonParsingTest.java +++ b/python/testSrc/com/jetbrains/python/PythonParsingTest.java @@ -16,12 +16,12 @@ public class PythonParsingTest extends ParsingTestCase { public PythonParsingTest() { super("psi", "py", new PythonParserDefinition()); - PyTestCase.initPlatformPrefix(); } @Override protected void setUp() throws Exception { super.setUp(); + PyTestCase.initPlatformPrefix(); registerExtensionPoint(PythonDialectsTokenSetContributor.EP_NAME, PythonDialectsTokenSetContributor.class); registerExtension(PythonDialectsTokenSetContributor.EP_NAME, new PythonTokenSetContributor()); registerExtension(PythonDialectsTokenSetContributor.EP_NAME, new CythonTokenSetContributor()); diff --git a/python/testSrc/com/jetbrains/python/fixtures/PyLexerTestCase.java b/python/testSrc/com/jetbrains/python/fixtures/PyLexerTestCase.java index 9e3a0f16e01d..40ddfaf84fc3 100644 --- a/python/testSrc/com/jetbrains/python/fixtures/PyLexerTestCase.java +++ b/python/testSrc/com/jetbrains/python/fixtures/PyLexerTestCase.java @@ -1,12 +1,24 @@ package com.jetbrains.python.fixtures; import com.intellij.lexer.Lexer; -import junit.framework.TestCase; +import com.intellij.testFramework.PlatformLiteFixture; +import com.jetbrains.cython.CythonTokenSetContributor; +import com.jetbrains.python.PythonDialectsTokenSetContributor; +import com.jetbrains.python.PythonTokenSetContributor; /** * @author yole */ -public abstract class PyLexerTestCase extends TestCase { +public abstract class PyLexerTestCase extends PlatformLiteFixture { + @Override + protected void setUp() throws Exception { + super.setUp(); + PyTestCase.initPlatformPrefix(); + registerExtensionPoint(PythonDialectsTokenSetContributor.EP_NAME, PythonDialectsTokenSetContributor.class); + registerExtension(PythonDialectsTokenSetContributor.EP_NAME, new PythonTokenSetContributor()); + registerExtension(PythonDialectsTokenSetContributor.EP_NAME, new CythonTokenSetContributor()); + } + public static void doLexerTest(String text, Lexer lexer, String... expectedTokens) { lexer.start(text); int idx = 0; From 904d0f9c28c47830403f17dc23ab777824b484ec Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Wed, 28 Aug 2013 20:42:48 +0400 Subject: [PATCH 10/11] Removed explicit references to REFERENCE_EXPRESSION --- .../python/console/parsing/PyConsoleParsingContext.java | 2 +- python/src/com/jetbrains/python/psi/impl/PyDecoratorImpl.java | 2 +- .../jetbrains/python/psi/impl/PyFromImportStatementImpl.java | 3 +-- .../src/com/jetbrains/python/psi/impl/PyImportElementImpl.java | 2 +- .../python/psi/impl/stubs/PyTargetExpressionElementType.java | 2 +- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/python/src/com/jetbrains/python/console/parsing/PyConsoleParsingContext.java b/python/src/com/jetbrains/python/console/parsing/PyConsoleParsingContext.java index bd2ffb22d7e4..ba6eaae70527 100644 --- a/python/src/com/jetbrains/python/console/parsing/PyConsoleParsingContext.java +++ b/python/src/com/jetbrains/python/console/parsing/PyConsoleParsingContext.java @@ -138,7 +138,7 @@ public class PyConsoleParsingContext extends ParsingContext { if (myBuilder.getTokenType() == PyTokenTypes.IDENTIFIER) { myBuilder.advanceLexer(); - command.done(PyElementTypes.REFERENCE_EXPRESSION); + command.done(getReferenceType()); } else { expr.drop(); diff --git a/python/src/com/jetbrains/python/psi/impl/PyDecoratorImpl.java b/python/src/com/jetbrains/python/psi/impl/PyDecoratorImpl.java index 05bee8401122..cb48d7fba867 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyDecoratorImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyDecoratorImpl.java @@ -47,7 +47,7 @@ public class PyDecoratorImpl extends StubBasedPsiElementBase im } public boolean isBuiltin() { - ASTNode node = getNode().findChildByType(PyElementTypes.REFERENCE_EXPRESSION); + ASTNode node = getNode().findChildByType(PyElementTypes.REFERENCE_EXPRESSION_SET); if (node != null) { PyReferenceExpression ref = (PyReferenceExpression)node.getPsi(); PsiElement target = ref.getReference().resolve(); diff --git a/python/src/com/jetbrains/python/psi/impl/PyFromImportStatementImpl.java b/python/src/com/jetbrains/python/psi/impl/PyFromImportStatementImpl.java index f887dece66af..6601f24d47b1 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyFromImportStatementImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyFromImportStatementImpl.java @@ -4,7 +4,6 @@ import com.intellij.lang.ASTNode; import com.intellij.psi.*; import com.intellij.psi.scope.PsiScopeProcessor; import com.intellij.psi.stubs.IStubElementType; -import com.intellij.psi.tree.TokenSet; import com.intellij.util.ArrayFactory; import com.jetbrains.python.PyElementTypes; import com.jetbrains.python.PyNames; @@ -50,7 +49,7 @@ public class PyFromImportStatementImpl extends PyBaseElementImpl @Nullable public PyReferenceExpression getImportReferenceExpression() { - final ASTNode node = getNode().findChildByType(PyElementTypes.REFERENCE_EXPRESSION); + final ASTNode node = getNode().findChildByType(PyElementTypes.REFERENCE_EXPRESSION_SET); return node == null ? null : (PyReferenceExpression) node.getPsi(); } diff --git a/python/src/com/jetbrains/python/psi/impl/stubs/PyTargetExpressionElementType.java b/python/src/com/jetbrains/python/psi/impl/stubs/PyTargetExpressionElementType.java index 205211190414..6c61d3ffd357 100644 --- a/python/src/com/jetbrains/python/psi/impl/stubs/PyTargetExpressionElementType.java +++ b/python/src/com/jetbrains/python/psi/impl/stubs/PyTargetExpressionElementType.java @@ -118,7 +118,7 @@ public class PyTargetExpressionElementType extends PyStubElementType Date: Wed, 28 Aug 2013 21:52:01 +0400 Subject: [PATCH 11/11] Extracted CythonReferenceExpression with its own getReference() --- .../python/PythonDialectsTokenSetContributor.java | 1 + .../src/com/jetbrains/python/PyElementTypes.java | 2 -- .../python/PythonDialectsTokenSetProvider.java | 8 ++++++++ .../python/PythonTokenSetContributor.java | 5 +++++ .../python/psi/impl/PyDecoratorImpl.java | 3 ++- .../psi/impl/PyFromImportStatementImpl.java | 3 ++- .../python/psi/impl/PyImportElementImpl.java | 3 ++- .../psi/impl/PyReferenceExpressionImpl.java | 15 --------------- .../impl/stubs/PyTargetExpressionElementType.java | 3 ++- 9 files changed, 22 insertions(+), 21 deletions(-) diff --git a/python/psi-api/src/com/jetbrains/python/PythonDialectsTokenSetContributor.java b/python/psi-api/src/com/jetbrains/python/PythonDialectsTokenSetContributor.java index ce5f560b56d9..798ec70e421f 100644 --- a/python/psi-api/src/com/jetbrains/python/PythonDialectsTokenSetContributor.java +++ b/python/psi-api/src/com/jetbrains/python/PythonDialectsTokenSetContributor.java @@ -16,4 +16,5 @@ public interface PythonDialectsTokenSetContributor { TokenSet getParameterTokens(); TokenSet getFunctionDeclarationTokens(); TokenSet getUnbalancedBracesRecoveryTokens(); + TokenSet getReferenceExpressionTokens(); } diff --git a/python/src/com/jetbrains/python/PyElementTypes.java b/python/src/com/jetbrains/python/PyElementTypes.java index a76d47d1184e..7b58d74ac685 100644 --- a/python/src/com/jetbrains/python/PyElementTypes.java +++ b/python/src/com/jetbrains/python/PyElementTypes.java @@ -72,8 +72,6 @@ public interface PyElementTypes { PyElementType EMPTY_EXPRESSION = new PyElementType("EMPTY_EXPRESSION", PyEmptyExpressionImpl.class); PyElementType REFERENCE_EXPRESSION = new PyElementType("REFERENCE_EXPRESSION", PyReferenceExpressionImpl.class); - TokenSet REFERENCE_EXPRESSION_SET = TokenSet.create(REFERENCE_EXPRESSION); - PyStubElementType TARGET_EXPRESSION = new PyTargetExpressionElementType(); PyElementType INTEGER_LITERAL_EXPRESSION = new PyElementType("INTEGER_LITERAL_EXPRESSION", PyNumericLiteralExpressionImpl.class); PyElementType FLOAT_LITERAL_EXPRESSION = new PyElementType("FLOAT_LITERAL_EXPRESSION", PyNumericLiteralExpressionImpl.class); diff --git a/python/src/com/jetbrains/python/PythonDialectsTokenSetProvider.java b/python/src/com/jetbrains/python/PythonDialectsTokenSetProvider.java index 469f6206a549..1e0da69a7807 100644 --- a/python/src/com/jetbrains/python/PythonDialectsTokenSetProvider.java +++ b/python/src/com/jetbrains/python/PythonDialectsTokenSetProvider.java @@ -16,6 +16,7 @@ public class PythonDialectsTokenSetProvider { private final TokenSet myParameterTokens; private final TokenSet myFunctionDeclarationTokens; private final TokenSet myUnbalancedBracesRecoveryTokens; + private final TokenSet myReferenceExpressionTokens; private PythonDialectsTokenSetProvider() { TokenSet stmts = TokenSet.EMPTY; @@ -25,6 +26,7 @@ public class PythonDialectsTokenSetProvider { TokenSet parameters = TokenSet.EMPTY; TokenSet functionDeclarations = TokenSet.EMPTY; TokenSet recoveryTokens = TokenSet.EMPTY; + TokenSet referenceExpressions = TokenSet.EMPTY; for(PythonDialectsTokenSetContributor contributor: Extensions.getExtensions(PythonDialectsTokenSetContributor.EP_NAME)) { stmts = TokenSet.orSet(stmts, contributor.getStatementTokens()); exprs = TokenSet.orSet(exprs, contributor.getExpressionTokens()); @@ -33,6 +35,7 @@ public class PythonDialectsTokenSetProvider { parameters = TokenSet.orSet(parameters, contributor.getParameterTokens()); functionDeclarations = TokenSet.orSet(functionDeclarations, contributor.getFunctionDeclarationTokens()); recoveryTokens = TokenSet.orSet(recoveryTokens, contributor.getUnbalancedBracesRecoveryTokens()); + referenceExpressions = TokenSet.orSet(referenceExpressions, contributor.getReferenceExpressionTokens()); } myStatementTokens = stmts; myExpressionTokens = exprs; @@ -41,6 +44,7 @@ public class PythonDialectsTokenSetProvider { myParameterTokens = parameters; myFunctionDeclarationTokens = functionDeclarations; myUnbalancedBracesRecoveryTokens = recoveryTokens; + myReferenceExpressionTokens = referenceExpressions; } public TokenSet getStatementTokens() { @@ -70,4 +74,8 @@ public class PythonDialectsTokenSetProvider { public TokenSet getUnbalancedBracesRecoveryTokens() { return myUnbalancedBracesRecoveryTokens; } + + public TokenSet getReferenceExpressionTokens() { + return myReferenceExpressionTokens; + } } diff --git a/python/src/com/jetbrains/python/PythonTokenSetContributor.java b/python/src/com/jetbrains/python/PythonTokenSetContributor.java index e47eec8bfa07..878043dcb078 100644 --- a/python/src/com/jetbrains/python/PythonTokenSetContributor.java +++ b/python/src/com/jetbrains/python/PythonTokenSetContributor.java @@ -70,4 +70,9 @@ public class PythonTokenSetContributor implements PythonDialectsTokenSetContribu return TokenSet.create(DEF_KEYWORD, CLASS_KEYWORD, RETURN_KEYWORD, WITH_KEYWORD, WHILE_KEYWORD, BREAK_KEYWORD, CONTINUE_KEYWORD, RAISE_KEYWORD, TRY_KEYWORD, EXCEPT_KEYWORD, FINALLY_KEYWORD); } + + @Override + public TokenSet getReferenceExpressionTokens() { + return TokenSet.create(REFERENCE_EXPRESSION); + } } diff --git a/python/src/com/jetbrains/python/psi/impl/PyDecoratorImpl.java b/python/src/com/jetbrains/python/psi/impl/PyDecoratorImpl.java index cb48d7fba867..efa10ccd8128 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyDecoratorImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyDecoratorImpl.java @@ -7,6 +7,7 @@ import com.intellij.psi.util.PsiTreeUtil; import com.intellij.util.IncorrectOperationException; import com.jetbrains.python.PyElementTypes; import com.jetbrains.python.PyTokenTypes; +import com.jetbrains.python.PythonDialectsTokenSetProvider; import com.jetbrains.python.psi.*; import com.jetbrains.python.psi.resolve.PyResolveContext; import com.jetbrains.python.psi.resolve.PyResolveUtil; @@ -47,7 +48,7 @@ public class PyDecoratorImpl extends StubBasedPsiElementBase im } public boolean isBuiltin() { - ASTNode node = getNode().findChildByType(PyElementTypes.REFERENCE_EXPRESSION_SET); + ASTNode node = getNode().findChildByType(PythonDialectsTokenSetProvider.INSTANCE.getReferenceExpressionTokens()); if (node != null) { PyReferenceExpression ref = (PyReferenceExpression)node.getPsi(); PsiElement target = ref.getReference().resolve(); diff --git a/python/src/com/jetbrains/python/psi/impl/PyFromImportStatementImpl.java b/python/src/com/jetbrains/python/psi/impl/PyFromImportStatementImpl.java index 6601f24d47b1..b43511d0d442 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyFromImportStatementImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyFromImportStatementImpl.java @@ -8,6 +8,7 @@ import com.intellij.util.ArrayFactory; import com.jetbrains.python.PyElementTypes; import com.jetbrains.python.PyNames; import com.jetbrains.python.PyTokenTypes; +import com.jetbrains.python.PythonDialectsTokenSetProvider; import com.jetbrains.python.psi.*; import com.jetbrains.python.psi.resolve.ResolveImportUtil; import com.jetbrains.python.psi.stubs.PyFromImportStatementStub; @@ -49,7 +50,7 @@ public class PyFromImportStatementImpl extends PyBaseElementImpl @Nullable public PyReferenceExpression getImportReferenceExpression() { - final ASTNode node = getNode().findChildByType(PyElementTypes.REFERENCE_EXPRESSION_SET); + final ASTNode node = getNode().findChildByType(PythonDialectsTokenSetProvider.INSTANCE.getReferenceExpressionTokens()); return node == null ? null : (PyReferenceExpression) node.getPsi(); } diff --git a/python/src/com/jetbrains/python/psi/impl/PyReferenceExpressionImpl.java b/python/src/com/jetbrains/python/psi/impl/PyReferenceExpressionImpl.java index 3e702bf69ef5..63e2e1902feb 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyReferenceExpressionImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyReferenceExpressionImpl.java @@ -6,11 +6,6 @@ import com.intellij.openapi.extensions.Extensions; import com.intellij.openapi.util.Ref; import com.intellij.psi.*; import com.intellij.psi.util.PsiTreeUtil; -import com.jetbrains.cython.CythonLanguageDialect; -import com.jetbrains.cython.psi.CythonCImportElement; -import com.jetbrains.cython.psi.CythonFromCImportStatement; -import com.jetbrains.cython.psi.CythonImportReference; -import com.jetbrains.cython.psi.CythonReference; import com.jetbrains.python.PyNames; import com.jetbrains.python.PyTokenTypes; import com.jetbrains.python.PythonDialectsTokenSetProvider; @@ -56,12 +51,6 @@ public class PyReferenceExpressionImpl extends PyElementImpl implements PyRefere final PyExpression qualifier = getQualifier(); // Handle import reference - final boolean inCythonFile = CythonLanguageDialect.isInsideCythonFile(this); - if (inCythonFile) { - if (PsiTreeUtil.getParentOfType(this, CythonCImportElement.class, CythonFromCImportStatement.class) != null) { - return new CythonImportReference(this, context); - } - } final PsiElement importParent = PsiTreeUtil.getParentOfType(this, PyImportElement.class, PyFromImportStatement.class); if (importParent != null) { return PyImportReference.forElement(this, importParent, context); @@ -80,10 +69,6 @@ public class PyReferenceExpressionImpl extends PyElementImpl implements PyRefere return new PyQualifiedReference(this, context); } - if (inCythonFile) { - return new CythonReference(this, context); - } - return new PyReferenceImpl(this, context); } diff --git a/python/src/com/jetbrains/python/psi/impl/stubs/PyTargetExpressionElementType.java b/python/src/com/jetbrains/python/psi/impl/stubs/PyTargetExpressionElementType.java index 6c61d3ffd357..4f004f594be9 100644 --- a/python/src/com/jetbrains/python/psi/impl/stubs/PyTargetExpressionElementType.java +++ b/python/src/com/jetbrains/python/psi/impl/stubs/PyTargetExpressionElementType.java @@ -12,6 +12,7 @@ import com.intellij.psi.util.PsiTreeUtil; import com.intellij.util.io.StringRef; import com.jetbrains.python.PyElementTypes; import com.jetbrains.python.PyNames; +import com.jetbrains.python.PythonDialectsTokenSetProvider; import com.jetbrains.python.documentation.DocStringUtil; import com.jetbrains.python.psi.*; import com.jetbrains.python.psi.impl.PyQualifiedName; @@ -118,7 +119,7 @@ public class PyTargetExpressionElementType extends PyStubElementType