From 8fec120fa48992a7712900b4067bc100c506e2e8 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Wed, 28 Aug 2013 20:29:05 +0400 Subject: [PATCH] Get unbalanced braces recovery tokens via PythonDialectsTokenSetProvider --- .../PythonDialectsTokenSetContributor.java | 1 + .../python/PythonDialectsTokenSetProvider.java | 8 ++++++++ .../python/PythonTokenSetContributor.java | 6 ++++++ .../python/lexer/PythonIndentingProcessor.java | 10 ++-------- .../com/jetbrains/python/PythonParsingTest.java | 2 +- .../python/fixtures/PyLexerTestCase.java | 16 ++++++++++++++-- 6 files changed, 32 insertions(+), 11 deletions(-) diff --git a/python/psi-api/src/com/jetbrains/python/PythonDialectsTokenSetContributor.java b/python/psi-api/src/com/jetbrains/python/PythonDialectsTokenSetContributor.java index 5a7ef32790ba..ce5f560b56d9 100644 --- a/python/psi-api/src/com/jetbrains/python/PythonDialectsTokenSetContributor.java +++ b/python/psi-api/src/com/jetbrains/python/PythonDialectsTokenSetContributor.java @@ -15,4 +15,5 @@ public interface PythonDialectsTokenSetContributor { TokenSet getKeywordTokens(); TokenSet getParameterTokens(); TokenSet getFunctionDeclarationTokens(); + TokenSet getUnbalancedBracesRecoveryTokens(); } diff --git a/python/src/com/jetbrains/python/PythonDialectsTokenSetProvider.java b/python/src/com/jetbrains/python/PythonDialectsTokenSetProvider.java index 73375523ad99..469f6206a549 100644 --- a/python/src/com/jetbrains/python/PythonDialectsTokenSetProvider.java +++ b/python/src/com/jetbrains/python/PythonDialectsTokenSetProvider.java @@ -15,6 +15,7 @@ public class PythonDialectsTokenSetProvider { private final TokenSet myKeywordTokens; private final TokenSet myParameterTokens; private final TokenSet myFunctionDeclarationTokens; + private final TokenSet myUnbalancedBracesRecoveryTokens; private PythonDialectsTokenSetProvider() { TokenSet stmts = TokenSet.EMPTY; @@ -23,6 +24,7 @@ public class PythonDialectsTokenSetProvider { TokenSet keywords = TokenSet.EMPTY; TokenSet parameters = TokenSet.EMPTY; TokenSet functionDeclarations = TokenSet.EMPTY; + TokenSet recoveryTokens = TokenSet.EMPTY; for(PythonDialectsTokenSetContributor contributor: Extensions.getExtensions(PythonDialectsTokenSetContributor.EP_NAME)) { stmts = TokenSet.orSet(stmts, contributor.getStatementTokens()); exprs = TokenSet.orSet(exprs, contributor.getExpressionTokens()); @@ -30,6 +32,7 @@ public class PythonDialectsTokenSetProvider { keywords = TokenSet.orSet(keywords, contributor.getKeywordTokens()); parameters = TokenSet.orSet(parameters, contributor.getParameterTokens()); functionDeclarations = TokenSet.orSet(functionDeclarations, contributor.getFunctionDeclarationTokens()); + recoveryTokens = TokenSet.orSet(recoveryTokens, contributor.getUnbalancedBracesRecoveryTokens()); } myStatementTokens = stmts; myExpressionTokens = exprs; @@ -37,6 +40,7 @@ public class PythonDialectsTokenSetProvider { myKeywordTokens = keywords; myParameterTokens = parameters; myFunctionDeclarationTokens = functionDeclarations; + myUnbalancedBracesRecoveryTokens = recoveryTokens; } public TokenSet getStatementTokens() { @@ -62,4 +66,8 @@ public class PythonDialectsTokenSetProvider { public TokenSet getFunctionDeclarationTokens() { return myFunctionDeclarationTokens; } + + public TokenSet getUnbalancedBracesRecoveryTokens() { + return myUnbalancedBracesRecoveryTokens; + } } diff --git a/python/src/com/jetbrains/python/PythonTokenSetContributor.java b/python/src/com/jetbrains/python/PythonTokenSetContributor.java index 4e1165a77868..e47eec8bfa07 100644 --- a/python/src/com/jetbrains/python/PythonTokenSetContributor.java +++ b/python/src/com/jetbrains/python/PythonTokenSetContributor.java @@ -64,4 +64,10 @@ public class PythonTokenSetContributor implements PythonDialectsTokenSetContribu public TokenSet getFunctionDeclarationTokens() { return TokenSet.create(FUNCTION_DECLARATION); } + + @Override + public TokenSet getUnbalancedBracesRecoveryTokens() { + return TokenSet.create(DEF_KEYWORD, CLASS_KEYWORD, RETURN_KEYWORD, WITH_KEYWORD, WHILE_KEYWORD, BREAK_KEYWORD, CONTINUE_KEYWORD, + RAISE_KEYWORD, TRY_KEYWORD, EXCEPT_KEYWORD, FINALLY_KEYWORD); + } } diff --git a/python/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java b/python/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java index 4ad93f840490..2cac358435f3 100644 --- a/python/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java +++ b/python/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java @@ -5,8 +5,8 @@ import com.intellij.lexer.FlexLexer; import com.intellij.lexer.MergingLexerAdapter; import com.intellij.psi.tree.IElementType; import com.intellij.psi.tree.TokenSet; -import com.jetbrains.cython.parser.CythonTokenTypes; import com.jetbrains.python.PyTokenTypes; +import com.jetbrains.python.PythonDialectsTokenSetProvider; import gnu.trove.TIntStack; import org.jetbrains.annotations.Nullable; @@ -21,13 +21,7 @@ public class PythonIndentingProcessor extends MergingLexerAdapter { private int myCurrentNewLineIndent = 0; private static final boolean DUMP_TOKENS = false; - private static final TokenSet RECOVERY_TOKENS = - TokenSet.create(PyTokenTypes.DEF_KEYWORD, PyTokenTypes.CLASS_KEYWORD, PyTokenTypes.RETURN_KEYWORD, PyTokenTypes.WITH_KEYWORD, - PyTokenTypes.WHILE_KEYWORD, PyTokenTypes.BREAK_KEYWORD, PyTokenTypes.CONTINUE_KEYWORD, PyTokenTypes.RAISE_KEYWORD, - PyTokenTypes.TRY_KEYWORD, PyTokenTypes.EXCEPT_KEYWORD, PyTokenTypes.FINALLY_KEYWORD, - // TODO: Cython dependency. Collect these tokens via the PythonDialectsTokenSetProvider (requires setting up the - // "Pythonid.dialectsTokenSetContributor" extension point in all the tests. - CythonTokenTypes.CDEF_KEYWORD, CythonTokenTypes.CPDEF_KEYWORD, CythonTokenTypes.CTYPEDEF_KEYWORD); + private final TokenSet RECOVERY_TOKENS = PythonDialectsTokenSetProvider.INSTANCE.getUnbalancedBracesRecoveryTokens(); public PythonIndentingProcessor(FlexLexer lexer, TokenSet tokens) { super(new FlexAdapter(lexer), tokens); diff --git a/python/testSrc/com/jetbrains/python/PythonParsingTest.java b/python/testSrc/com/jetbrains/python/PythonParsingTest.java index 53687856cf7a..75da8f3d51c5 100644 --- a/python/testSrc/com/jetbrains/python/PythonParsingTest.java +++ b/python/testSrc/com/jetbrains/python/PythonParsingTest.java @@ -16,12 +16,12 @@ public class PythonParsingTest extends ParsingTestCase { public PythonParsingTest() { super("psi", "py", new PythonParserDefinition()); - PyTestCase.initPlatformPrefix(); } @Override protected void setUp() throws Exception { super.setUp(); + PyTestCase.initPlatformPrefix(); registerExtensionPoint(PythonDialectsTokenSetContributor.EP_NAME, PythonDialectsTokenSetContributor.class); registerExtension(PythonDialectsTokenSetContributor.EP_NAME, new PythonTokenSetContributor()); registerExtension(PythonDialectsTokenSetContributor.EP_NAME, new CythonTokenSetContributor()); diff --git a/python/testSrc/com/jetbrains/python/fixtures/PyLexerTestCase.java b/python/testSrc/com/jetbrains/python/fixtures/PyLexerTestCase.java index 9e3a0f16e01d..40ddfaf84fc3 100644 --- a/python/testSrc/com/jetbrains/python/fixtures/PyLexerTestCase.java +++ b/python/testSrc/com/jetbrains/python/fixtures/PyLexerTestCase.java @@ -1,12 +1,24 @@ package com.jetbrains.python.fixtures; import com.intellij.lexer.Lexer; -import junit.framework.TestCase; +import com.intellij.testFramework.PlatformLiteFixture; +import com.jetbrains.cython.CythonTokenSetContributor; +import com.jetbrains.python.PythonDialectsTokenSetContributor; +import com.jetbrains.python.PythonTokenSetContributor; /** * @author yole */ -public abstract class PyLexerTestCase extends TestCase { +public abstract class PyLexerTestCase extends PlatformLiteFixture { + @Override + protected void setUp() throws Exception { + super.setUp(); + PyTestCase.initPlatformPrefix(); + registerExtensionPoint(PythonDialectsTokenSetContributor.EP_NAME, PythonDialectsTokenSetContributor.class); + registerExtension(PythonDialectsTokenSetContributor.EP_NAME, new PythonTokenSetContributor()); + registerExtension(PythonDialectsTokenSetContributor.EP_NAME, new CythonTokenSetContributor()); + } + public static void doLexerTest(String text, Lexer lexer, String... expectedTokens) { lexer.start(text); int idx = 0;