MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

Get unbalanced braces recovery tokens via PythonDialectsTokenSetProvider

Browse Source
This commit is contained in:
Andrey Vlasovskikh
2013-08-28 20:29:05 +04:00
parent 771dae3da8
commit 8fec120fa4
6 changed files with 32 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/psi-api/src/com/jetbrains/python/PythonDialectsTokenSetContributor.java
+1
View File
@@ -15,4 +15,5 @@ public interface PythonDialectsTokenSetContributor {
TokenSet getKeywordTokens();
TokenSet getParameterTokens();
TokenSet getFunctionDeclarationTokens();
TokenSet getUnbalancedBracesRecoveryTokens();
}
python/src/com/jetbrains/python/PythonDialectsTokenSetProvider.java
+8
View File
@@ -15,6 +15,7 @@ public class PythonDialectsTokenSetProvider {
private final TokenSet myKeywordTokens;
private final TokenSet myParameterTokens;
private final TokenSet myFunctionDeclarationTokens;
private final TokenSet myUnbalancedBracesRecoveryTokens;
private PythonDialectsTokenSetProvider() {
TokenSet stmts = TokenSet.EMPTY;
@@ -23,6 +24,7 @@ public class PythonDialectsTokenSetProvider {
TokenSet keywords = TokenSet.EMPTY;
TokenSet parameters = TokenSet.EMPTY;
TokenSet functionDeclarations = TokenSet.EMPTY;
TokenSet recoveryTokens = TokenSet.EMPTY;
for(PythonDialectsTokenSetContributor contributor: Extensions.getExtensions(PythonDialectsTokenSetContributor.EP_NAME)) {
stmts = TokenSet.orSet(stmts, contributor.getStatementTokens());
exprs = TokenSet.orSet(exprs, contributor.getExpressionTokens());
@@ -30,6 +32,7 @@ public class PythonDialectsTokenSetProvider {
keywords = TokenSet.orSet(keywords, contributor.getKeywordTokens());
parameters = TokenSet.orSet(parameters, contributor.getParameterTokens());
functionDeclarations = TokenSet.orSet(functionDeclarations, contributor.getFunctionDeclarationTokens());
recoveryTokens = TokenSet.orSet(recoveryTokens, contributor.getUnbalancedBracesRecoveryTokens());
}
myStatementTokens = stmts;
myExpressionTokens = exprs;
@@ -37,6 +40,7 @@ public class PythonDialectsTokenSetProvider {
myKeywordTokens = keywords;
myParameterTokens = parameters;
myFunctionDeclarationTokens = functionDeclarations;
myUnbalancedBracesRecoveryTokens = recoveryTokens;
}
public TokenSet getStatementTokens() {
@@ -62,4 +66,8 @@ public class PythonDialectsTokenSetProvider {
public TokenSet getFunctionDeclarationTokens() {
return myFunctionDeclarationTokens;
}
public TokenSet getUnbalancedBracesRecoveryTokens() {
return myUnbalancedBracesRecoveryTokens;
}
}
python/src/com/jetbrains/python/PythonTokenSetContributor.java
+6
View File
@@ -64,4 +64,10 @@ public class PythonTokenSetContributor implements PythonDialectsTokenSetContribu
public TokenSet getFunctionDeclarationTokens() {
return TokenSet.create(FUNCTION_DECLARATION);
}
@Override
public TokenSet getUnbalancedBracesRecoveryTokens() {
return TokenSet.create(DEF_KEYWORD, CLASS_KEYWORD, RETURN_KEYWORD, WITH_KEYWORD, WHILE_KEYWORD, BREAK_KEYWORD, CONTINUE_KEYWORD,
RAISE_KEYWORD, TRY_KEYWORD, EXCEPT_KEYWORD, FINALLY_KEYWORD);
}
}
python/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java
+2 -8
View File
@@ -5,8 +5,8 @@ import com.intellij.lexer.FlexLexer;
import com.intellij.lexer.MergingLexerAdapter;
import com.intellij.psi.tree.IElementType;
import com.intellij.psi.tree.TokenSet;
import com.jetbrains.cython.parser.CythonTokenTypes;
import com.jetbrains.python.PyTokenTypes;
import com.jetbrains.python.PythonDialectsTokenSetProvider;
import gnu.trove.TIntStack;
import org.jetbrains.annotations.Nullable;
@@ -21,13 +21,7 @@ public class PythonIndentingProcessor extends MergingLexerAdapter {
private int myCurrentNewLineIndent = 0;
private static final boolean DUMP_TOKENS = false;
private static final TokenSet RECOVERY_TOKENS =
TokenSet.create(PyTokenTypes.DEF_KEYWORD, PyTokenTypes.CLASS_KEYWORD, PyTokenTypes.RETURN_KEYWORD, PyTokenTypes.WITH_KEYWORD,
PyTokenTypes.WHILE_KEYWORD, PyTokenTypes.BREAK_KEYWORD, PyTokenTypes.CONTINUE_KEYWORD, PyTokenTypes.RAISE_KEYWORD,
PyTokenTypes.TRY_KEYWORD, PyTokenTypes.EXCEPT_KEYWORD, PyTokenTypes.FINALLY_KEYWORD,
// TODO: Cython dependency. Collect these tokens via the PythonDialectsTokenSetProvider (requires setting up the
// "Pythonid.dialectsTokenSetContributor" extension point in all the tests.
CythonTokenTypes.CDEF_KEYWORD, CythonTokenTypes.CPDEF_KEYWORD, CythonTokenTypes.CTYPEDEF_KEYWORD);
private final TokenSet RECOVERY_TOKENS = PythonDialectsTokenSetProvider.INSTANCE.getUnbalancedBracesRecoveryTokens();
public PythonIndentingProcessor(FlexLexer lexer, TokenSet tokens) {
super(new FlexAdapter(lexer), tokens);
python/testSrc/com/jetbrains/python/PythonParsingTest.java
+1 -1
View File
@@ -16,12 +16,12 @@ public class PythonParsingTest extends ParsingTestCase {
public PythonParsingTest() {
super("psi", "py", new PythonParserDefinition());
PyTestCase.initPlatformPrefix();
}
@Override
protected void setUp() throws Exception {
super.setUp();
PyTestCase.initPlatformPrefix();
registerExtensionPoint(PythonDialectsTokenSetContributor.EP_NAME, PythonDialectsTokenSetContributor.class);
registerExtension(PythonDialectsTokenSetContributor.EP_NAME, new PythonTokenSetContributor());
registerExtension(PythonDialectsTokenSetContributor.EP_NAME, new CythonTokenSetContributor());
python/testSrc/com/jetbrains/python/fixtures/PyLexerTestCase.java
+14 -2
View File
@@ -1,12 +1,24 @@
package com.jetbrains.python.fixtures;
import com.intellij.lexer.Lexer;
import junit.framework.TestCase;
import com.intellij.testFramework.PlatformLiteFixture;
import com.jetbrains.cython.CythonTokenSetContributor;
import com.jetbrains.python.PythonDialectsTokenSetContributor;
import com.jetbrains.python.PythonTokenSetContributor;
/**
* @author yole
*/
public abstract class PyLexerTestCase extends TestCase {
public abstract class PyLexerTestCase extends PlatformLiteFixture {
@Override
protected void setUp() throws Exception {
super.setUp();
PyTestCase.initPlatformPrefix();
registerExtensionPoint(PythonDialectsTokenSetContributor.EP_NAME, PythonDialectsTokenSetContributor.class);
registerExtension(PythonDialectsTokenSetContributor.EP_NAME, new PythonTokenSetContributor());
registerExtension(PythonDialectsTokenSetContributor.EP_NAME, new CythonTokenSetContributor());
}
public static void doLexerTest(String text, Lexer lexer, String... expectedTokens) {
lexer.start(text);
int idx = 0;