Merge branch 'remove-cython'

This commit is contained in:
Andrey Vlasovskikh
2013-08-28 22:01:59 +04:00
24 changed files with 86 additions and 78 deletions
@@ -15,4 +15,6 @@ public interface PythonDialectsTokenSetContributor {
TokenSet getKeywordTokens();
TokenSet getParameterTokens();
TokenSet getFunctionDeclarationTokens();
TokenSet getUnbalancedBracesRecoveryTokens();
TokenSet getReferenceExpressionTokens();
}
@@ -5,10 +5,10 @@ import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
/**
* PSI element that (re)defnies names in following namespace, e.g. as assignment statement does.
* PSI element that (re)defines names in following namespace, e.g. as assignment statement does.
*
* NOTE: When declaring additional elements as name definers, please also adjust the token set via
* {@link PythonDialectsTokenSetContributor#getNameDefinerTokens()}.
* {@link com.jetbrains.python.PythonDialectsTokenSetContributor#getNameDefinerTokens()}.
*
* @author dcheryasov
*/
@@ -9,7 +9,7 @@ import org.jetbrains.annotations.Nullable;
/**
* @author yole
*/
public interface PyImportElement extends PyElement, NameDefiner, StubBasedPsiElement<PyImportElementStub> {
public interface PyImportElement extends PyElement, PyImportedNameDefiner, StubBasedPsiElement<PyImportElementStub> {
@Nullable
PyReferenceExpression getImportReferenceExpression();
@@ -0,0 +1,9 @@
package com.jetbrains.python.psi;
/**
* Name definer that defines names imported somehow from other modules.
*
* @author vlan
*/
public interface PyImportedNameDefiner extends NameDefiner {
}
@@ -5,5 +5,5 @@ package com.jetbrains.python.psi;
* User: dcheryasov
* Date: Jul 28, 2008
*/
public interface PyStarImportElement extends PyElement, NameDefiner {
public interface PyStarImportElement extends PyElement, PyImportedNameDefiner {
}
@@ -72,8 +72,6 @@ public interface PyElementTypes {
PyElementType EMPTY_EXPRESSION = new PyElementType("EMPTY_EXPRESSION", PyEmptyExpressionImpl.class);
PyElementType REFERENCE_EXPRESSION = new PyElementType("REFERENCE_EXPRESSION", PyReferenceExpressionImpl.class);
TokenSet REFERENCE_EXPRESSION_SET = TokenSet.create(REFERENCE_EXPRESSION);
PyStubElementType<PyTargetExpressionStub, PyTargetExpression> TARGET_EXPRESSION = new PyTargetExpressionElementType();
PyElementType INTEGER_LITERAL_EXPRESSION = new PyElementType("INTEGER_LITERAL_EXPRESSION", PyNumericLiteralExpressionImpl.class);
PyElementType FLOAT_LITERAL_EXPRESSION = new PyElementType("FLOAT_LITERAL_EXPRESSION", PyNumericLiteralExpressionImpl.class);
@@ -15,6 +15,8 @@ public class PythonDialectsTokenSetProvider {
private final TokenSet myKeywordTokens;
private final TokenSet myParameterTokens;
private final TokenSet myFunctionDeclarationTokens;
private final TokenSet myUnbalancedBracesRecoveryTokens;
private final TokenSet myReferenceExpressionTokens;
private PythonDialectsTokenSetProvider() {
TokenSet stmts = TokenSet.EMPTY;
@@ -23,6 +25,8 @@ public class PythonDialectsTokenSetProvider {
TokenSet keywords = TokenSet.EMPTY;
TokenSet parameters = TokenSet.EMPTY;
TokenSet functionDeclarations = TokenSet.EMPTY;
TokenSet recoveryTokens = TokenSet.EMPTY;
TokenSet referenceExpressions = TokenSet.EMPTY;
for(PythonDialectsTokenSetContributor contributor: Extensions.getExtensions(PythonDialectsTokenSetContributor.EP_NAME)) {
stmts = TokenSet.orSet(stmts, contributor.getStatementTokens());
exprs = TokenSet.orSet(exprs, contributor.getExpressionTokens());
@@ -30,6 +34,8 @@ public class PythonDialectsTokenSetProvider {
keywords = TokenSet.orSet(keywords, contributor.getKeywordTokens());
parameters = TokenSet.orSet(parameters, contributor.getParameterTokens());
functionDeclarations = TokenSet.orSet(functionDeclarations, contributor.getFunctionDeclarationTokens());
recoveryTokens = TokenSet.orSet(recoveryTokens, contributor.getUnbalancedBracesRecoveryTokens());
referenceExpressions = TokenSet.orSet(referenceExpressions, contributor.getReferenceExpressionTokens());
}
myStatementTokens = stmts;
myExpressionTokens = exprs;
@@ -37,6 +43,8 @@ public class PythonDialectsTokenSetProvider {
myKeywordTokens = keywords;
myParameterTokens = parameters;
myFunctionDeclarationTokens = functionDeclarations;
myUnbalancedBracesRecoveryTokens = recoveryTokens;
myReferenceExpressionTokens = referenceExpressions;
}
public TokenSet getStatementTokens() {
@@ -62,4 +70,12 @@ public class PythonDialectsTokenSetProvider {
public TokenSet getFunctionDeclarationTokens() {
return myFunctionDeclarationTokens;
}
public TokenSet getUnbalancedBracesRecoveryTokens() {
return myUnbalancedBracesRecoveryTokens;
}
public TokenSet getReferenceExpressionTokens() {
return myReferenceExpressionTokens;
}
}
@@ -64,4 +64,15 @@ public class PythonTokenSetContributor implements PythonDialectsTokenSetContribu
public TokenSet getFunctionDeclarationTokens() {
return TokenSet.create(FUNCTION_DECLARATION);
}
@Override
public TokenSet getUnbalancedBracesRecoveryTokens() {
return TokenSet.create(DEF_KEYWORD, CLASS_KEYWORD, RETURN_KEYWORD, WITH_KEYWORD, WHILE_KEYWORD, BREAK_KEYWORD, CONTINUE_KEYWORD,
RAISE_KEYWORD, TRY_KEYWORD, EXCEPT_KEYWORD, FINALLY_KEYWORD);
}
@Override
public TokenSet getReferenceExpressionTokens() {
return TokenSet.create(REFERENCE_EXPRESSION);
}
}
@@ -3,7 +3,7 @@ package com.jetbrains.python.codeInsight.dataflow.scope;
import com.intellij.codeInsight.dataflow.DFALimitExceededException;
import com.intellij.psi.PsiElement;
import com.intellij.psi.PsiNamedElement;
import com.jetbrains.python.psi.NameDefiner;
import com.jetbrains.python.psi.PyImportedNameDefiner;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -27,7 +27,7 @@ public interface Scope {
boolean containsDeclaration(String name);
@NotNull
List<NameDefiner> getNameDefiners();
List<PyImportedNameDefiner> getImportedNameDefiners();
@Nullable
PsiNamedElement getNamedElement(String name);
@@ -6,7 +6,6 @@ import com.intellij.codeInsight.dataflow.map.DFAMap;
import com.intellij.codeInsight.dataflow.map.DFAMapEngine;
import com.intellij.psi.PsiElement;
import com.intellij.psi.PsiNamedElement;
import com.jetbrains.cython.psi.CythonIncludeStatement;
import com.jetbrains.python.codeInsight.controlflow.ControlFlowCache;
import com.jetbrains.python.codeInsight.controlflow.ScopeOwner;
import com.jetbrains.python.codeInsight.dataflow.PyReachingDefsDfaInstance;
@@ -31,7 +30,7 @@ public class ScopeImpl implements Scope {
private volatile List<Scope> myNestedScopes;
private final ScopeOwner myFlowOwner;
private volatile Map<String, PsiNamedElement> myNamedElements;
private volatile List<NameDefiner> myNameDefiners; // declarations which declare unknown set of names, such as 'from ... import *'
private volatile List<PyImportedNameDefiner> myImportedNameDefiners; // Declarations which declare unknown set of imported names
private volatile Set<String> myAugAssignments;
public ScopeImpl(final ScopeOwner flowOwner) {
@@ -97,7 +96,7 @@ public class ScopeImpl implements Scope {
}
public boolean containsDeclaration(final String name) {
if (myNamedElements == null || myNameDefiners == null) {
if (myNamedElements == null || myImportedNameDefiners == null) {
collectDeclarations();
}
if (isNonlocal(name)) {
@@ -109,7 +108,7 @@ public class ScopeImpl implements Scope {
if (isAugAssignment(name)) {
return true;
}
for (NameDefiner definer : getNameDefiners()) {
for (NameDefiner definer : getImportedNameDefiners()) {
if (definer.getElementNamed(name) != null) {
return true;
}
@@ -119,11 +118,11 @@ public class ScopeImpl implements Scope {
@NotNull
@Override
public List<NameDefiner> getNameDefiners() {
if (myNameDefiners == null) {
public List<PyImportedNameDefiner> getImportedNameDefiners() {
if (myImportedNameDefiners == null) {
collectDeclarations();
}
return myNameDefiners;
return myImportedNameDefiners;
}
@Nullable
@@ -158,7 +157,7 @@ public class ScopeImpl implements Scope {
private void collectDeclarations() {
final Map<String, PsiNamedElement> namedElements = new HashMap<String, PsiNamedElement>();
final List<NameDefiner> nameDefiners = new ArrayList<NameDefiner>();
final List<PyImportedNameDefiner> importedNameDefiners = new ArrayList<PyImportedNameDefiner>();
final List<Scope> nestedScopes = new ArrayList<Scope>();
final Set<String> globals = new HashSet<String>();
final Set<String> nonlocals = new HashSet<String>();
@@ -214,9 +213,8 @@ public class ScopeImpl implements Scope {
if (node instanceof PsiNamedElement && !(node instanceof PyKeywordArgument)) {
namedElements.put(node.getName(), (PsiNamedElement)node);
}
// TODO: Cython-specific code
if (node instanceof PyStarImportElement || node instanceof PyImportElement || node instanceof CythonIncludeStatement) {
nameDefiners.add((NameDefiner)node);
if (node instanceof PyImportedNameDefiner) {
importedNameDefiners.add((PyImportedNameDefiner)node);
}
if (node instanceof ScopeOwner) {
final Scope scope = ControlFlowCache.getScope((ScopeOwner)node);
@@ -228,7 +226,7 @@ public class ScopeImpl implements Scope {
}
});
Collections.sort(nameDefiners, new Comparator<NameDefiner>() {
Collections.sort(importedNameDefiners, new Comparator<NameDefiner>() {
@Override
public int compare(NameDefiner d1, NameDefiner d2) {
return getPriority(d2) - getPriority(d1);
@@ -243,7 +241,7 @@ public class ScopeImpl implements Scope {
});
myNamedElements = namedElements;
myNameDefiners = nameDefiners;
myImportedNameDefiners = importedNameDefiners;
myNestedScopes = nestedScopes;
myGlobals = globals;
myNonlocals = nonlocals;
@@ -138,7 +138,7 @@ public class PyConsoleParsingContext extends ParsingContext {
if (myBuilder.getTokenType() == PyTokenTypes.IDENTIFIER) {
myBuilder.advanceLexer();
command.done(PyElementTypes.REFERENCE_EXPRESSION);
command.done(getReferenceType());
}
else {
expr.drop();
@@ -5,8 +5,8 @@ import com.intellij.lexer.FlexLexer;
import com.intellij.lexer.MergingLexerAdapter;
import com.intellij.psi.tree.IElementType;
import com.intellij.psi.tree.TokenSet;
import com.jetbrains.cython.parser.CythonTokenTypes;
import com.jetbrains.python.PyTokenTypes;
import com.jetbrains.python.PythonDialectsTokenSetProvider;
import gnu.trove.TIntStack;
import org.jetbrains.annotations.Nullable;
@@ -21,13 +21,7 @@ public class PythonIndentingProcessor extends MergingLexerAdapter {
private int myCurrentNewLineIndent = 0;
private static final boolean DUMP_TOKENS = false;
private static final TokenSet RECOVERY_TOKENS =
TokenSet.create(PyTokenTypes.DEF_KEYWORD, PyTokenTypes.CLASS_KEYWORD, PyTokenTypes.RETURN_KEYWORD, PyTokenTypes.WITH_KEYWORD,
PyTokenTypes.WHILE_KEYWORD, PyTokenTypes.BREAK_KEYWORD, PyTokenTypes.CONTINUE_KEYWORD, PyTokenTypes.RAISE_KEYWORD,
PyTokenTypes.TRY_KEYWORD, PyTokenTypes.EXCEPT_KEYWORD, PyTokenTypes.FINALLY_KEYWORD,
// TODO: Cython dependency. Collect these tokens via the PythonDialectsTokenSetProvider (requires setting up the
// "Pythonid.dialectsTokenSetContributor" extension point in all the tests.
CythonTokenTypes.CDEF_KEYWORD, CythonTokenTypes.CPDEF_KEYWORD, CythonTokenTypes.CTYPEDEF_KEYWORD);
private final TokenSet RECOVERY_TOKENS = PythonDialectsTokenSetProvider.INSTANCE.getUnbalancedBracesRecoveryTokens();
public PythonIndentingProcessor(FlexLexer lexer, TokenSet tokens) {
super(new FlexAdapter(lexer), tokens);
@@ -7,6 +7,7 @@ import com.intellij.psi.util.PsiTreeUtil;
import com.intellij.util.IncorrectOperationException;
import com.jetbrains.python.PyElementTypes;
import com.jetbrains.python.PyTokenTypes;
import com.jetbrains.python.PythonDialectsTokenSetProvider;
import com.jetbrains.python.psi.*;
import com.jetbrains.python.psi.resolve.PyResolveContext;
import com.jetbrains.python.psi.resolve.PyResolveUtil;
@@ -47,7 +48,7 @@ public class PyDecoratorImpl extends StubBasedPsiElementBase<PyDecoratorStub> im
}
public boolean isBuiltin() {
ASTNode node = getNode().findChildByType(PyElementTypes.REFERENCE_EXPRESSION);
ASTNode node = getNode().findChildByType(PythonDialectsTokenSetProvider.INSTANCE.getReferenceExpressionTokens());
if (node != null) {
PyReferenceExpression ref = (PyReferenceExpression)node.getPsi();
PsiElement target = ref.getReference().resolve();
@@ -4,11 +4,11 @@ import com.intellij.lang.ASTNode;
import com.intellij.psi.*;
import com.intellij.psi.scope.PsiScopeProcessor;
import com.intellij.psi.stubs.IStubElementType;
import com.intellij.psi.tree.TokenSet;
import com.intellij.util.ArrayFactory;
import com.jetbrains.python.PyElementTypes;
import com.jetbrains.python.PyNames;
import com.jetbrains.python.PyTokenTypes;
import com.jetbrains.python.PythonDialectsTokenSetProvider;
import com.jetbrains.python.psi.*;
import com.jetbrains.python.psi.resolve.ResolveImportUtil;
import com.jetbrains.python.psi.stubs.PyFromImportStatementStub;
@@ -50,7 +50,7 @@ public class PyFromImportStatementImpl extends PyBaseElementImpl<PyFromImportSta
@Nullable
public PyReferenceExpression getImportSource() {
return childToPsi(TokenSet.create(PyElementTypes.REFERENCE_EXPRESSION), 0);
return childToPsi(PythonDialectsTokenSetProvider.INSTANCE.getReferenceExpressionTokens(), 0);
}
public PyQualifiedName getImportSourceQName() {
@@ -10,6 +10,7 @@ import com.intellij.psi.stubs.IStubElementType;
import com.intellij.psi.util.PsiTreeUtil;
import com.intellij.util.containers.EmptyIterable;
import com.jetbrains.python.PyElementTypes;
import com.jetbrains.python.PythonDialectsTokenSetProvider;
import com.jetbrains.python.psi.*;
import com.jetbrains.python.psi.resolve.PyResolveUtil;
import com.jetbrains.python.psi.resolve.ResolveImportUtil;
@@ -41,7 +42,7 @@ public class PyImportElementImpl extends PyBaseElementImpl<PyImportElementStub>
@Nullable
public PyReferenceExpression getImportReferenceExpression() {
final ASTNode node = getNode().findChildByType(PyElementTypes.REFERENCE_EXPRESSION);
final ASTNode node = getNode().findChildByType(PythonDialectsTokenSetProvider.INSTANCE.getReferenceExpressionTokens());
return node == null ? null : (PyReferenceExpression) node.getPsi();
}
@@ -12,7 +12,6 @@ import com.intellij.psi.util.PsiTreeUtil;
import com.intellij.util.IncorrectOperationException;
import com.intellij.util.PlatformIcons;
import com.intellij.util.Processor;
import com.jetbrains.cython.psi.CythonClass;
import com.jetbrains.python.PyElementTypes;
import com.jetbrains.python.PyNames;
import com.jetbrains.python.PyTokenTypes;
@@ -310,9 +309,6 @@ public class PyNamedParameterImpl extends PyPresentableElementImpl<PyNamedParame
}
final PyClass cls = function.getContainingClass();
if (cls != null && function.getParameterList().getParameters()[0] == this) {
if (cls instanceof CythonClass && ((CythonClass)cls).isCppClass()) {
return false;
}
if (PyNames.NEW.equals(function.getName())) {
return true;
}
@@ -6,11 +6,6 @@ import com.intellij.openapi.extensions.Extensions;
import com.intellij.openapi.util.Ref;
import com.intellij.psi.*;
import com.intellij.psi.util.PsiTreeUtil;
import com.jetbrains.cython.CythonLanguageDialect;
import com.jetbrains.cython.psi.CythonCImportElement;
import com.jetbrains.cython.psi.CythonFromCImportStatement;
import com.jetbrains.cython.psi.CythonImportReference;
import com.jetbrains.cython.psi.CythonReference;
import com.jetbrains.python.PyNames;
import com.jetbrains.python.PyTokenTypes;
import com.jetbrains.python.PythonDialectsTokenSetProvider;
@@ -56,12 +51,6 @@ public class PyReferenceExpressionImpl extends PyElementImpl implements PyRefere
final PyExpression qualifier = getQualifier();
// Handle import reference
final boolean inCythonFile = CythonLanguageDialect.isInsideCythonFile(this);
if (inCythonFile) {
if (PsiTreeUtil.getParentOfType(this, CythonCImportElement.class, CythonFromCImportStatement.class) != null) {
return new CythonImportReference(this, context);
}
}
final PsiElement importParent = PsiTreeUtil.getParentOfType(this, PyImportElement.class, PyFromImportStatement.class);
if (importParent != null) {
return PyImportReference.forElement(this, importParent, context);
@@ -80,10 +69,6 @@ public class PyReferenceExpressionImpl extends PyElementImpl implements PyRefere
return new PyQualifiedReference(this, context);
}
if (inCythonFile) {
return new CythonReference(this, context);
}
return new PyReferenceImpl(this, context);
}
@@ -18,7 +18,6 @@ import com.intellij.util.ArrayUtil;
import com.intellij.util.PlatformIcons;
import com.intellij.util.ProcessingContext;
import com.intellij.util.indexing.FileBasedIndex;
import com.jetbrains.cython.types.CythonStructType;
import com.jetbrains.python.PyNames;
import com.jetbrains.python.codeInsight.controlflow.ScopeOwner;
import com.jetbrains.python.psi.*;
@@ -199,10 +198,6 @@ public class PyQualifiedReference extends PyReferenceImpl {
}
}
}
// TODO: Cython-specific code without CythonLanguageDialect.isInsideCythonFile() check
if (qualType instanceof CythonStructType) {
return false;
}
return true;
}
@@ -16,10 +16,6 @@ import com.intellij.psi.util.PsiTreeUtil;
import com.intellij.util.IncorrectOperationException;
import com.intellij.util.PlatformIcons;
import com.intellij.util.ProcessingContext;
import com.jetbrains.cython.CythonLanguageDialect;
import com.jetbrains.cython.psi.CythonIncludeStatement;
import com.jetbrains.cython.psi.CythonNamedElement;
import com.jetbrains.cython.psi.CythonVariable;
import com.jetbrains.python.PyNames;
import com.jetbrains.python.codeInsight.controlflow.ControlFlowCache;
import com.jetbrains.python.codeInsight.controlflow.ReadWriteInstruction;
@@ -279,7 +275,7 @@ public class PyReferenceImpl implements PsiReferenceEx, PsiPolyVariantReference
definers = processor.getDefiners();
}
}
else if (!isCythonLevel(myElement)) {
else {
uexpr = null;
}
}
@@ -329,10 +325,6 @@ public class PyReferenceImpl implements PsiReferenceEx, PsiPolyVariantReference
return ret;
}
private static boolean isCythonLevel(@Nullable PsiElement element) {
return PsiTreeUtil.getParentOfType(element, CythonNamedElement.class) != null;
}
private PsiElement findResolveRoof(String referencedName, PsiElement realContext) {
if (PyUtil.isClassPrivateName(referencedName)) {
// a class-private name; limited by either class or this file
@@ -373,10 +365,7 @@ public class PyReferenceImpl implements PsiReferenceEx, PsiPolyVariantReference
public static int getRate(PsiElement elt) {
int rate;
if (CythonLanguageDialect.isInsideCythonFile(elt) && elt instanceof CythonIncludeStatement) {
rate = RatedResolveResult.RATE_LOW;
}
else if (elt instanceof PyImportElement || elt instanceof PyStarImportElement || elt instanceof PyReferenceExpression) {
if (elt instanceof PyImportedNameDefiner || elt instanceof PyReferenceExpression) {
rate = RatedResolveResult.RATE_LOW;
}
else if (elt instanceof PyFile) {
@@ -443,8 +432,7 @@ public class PyReferenceImpl implements PsiReferenceEx, PsiPolyVariantReference
if (!haveQualifiers(element)) {
final ScopeOwner ourScopeOwner = ScopeUtil.getScopeOwner(getElement());
final ScopeOwner theirScopeOwner = ScopeUtil.getScopeOwner(element);
// TODO: Cython-dependent code without CythonLanguageDialect.isInsideCythonFile() check
if (element instanceof PyParameter || element instanceof PyTargetExpression || element instanceof CythonVariable) {
if (element instanceof PyParameter || element instanceof PyTargetExpression) {
// Check if the reference is in the same or inner scope of the element scope, not shadowed by an intermediate declaration
if (resolvesToSameLocal(element, elementName, ourScopeOwner, theirScopeOwner)) {
return true;
@@ -12,6 +12,7 @@ import com.intellij.psi.util.PsiTreeUtil;
import com.intellij.util.io.StringRef;
import com.jetbrains.python.PyElementTypes;
import com.jetbrains.python.PyNames;
import com.jetbrains.python.PythonDialectsTokenSetProvider;
import com.jetbrains.python.documentation.DocStringUtil;
import com.jetbrains.python.psi.*;
import com.jetbrains.python.psi.impl.PyQualifiedName;
@@ -118,7 +119,7 @@ public class PyTargetExpressionElementType extends PyStubElementType<PyTargetExp
return false;
}
final ASTNode functionNode = TreeUtil.findParent(node, PyElementTypes.FUNCTION_DECLARATION);
final ASTNode qualifierNode = node.findChildByType(PyElementTypes.REFERENCE_EXPRESSION);
final ASTNode qualifierNode = node.findChildByType(PythonDialectsTokenSetProvider.INSTANCE.getReferenceExpressionTokens());
if (functionNode != null && qualifierNode != null) {
final PsiElement function = functionNode.getPsi();
if (function instanceof PyFunction && PyNames.NEW.equals(((PyFunction)function).getName())) {
@@ -125,7 +125,7 @@ public class PyResolveUtil {
}
}
}
for (NameDefiner definer : scope.getNameDefiners()) {
for (NameDefiner definer : scope.getImportedNameDefiners()) {
if (!processor.execute(definer, ResolveState.initial())) {
found = true;
break;
@@ -25,6 +25,7 @@ public class PythonAllTestsSuite {
public static final Class[] tests = {
PythonLexerTest.class,
PyStringLiteralLexerTest.class,
CythonLexerTest.class,
PyStringFormatParserTest.class,
PyEncodingTest.class,
PythonParsingTest.class,
@@ -16,12 +16,12 @@ public class PythonParsingTest extends ParsingTestCase {
public PythonParsingTest() {
super("psi", "py", new PythonParserDefinition());
PyTestCase.initPlatformPrefix();
}
@Override
protected void setUp() throws Exception {
super.setUp();
PyTestCase.initPlatformPrefix();
registerExtensionPoint(PythonDialectsTokenSetContributor.EP_NAME, PythonDialectsTokenSetContributor.class);
registerExtension(PythonDialectsTokenSetContributor.EP_NAME, new PythonTokenSetContributor());
registerExtension(PythonDialectsTokenSetContributor.EP_NAME, new CythonTokenSetContributor());
@@ -1,12 +1,24 @@
package com.jetbrains.python.fixtures;
import com.intellij.lexer.Lexer;
import junit.framework.TestCase;
import com.intellij.testFramework.PlatformLiteFixture;
import com.jetbrains.cython.CythonTokenSetContributor;
import com.jetbrains.python.PythonDialectsTokenSetContributor;
import com.jetbrains.python.PythonTokenSetContributor;
/**
* @author yole
*/
public abstract class PyLexerTestCase extends TestCase {
public abstract class PyLexerTestCase extends PlatformLiteFixture {
@Override
protected void setUp() throws Exception {
super.setUp();
PyTestCase.initPlatformPrefix();
registerExtensionPoint(PythonDialectsTokenSetContributor.EP_NAME, PythonDialectsTokenSetContributor.class);
registerExtension(PythonDialectsTokenSetContributor.EP_NAME, new PythonTokenSetContributor());
registerExtension(PythonDialectsTokenSetContributor.EP_NAME, new CythonTokenSetContributor());
}
public static void doLexerTest(String text, Lexer lexer, String... expectedTokens) {
lexer.start(text);
int idx = 0;