MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

Handle '​from _​_​future​__ import print​_​function' in Python 2​.​6 (PY-314)

Browse Source
This commit is contained in:
Dmitry Jemerov
2010-01-15 16:16:32 +03:00
parent 51a4f1dff4
commit b26aa629a7
14 changed files with 424 additions and 1342 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/src/com/jetbrains/python/lexer/BuildLexer.bat
-4
View File
@@ -1,4 +0,0 @@
set JAVA_HOME="C:\Program Files\Java\jdk1.5.0_12"
call C:\Src\jflex-1.4.1\bin\jflex.bat --table --skel idea-skeleton Python.flex
python FixLexer.py
python/src/com/jetbrains/python/lexer/FixLexer.py
-13
View File
@@ -1,13 +0,0 @@
import os, os.path
if os.path.exists("_PythonLexerBad.java"): os.unlink("_PythonLexerBad.java")
os.rename("_PythonLexer.java", "_PythonLexerBad.java")
f = open("_PythonLexerBad.java", "r")
out = open("_PythonLexer.java", "w")
for line in f.readlines():
i = line.find("zzBufferL[")
if i >= 0:
line = line [0:i-1] + "zzBufferL.charAt(" + line [i+10:-3] + ");\n"
out.write(line)
f.close()
out.close()
os.unlink("_PythonLexerBad.java")
python/src/com/jetbrains/python/lexer/Python.flex
+1 -1
View File
@@ -3,6 +3,7 @@ package com.jetbrains.python.lexer;
import com.intellij.lexer.FlexLexer;
import com.intellij.psi.tree.IElementType;
import com.jetbrains.python.PyTokenTypes;
%%
@@ -100,7 +101,6 @@ TRIPLE_APOS_LITERAL = {THREE_APOS} {STRING_3CHAR_APOS}* {THREE_APOS}
"not" { return PyTokenTypes.NOT_KEYWORD; }
"or" { return PyTokenTypes.OR_KEYWORD; }
"pass" { return PyTokenTypes.PASS_KEYWORD; }
"print" { return PyTokenTypes.PRINT_KEYWORD; }
"raise" { return PyTokenTypes.RAISE_KEYWORD; }
"return" { return PyTokenTypes.RETURN_KEYWORD; }
"try" { return PyTokenTypes.TRY_KEYWORD; }
python/src/com/jetbrains/python/lexer/PythonHighlightingLexer.java
+4
View File
@@ -21,6 +21,10 @@ public class PythonHighlightingLexer extends PythonLexer {
if (tokenText.equals("with")) return PyTokenTypes.WITH_KEYWORD;
if (tokenText.equals("as")) return PyTokenTypes.AS_KEYWORD;
}
if (myLanguageLevel.hasPrintStatement()) {
final String tokenText = getTokenText();
if (tokenText.equals("print")) return PyTokenTypes.PRINT_KEYWORD;
}
return super.getTokenType();
}
}
python/src/com/jetbrains/python/lexer/_PythonLexer.java
+341 -1057
View File
File diff suppressed because it is too large Load Diff
python/src/com/jetbrains/python/lexer/idea-skeleton
-256
View File
@@ -1,256 +0,0 @@
/** initial size of the lookahead buffer */
--- private static final int ZZ_BUFFERSIZE = ...;
/** lexical states */
--- lexical states, charmap
/* error codes */
private static final int ZZ_UNKNOWN_ERROR = 0;
private static final int ZZ_NO_MATCH = 1;
private static final int ZZ_PUSHBACK_2BIG = 2;
private static final char[] EMPTY_BUFFER = new char[0];
private static final int YYEOF = -1;
private static java.io.Reader zzReader = null; // Fake
/* error messages for the codes above */
private static final String ZZ_ERROR_MSG[] = {
"Unkown internal scanner error",
"Error: could not match input",
"Error: pushback value was too large"
};
--- isFinal list
/** the current state of the DFA */
private int zzState;
/** the current lexical state */
private int zzLexicalState = YYINITIAL;
/** this buffer contains the current text to be matched and is
the source of the yytext() string */
private CharSequence zzBuffer = "";
/** this buffer may contains the current text array to be matched when it is cheap to acquire it */
private char[] zzBufferArray;
/** the textposition at the last accepting state */
private int zzMarkedPos;
/** the textposition at the last state to be included in yytext */
private int zzPushbackPos;
/** the current text position in the buffer */
private int zzCurrentPos;
/** startRead marks the beginning of the yytext() string in the buffer */
private int zzStartRead;
/** endRead marks the last character in the buffer, that has been read
from input */
private int zzEndRead;
/**
* zzAtBOL == true <=> the scanner is currently at the beginning of a line
*/
private boolean zzAtBOL = true;
/** zzAtEOF == true <=> the scanner is at the EOF */
private boolean zzAtEOF;
--- user class code
--- constructor declaration
public final int getTokenStart(){
return zzStartRead;
}
public final int getTokenEnd(){
return getTokenStart() + yylength();
}
public void reset(CharSequence buffer, int start, int end,int initialState){
zzBuffer = buffer;
zzBufferArray = com.intellij.util.text.CharArrayUtil.fromSequenceWithoutCopying(buffer);
zzCurrentPos = zzMarkedPos = zzStartRead = start;
zzPushbackPos = 0;
zzAtEOF = false;
zzAtBOL = true;
zzEndRead = end;
yybegin(initialState);
}
// For Demetra compatibility
public void reset(CharSequence buffer, int initialState){
reset(buffer, 0, buffer.length(), initialState);
}
/**
* Refills the input buffer.
*
* @return <code>false</code>, iff there was new input.
*
* @exception java.io.IOException if any I/O-Error occurs
*/
private boolean zzRefill() throws java.io.IOException {
return true;
}
/**
* Returns the current lexical state.
*/
public final int yystate() {
return zzLexicalState;
}
/**
* Enters a new lexical state
*
* @param newState the new lexical state
*/
public final void yybegin(int newState) {
zzLexicalState = newState;
}
/**
* Returns the text matched by the current regular expression.
*/
public final CharSequence yytext() {
return zzBuffer.subSequence(zzStartRead, zzMarkedPos);
}
/**
* Returns the character at position <tt>pos</tt> from the
* matched text.
*
* It is equivalent to yytext().charAt(pos), but faster
*
* @param pos the position of the character to fetch.
* A value from 0 to yylength()-1.
*
* @return the character at position pos
*/
public final char yycharat(int pos) {
return zzBufferArray != null ? zzBufferArray[zzStartRead+pos]:zzBuffer.charAt(zzStartRead+pos);
}
/**
* Returns the length of the matched text region.
*/
public final int yylength() {
return zzMarkedPos-zzStartRead;
}
/**
* Reports an error that occured while scanning.
*
* In a wellformed scanner (no or only correct usage of
* yypushback(int) and a match-all fallback rule) this method
* will only be called with things that "Can't Possibly Happen".
* If this method is called, something is seriously wrong
* (e.g. a JFlex bug producing a faulty scanner etc.).
*
* Usual syntax/scanner level error handling should be done
* in error fallback rules.
*
* @param errorCode the code of the errormessage to display
*/
--- zzScanError declaration
String message;
try {
message = ZZ_ERROR_MSG[errorCode];
}
catch (ArrayIndexOutOfBoundsException e) {
message = ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR];
}
--- throws clause
}
/**
* Pushes the specified amount of characters back into the input stream.
*
* They will be read again by then next call of the scanning method
*
* @param number the number of characters to be read again.
* This number must not be greater than yylength()!
*/
--- yypushback decl (contains zzScanError exception)
if ( number > yylength() )
zzScanError(ZZ_PUSHBACK_2BIG);
zzMarkedPos -= number;
}
--- zzDoEOF
/**
* Resumes scanning until the next regular expression is matched,
* the end of input is encountered or an I/O-Error occurs.
*
* @return the next token
* @exception java.io.IOException if any I/O-Error occurs
*/
--- yylex declaration
int zzInput;
int zzAction;
// cached fields:
int zzCurrentPosL;
int zzMarkedPosL;
int zzEndReadL = zzEndRead;
CharSequence zzBufferL = zzBuffer;
char[] zzBufferArrayL = zzBufferArray;
char [] zzCMapL = ZZ_CMAP;
--- local declarations
while (true) {
zzMarkedPosL = zzMarkedPos;
--- start admin (line, char, col count)
zzAction = -1;
zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;
--- start admin (lexstate etc)
zzForAction: {
while (true) {
--- next input, line, col, char count, next transition, isFinal action
zzAction = zzState;
zzMarkedPosL = zzCurrentPosL;
--- line count update
}
}
}
// store back cached position
zzMarkedPos = zzMarkedPosL;
--- char count update
--- actions
default:
if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
zzAtEOF = true;
--- eofvalue
}
else {
--- no match
}
}
}
}
--- main
}
python/src/com/jetbrains/python/parsing/ParsingContext.java
+2 -2
View File
@@ -25,10 +25,10 @@ public class ParsingContext {
private final PsiBuilder myBuilder;
private final LanguageLevel myLanguageLevel;
public ParsingContext(final PsiBuilder builder, LanguageLevel languageLevel) {
public ParsingContext(final PsiBuilder builder, LanguageLevel languageLevel, StatementParsing.FUTURE futureFlag) {
myBuilder = builder;
myLanguageLevel = languageLevel;
stmtParser = new StatementParsing(this);
stmtParser = new StatementParsing(this, futureFlag);
expressionParser = new ExpressionParsing(this);
functionParser = new FunctionParsing(this);
}
python/src/com/jetbrains/python/parsing/PyParser.java
+6 -1
View File
@@ -15,6 +15,7 @@ public class PyParser implements PsiParser {
private static final Logger LOGGER = Logger.getInstance(PyParser.class.getName());
private final LanguageLevel myLanguageLevel;
private StatementParsing.FUTURE myFutureFlag;
public PyParser() {
myLanguageLevel = LanguageLevel.getDefault();
@@ -29,7 +30,7 @@ public class PyParser implements PsiParser {
builder.setDebugMode(false);
long start = System.currentTimeMillis();
final PsiBuilder.Marker rootMarker = builder.mark();
ParsingContext context = new ParsingContext(builder, myLanguageLevel);
ParsingContext context = new ParsingContext(builder, myLanguageLevel, myFutureFlag);
StatementParsing stmt_parser = context.getStatementParser();
builder.setTokenTypeRemapper(stmt_parser); // must be done before touching the caching lexer with eof() call.
while (!builder.eof()) {
@@ -42,4 +43,8 @@ public class PyParser implements PsiParser {
LOGGER.debug("Parsed " + String.format("%.1f", kb) + "K file in " + diff + "ms");
return ast;
}
public void setFutureFlag(StatementParsing.FUTURE future) {
myFutureFlag = future;
}
}
python/src/com/jetbrains/python/parsing/StatementParsing.java
+19 -3
View File
@@ -22,18 +22,23 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper {
@NonNls protected static final String TOK_FUTURE_IMPORT = "__future__";
@NonNls protected static final String TOK_WITH_STATEMENT = "with_statement";
@NonNls protected static final String TOK_NESTED_SCOPES = "nested_scopes";
@NonNls protected static final String TOK_PRINT_FUNCTION = "print_function";
@NonNls protected static final String TOK_WITH = "with";
@NonNls protected static final String TOK_AS = "as";
@NonNls protected static final String TOK_PRINT = "print";
protected enum Phase {NONE, FROM, FUTURE, IMPORT} // 'from __future__ import' phase
private Phase myFutureImportPhase = Phase.NONE;
private boolean myExpectAsKeyword = false;
protected enum FUTURE {ABSOLUTE_IMPORT, DIVISION, GENERATORS, NESTED_SCOPES, WITH_STATEMENT}
public enum FUTURE {ABSOLUTE_IMPORT, DIVISION, GENERATORS, NESTED_SCOPES, WITH_STATEMENT, PRINT_FUNCTION}
protected Set<FUTURE> myFutureFlags = EnumSet.noneOf(FUTURE.class);
protected StatementParsing(ParsingContext context) {
protected StatementParsing(ParsingContext context, @Nullable FUTURE futureFlag) {
super(context);
if (futureFlag != null) {
myFutureFlags.add(futureFlag);
}
}
public void parseStatement() {
@@ -89,7 +94,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper {
if (firstToken == null) {
return;
}
if (firstToken == PyTokenTypes.PRINT_KEYWORD) {
if (firstToken == PyTokenTypes.PRINT_KEYWORD && hasPrintStatement()) {
parsePrintStatement(builder, inSuite);
return;
}
@@ -193,6 +198,10 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper {
builder.error("statement expected, found " + firstToken.toString());
}
private boolean hasPrintStatement() {
return myContext.getLanguageLevel().hasPrintStatement() && !myFutureFlags.contains(FUTURE.PRINT_FUNCTION);
}
private void checkEndOfStatement(boolean inSuite) {
PsiBuilder builder = myContext.getBuilder();
if (builder.getTokenType() == PyTokenTypes.STATEMENT_BREAK) {
@@ -398,6 +407,9 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper {
else if (TOK_NESTED_SCOPES.equals(token_text)) {
myFutureFlags.add(FUTURE.NESTED_SCOPES);
}
else if (TOK_PRINT_FUNCTION.equals(token_text)) {
myFutureFlags.add(FUTURE.PRINT_FUNCTION);
}
}
}
myExpectAsKeyword = true; // possible 'as' comes as an ident; reparse it as keyword if found
@@ -723,6 +735,10 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper {
) {
return PyTokenTypes.WITH_KEYWORD;
}
else if (hasPrintStatement() && source == PyTokenTypes.IDENTIFIER &&
isWordAtPosition(text, start, end, TOK_PRINT)) {
return PyTokenTypes.PRINT_KEYWORD;
}
return source;
}
python/src/com/jetbrains/python/psi/LanguageLevel.java
+11 -2
View File
@@ -10,22 +10,28 @@ import org.jetbrains.annotations.NotNull;
* @author yole
*/
public enum LanguageLevel {
PYTHON24(false), PYTHON25(false), PYTHON26(true);
PYTHON24(false, true), PYTHON25(false, true), PYTHON26(true, true), PYTHON30(true, false);
public static LanguageLevel getDefault() {
return PYTHON26;
}
private boolean myHasWithStatement;
private boolean myHasPrintStatement;
LanguageLevel(boolean hasWithStatement) {
LanguageLevel(boolean hasWithStatement, boolean hasPrintStatement) {
myHasWithStatement = hasWithStatement;
myHasPrintStatement = hasPrintStatement;
}
public boolean hasWithStatement() {
return myHasWithStatement;
}
public boolean hasPrintStatement() {
return myHasPrintStatement;
}
public static LanguageLevel fromPythonVersion(String pythonVersion) {
if (pythonVersion.startsWith("2.6")) {
return PYTHON26;
@@ -33,6 +39,9 @@ public enum LanguageLevel {
if (pythonVersion.startsWith("2.5")) {
return PYTHON25;
}
if (pythonVersion.startsWith("3.0") || pythonVersion.startsWith("3.1")) {
return PYTHON30;
}
return PYTHON24;
}
python/src/com/jetbrains/python/psi/PyFileElementType.java
+12 -3
View File
@@ -1,6 +1,9 @@
package com.jetbrains.python.psi;
import com.intellij.lang.*;
import com.intellij.lang.ASTNode;
import com.intellij.lang.Language;
import com.intellij.lang.PsiBuilder;
import com.intellij.lang.PsiBuilderFactory;
import com.intellij.lexer.Lexer;
import com.intellij.openapi.project.Project;
import com.intellij.psi.PsiElement;
@@ -9,6 +12,7 @@ import com.intellij.psi.impl.source.tree.FileElement;
import com.intellij.psi.tree.IStubFileElementType;
import com.jetbrains.python.lexer.PythonIndentingLexer;
import com.jetbrains.python.parsing.PyParser;
import com.jetbrains.python.parsing.StatementParsing;
/**
* @author yole
@@ -20,7 +24,7 @@ public class PyFileElementType extends IStubFileElementType {
@Override
public int getStubVersion() {
return 4;
return 5;
}
@Override
@@ -34,7 +38,12 @@ public class PyFileElementType extends IStubFileElementType {
final PsiBuilder builder = factory.createBuilder(project, chameleon, lexer, getLanguage(), chameleon.getChars());
final PsiParser parser = new PyParser(languageLevel);
final PyParser parser = new PyParser(languageLevel);
if (languageLevel == LanguageLevel.PYTHON26 &&
node.getPsi().getContainingFile().getName().equals("__builtin__.py")) {
parser.setFutureFlag(StatementParsing.FUTURE.PRINT_FUNCTION);
}
return parser.parse(this, builder).getFirstChildNode();
}
python/testData/psi/PrintAsFunction26.py
+2
View File
@@ -0,0 +1,2 @@
from __future__ import print_function
print('a')
python/testData/psi/PrintAsFunction26.txt
+22
View File
@@ -0,0 +1,22 @@
PyFile:PrintAsFunction26.py
PyFromImportStatement
PsiElement(Py:FROM_KEYWORD)('from')
PsiWhiteSpace(' ')
PyReferenceExpression: __future__
PsiElement(Py:IDENTIFIER)('__future__')
PsiWhiteSpace(' ')
PsiElement(Py:IMPORT_KEYWORD)('import')
PsiWhiteSpace(' ')
PyImportElement
PyReferenceExpression: print_function
PsiElement(Py:IDENTIFIER)('print_function')
PsiWhiteSpace('\n')
PyExpressionStatement
PyCallExpression: print
PyReferenceExpression: print
PsiElement(Py:IDENTIFIER)('print')
PyArgumentList
PsiElement(Py:LPAR)('(')
PyStringLiteralExpression: a
PsiElement(Py:STRING_LITERAL)(''a'')
PsiElement(Py:RPAR)(')')
python/testSrc/com/jetbrains/python/PythonParsingTest.java
+4
View File
@@ -100,6 +100,10 @@ public class PythonParsingTest extends ParsingTestCase {
doTest(LanguageLevel.PYTHON26);
}
public void testPrintAsFunction26() throws Exception {
doTest(LanguageLevel.PYTHON26);
}
public void doTest() throws Exception {
doTest(LanguageLevel.PYTHON25);
}