From 6528173bdc49ebfc49908606a731e6e34570084c Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Thu, 14 Jan 2010 21:05:05 +0300 Subject: [PATCH] Python language level support (PY-261), correct parsing of with statements in Python 2.6 (PY-259) --- python/src/META-INF/python-plugin-common.xml | 3 +- .../com/jetbrains/python/PyHighlighter.java | 28 +-- .../python/PySyntaxHighlighterFactory.java | 15 +- .../com/jetbrains/python/PythonLanguage.java | 10 +- .../python/PythonParserDefinition.java | 16 -- .../python/lexer/PythonHighlightingLexer.java | 26 +++ .../python/parsing/ParsingContext.java | 9 +- .../jetbrains/python/parsing/PyParser.java | 38 ++-- .../python/parsing/StatementParsing.java | 20 ++- .../jetbrains/python/psi/LanguageLevel.java | 57 ++++++ .../src/com/jetbrains/python/psi/PyFile.java | 2 + .../python/psi/PyFileElementType.java | 50 ++++++ .../jetbrains/python/psi/impl/PyFileImpl.java | 32 ++-- .../psi/impl/PythonLanguageLevelPusher.java | 165 ++++++++++++++++++ .../jetbrains/python/sdk/PythonSdkType.java | 15 ++ python/testData/psi/WithStatement26.py | 2 + python/testData/psi/WithStatement26.txt | 16 ++ .../jetbrains/python/PythonParsingTest.java | 61 ++++--- 18 files changed, 451 insertions(+), 114 deletions(-) create mode 100644 python/src/com/jetbrains/python/lexer/PythonHighlightingLexer.java create mode 100644 python/src/com/jetbrains/python/psi/LanguageLevel.java create mode 100644 python/src/com/jetbrains/python/psi/PyFileElementType.java create mode 100644 python/src/com/jetbrains/python/psi/impl/PythonLanguageLevelPusher.java create mode 100644 python/testData/psi/WithStatement26.py create mode 100644 python/testData/psi/WithStatement26.txt diff --git a/python/src/META-INF/python-plugin-common.xml b/python/src/META-INF/python-plugin-common.xml index d79422e62432..19ee2ad9dddc 100644 --- a/python/src/META-INF/python-plugin-common.xml +++ b/python/src/META-INF/python-plugin-common.xml @@ -1,4 +1,3 @@ - @@ -99,6 +98,8 @@ + + diff --git a/python/src/com/jetbrains/python/PyHighlighter.java b/python/src/com/jetbrains/python/PyHighlighter.java index 4f268ccaef60..a80eba876cb0 100644 --- a/python/src/com/jetbrains/python/PyHighlighter.java +++ b/python/src/com/jetbrains/python/PyHighlighter.java @@ -1,47 +1,34 @@ -/* - * Copyright 2005 Pythonid Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS"; BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package com.jetbrains.python; import com.intellij.lexer.LayeredLexer; import com.intellij.lexer.Lexer; import com.intellij.openapi.editor.HighlighterColors; -import static com.intellij.openapi.editor.SyntaxHighlighterColors.*; import com.intellij.openapi.editor.colors.TextAttributesKey; import com.intellij.openapi.editor.markup.TextAttributes; import com.intellij.openapi.fileTypes.SyntaxHighlighterBase; import com.intellij.psi.StringEscapesTokenTypes; import com.intellij.psi.tree.IElementType; import com.jetbrains.python.lexer.PyStringLiteralLexer; -import com.jetbrains.python.lexer.PythonFutureAwareLexer; +import com.jetbrains.python.lexer.PythonHighlightingLexer; +import com.jetbrains.python.psi.LanguageLevel; import org.jetbrains.annotations.NotNull; import java.awt.*; import java.util.HashMap; import java.util.Map; +import static com.intellij.openapi.editor.SyntaxHighlighterColors.*; + /** * Colors and lexer(s) needed for highlighting. */ public class PyHighlighter extends SyntaxHighlighterBase { private static Map keys1; + private final LanguageLevel myLanguageLevel; @NotNull public Lexer getHighlightingLexer() { - LayeredLexer ret = new LayeredLexer(new PythonFutureAwareLexer()); + LayeredLexer ret = new LayeredLexer(new PythonHighlightingLexer(myLanguageLevel)); ret.registerSelfStoppingLayer( new PyStringLiteralLexer(PyTokenTypes.STRING_LITERAL, false), // TODO: set according to 2.x or 3.0 lang level new IElementType[]{PyTokenTypes.STRING_LITERAL}, IElementType.EMPTY_ARRAY @@ -103,7 +90,8 @@ public class PyHighlighter extends SyntaxHighlighterBase { public static final TextAttributesKey PY_INVALID_STRING_ESCAPE = _copy("PY.INVALID_STRING_ESCAPE", INVALID_STRING_ESCAPE); - public PyHighlighter() { + public PyHighlighter(LanguageLevel languageLevel) { + myLanguageLevel = languageLevel; keys1 = new HashMap(); fillMap(keys1, PyTokenTypes.KEYWORDS, PY_KEYWORD); diff --git a/python/src/com/jetbrains/python/PySyntaxHighlighterFactory.java b/python/src/com/jetbrains/python/PySyntaxHighlighterFactory.java index 150b86c41c68..5c5f970eb2aa 100644 --- a/python/src/com/jetbrains/python/PySyntaxHighlighterFactory.java +++ b/python/src/com/jetbrains/python/PySyntaxHighlighterFactory.java @@ -1,19 +1,28 @@ package com.jetbrains.python; -import com.intellij.openapi.fileTypes.SyntaxHighlighterFactory; import com.intellij.openapi.fileTypes.SyntaxHighlighter; +import com.intellij.openapi.fileTypes.SyntaxHighlighterFactory; import com.intellij.openapi.project.Project; import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.util.containers.FactoryMap; +import com.jetbrains.python.psi.LanguageLevel; import org.jetbrains.annotations.NotNull; /** * @author yole */ public class PySyntaxHighlighterFactory extends SyntaxHighlighterFactory { - private final PyHighlighter myHighlighter = new PyHighlighter(); + @SuppressWarnings({"MismatchedQueryAndUpdateOfCollection"}) + private FactoryMap myMap = new FactoryMap() { + @Override + protected PyHighlighter create(LanguageLevel key) { + return new PyHighlighter(key); + } + }; @NotNull public SyntaxHighlighter getSyntaxHighlighter(final Project project, final VirtualFile virtualFile) { - return myHighlighter; + LanguageLevel languageLevel = LanguageLevel.forFile(virtualFile); + return myMap.get(languageLevel); } } diff --git a/python/src/com/jetbrains/python/PythonLanguage.java b/python/src/com/jetbrains/python/PythonLanguage.java index 4ab403b57e35..5df4d56b4a0c 100644 --- a/python/src/com/jetbrains/python/PythonLanguage.java +++ b/python/src/com/jetbrains/python/PythonLanguage.java @@ -5,8 +5,8 @@ import com.intellij.openapi.project.Project; import com.intellij.psi.PsiFile; import com.intellij.psi.PsiFileFactory; import com.intellij.psi.tree.IFileElementType; -import com.intellij.psi.tree.IStubFileElementType; import com.jetbrains.python.psi.PyElementGenerator; +import com.jetbrains.python.psi.PyFileElementType; import com.jetbrains.python.psi.impl.PyElementGeneratorImpl; import com.jetbrains.python.validation.*; @@ -18,12 +18,7 @@ import java.util.concurrent.CopyOnWriteArraySet; */ public class PythonLanguage extends Language { private final PyElementGenerator elementGenerator = new PyElementGeneratorImpl(this); - private final IFileElementType ELTYPE_FILE = new IStubFileElementType(this) { - @Override - public int getStubVersion() { - return 3; - } - }; + private final IFileElementType ELTYPE_FILE; private final Set> _annotators = new CopyOnWriteArraySet>(); public static PythonLanguage getInstance() { @@ -47,6 +42,7 @@ public class PythonLanguage extends Language { protected PythonLanguage() { super("Python"); + ELTYPE_FILE = new PyFileElementType(this); } public IFileElementType getFileElementType() { diff --git a/python/src/com/jetbrains/python/PythonParserDefinition.java b/python/src/com/jetbrains/python/PythonParserDefinition.java index 18717cd4ce15..756e0711f416 100644 --- a/python/src/com/jetbrains/python/PythonParserDefinition.java +++ b/python/src/com/jetbrains/python/PythonParserDefinition.java @@ -1,19 +1,3 @@ -/* - * Copyright 2005 Pythonid Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS"; BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package com.jetbrains.python; import com.intellij.extapi.psi.ASTWrapperPsiElement; diff --git a/python/src/com/jetbrains/python/lexer/PythonHighlightingLexer.java b/python/src/com/jetbrains/python/lexer/PythonHighlightingLexer.java new file mode 100644 index 000000000000..4e92461223f7 --- /dev/null +++ b/python/src/com/jetbrains/python/lexer/PythonHighlightingLexer.java @@ -0,0 +1,26 @@ +package com.jetbrains.python.lexer; + +import com.intellij.psi.tree.IElementType; +import com.jetbrains.python.PyTokenTypes; +import com.jetbrains.python.psi.LanguageLevel; + +/** + * @author yole + */ +public class PythonHighlightingLexer extends PythonLexer { + private final LanguageLevel myLanguageLevel; + + public PythonHighlightingLexer(LanguageLevel languageLevel) { + myLanguageLevel = languageLevel; + } + + @Override + public IElementType getTokenType() { + if (myLanguageLevel.hasWithStatement()) { + final String tokenText = getTokenText(); + if (tokenText.equals("with")) return PyTokenTypes.WITH_KEYWORD; + if (tokenText.equals("as")) return PyTokenTypes.AS_KEYWORD; + } + return super.getTokenType(); + } +} diff --git a/python/src/com/jetbrains/python/parsing/ParsingContext.java b/python/src/com/jetbrains/python/parsing/ParsingContext.java index db4714acab84..99b2fa9e9b9f 100644 --- a/python/src/com/jetbrains/python/parsing/ParsingContext.java +++ b/python/src/com/jetbrains/python/parsing/ParsingContext.java @@ -16,15 +16,18 @@ package com.jetbrains.python.parsing; import com.intellij.lang.PsiBuilder; +import com.jetbrains.python.psi.LanguageLevel; public class ParsingContext { private final StatementParsing stmtParser; private final ExpressionParsing expressionParser; private final FunctionParsing functionParser; private final PsiBuilder myBuilder; + private final LanguageLevel myLanguageLevel; - public ParsingContext(final PsiBuilder builder) { + public ParsingContext(final PsiBuilder builder, LanguageLevel languageLevel) { myBuilder = builder; + myLanguageLevel = languageLevel; stmtParser = new StatementParsing(this); expressionParser = new ExpressionParsing(this); functionParser = new FunctionParsing(this); @@ -45,4 +48,8 @@ public class ParsingContext { public PsiBuilder getBuilder() { return myBuilder; } + + public LanguageLevel getLanguageLevel() { + return myLanguageLevel; + } } diff --git a/python/src/com/jetbrains/python/parsing/PyParser.java b/python/src/com/jetbrains/python/parsing/PyParser.java index 17b97d3f0246..13eb44bd9af4 100644 --- a/python/src/com/jetbrains/python/parsing/PyParser.java +++ b/python/src/com/jetbrains/python/parsing/PyParser.java @@ -1,19 +1,3 @@ -/* - * Copyright 2005 Pythonid Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS"; BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package com.jetbrains.python.parsing; import com.intellij.lang.ASTNode; @@ -21,25 +5,31 @@ import com.intellij.lang.PsiBuilder; import com.intellij.lang.PsiParser; import com.intellij.openapi.diagnostic.Logger; import com.intellij.psi.tree.IElementType; +import com.jetbrains.python.psi.LanguageLevel; import org.jetbrains.annotations.NotNull; /** - * Created by IntelliJ IDEA. - * User: yole - * Date: 28.05.2005 - * Time: 10:06:10 - * To change this template use File | Settings | File Templates. + * @author yole */ public class PyParser implements PsiParser { private static final Logger LOGGER = Logger.getInstance(PyParser.class.getName()); - public + private final LanguageLevel myLanguageLevel; + + public PyParser() { + myLanguageLevel = LanguageLevel.getDefault(); + } + + public PyParser(LanguageLevel languageLevel) { + myLanguageLevel = languageLevel; + } + @NotNull - ASTNode parse(IElementType root, PsiBuilder builder) { + public ASTNode parse(IElementType root, PsiBuilder builder) { builder.setDebugMode(false); long start = System.currentTimeMillis(); final PsiBuilder.Marker rootMarker = builder.mark(); - ParsingContext context = new ParsingContext(builder); + ParsingContext context = new ParsingContext(builder, myLanguageLevel); StatementParsing stmt_parser = context.getStatementParser(); builder.setTokenTypeRemapper(stmt_parser); // must be done before touching the caching lexer with eof() call. while (!builder.eof()) { diff --git a/python/src/com/jetbrains/python/parsing/StatementParsing.java b/python/src/com/jetbrains/python/parsing/StatementParsing.java index d7afeb60a51c..aba42aacb7af 100644 --- a/python/src/com/jetbrains/python/parsing/StatementParsing.java +++ b/python/src/com/jetbrains/python/parsing/StatementParsing.java @@ -630,6 +630,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { final PsiBuilder.Marker statement = myBuilder.mark(); myBuilder.advanceLexer(); getExpressionParser().parseExpression(); + myExpectAsKeyword = true; if (myBuilder.getTokenType() == PyTokenTypes.AS_KEYWORD) { myBuilder.advanceLexer(); getExpressionParser().parseExpression(true, true); // 'as' is followed by a target @@ -702,27 +703,34 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { } public IElementType filter(final IElementType source, final int start, final int end, final CharSequence text) { if ( - (myFutureFlags.contains(FUTURE.WITH_STATEMENT) || myExpectAsKeyword) && - source == PyTokenTypes.IDENTIFIER && - CharArrayUtil.regionMatches(text, start, end, TOK_AS) + (myExpectAsKeyword || myContext.getLanguageLevel().hasWithStatement()) && + source == PyTokenTypes.IDENTIFIER && isWordAtPosition(text, start, end, TOK_AS) ) { return PyTokenTypes.AS_KEYWORD; } else if ( // filter (myFutureImportPhase == Phase.FROM) && source == PyTokenTypes.IDENTIFIER && - CharArrayUtil.regionMatches(text, start, end, TOK_FUTURE_IMPORT) + isWordAtPosition(text, start, end, TOK_FUTURE_IMPORT) ) { myFutureImportPhase = Phase.FUTURE; return source; } else if ( - myFutureFlags.contains(FUTURE.WITH_STATEMENT) && + hasWithStatement() && source == PyTokenTypes.IDENTIFIER && - CharArrayUtil.regionMatches(text, start, end, TOK_WITH) + isWordAtPosition(text, start, end, TOK_WITH) ) { return PyTokenTypes.WITH_KEYWORD; } return source; } + + private static boolean isWordAtPosition(CharSequence text, int start, int end, final String tokenText) { + return CharArrayUtil.regionMatches(text, start, end, tokenText) && end - start == tokenText.length(); + } + + private boolean hasWithStatement() { + return myContext.getLanguageLevel().hasWithStatement() || myFutureFlags.contains(FUTURE.WITH_STATEMENT); + } } diff --git a/python/src/com/jetbrains/python/psi/LanguageLevel.java b/python/src/com/jetbrains/python/psi/LanguageLevel.java new file mode 100644 index 000000000000..80a18ffccb1d --- /dev/null +++ b/python/src/com/jetbrains/python/psi/LanguageLevel.java @@ -0,0 +1,57 @@ +package com.jetbrains.python.psi; + +import com.intellij.openapi.application.ApplicationManager; +import com.intellij.openapi.util.Key; +import com.intellij.openapi.vfs.VirtualFile; +import com.jetbrains.python.psi.impl.PythonLanguageLevelPusher; +import org.jetbrains.annotations.NotNull; + +/** + * @author yole + */ +public enum LanguageLevel { + PYTHON24(false), PYTHON25(false), PYTHON26(true); + + public static LanguageLevel getDefault() { + return PYTHON26; + } + + private boolean myHasWithStatement; + + LanguageLevel(boolean hasWithStatement) { + myHasWithStatement = hasWithStatement; + } + + public boolean hasWithStatement() { + return myHasWithStatement; + } + + public static LanguageLevel fromPythonVersion(String pythonVersion) { + if (pythonVersion.startsWith("2.6")) { + return PYTHON26; + } + if (pythonVersion.startsWith("2.5")) { + return PYTHON25; + } + return PYTHON24; + } + + public static final Key KEY = new Key("python.language.level"); + + @NotNull + public static LanguageLevel forFile(VirtualFile virtualFile) { + final VirtualFile folder = virtualFile.getParent(); + if (folder != null) { + final LanguageLevel level = folder.getUserData(KEY); + if (level != null) return level; + } + else if (ApplicationManager.getApplication().isUnitTestMode()) { + final LanguageLevel languageLevel = PythonLanguageLevelPusher.FORCE_LANGUAGE_LEVEL; + if (languageLevel != null) { + return languageLevel; + } + } + + return getDefault(); + } +} diff --git a/python/src/com/jetbrains/python/psi/PyFile.java b/python/src/com/jetbrains/python/psi/PyFile.java index db339e85e158..9e05975cde14 100644 --- a/python/src/com/jetbrains/python/psi/PyFile.java +++ b/python/src/com/jetbrains/python/psi/PyFile.java @@ -40,4 +40,6 @@ public interface PyFile extends PyElement, PsiFile, PyDocStringOwner { @Nullable PyFunction findTopLevelFunction(String name); + + LanguageLevel getLanguageLevel(); } diff --git a/python/src/com/jetbrains/python/psi/PyFileElementType.java b/python/src/com/jetbrains/python/psi/PyFileElementType.java new file mode 100644 index 000000000000..ce0e227461ea --- /dev/null +++ b/python/src/com/jetbrains/python/psi/PyFileElementType.java @@ -0,0 +1,50 @@ +package com.jetbrains.python.psi; + +import com.intellij.lang.*; +import com.intellij.lexer.Lexer; +import com.intellij.openapi.project.Project; +import com.intellij.psi.PsiElement; +import com.intellij.psi.PsiFile; +import com.intellij.psi.impl.source.tree.FileElement; +import com.intellij.psi.tree.IStubFileElementType; +import com.jetbrains.python.lexer.PythonIndentingLexer; +import com.jetbrains.python.parsing.PyParser; + +/** +* @author yole +*/ +public class PyFileElementType extends IStubFileElementType { + public PyFileElementType(Language language) { + super(language); + } + + @Override + public int getStubVersion() { + return 4; + } + + @Override + public ASTNode parseContents(ASTNode chameleon) { + final FileElement node = (FileElement)chameleon; + final LanguageLevel languageLevel = getLanguageLevel(node.getPsi()); + final Lexer lexer = new PythonIndentingLexer(); + + final Project project = chameleon.getPsi().getProject(); + final PsiBuilderFactory factory = PsiBuilderFactory.getInstance(); + + final PsiBuilder builder = factory.createBuilder(project, chameleon, lexer, getLanguage(), chameleon.getChars()); + + final PsiParser parser = new PyParser(languageLevel); + return parser.parse(this, builder).getFirstChildNode(); + } + + private static LanguageLevel getLanguageLevel(PsiElement psi) { + final PsiFile file = psi.getContainingFile(); + if (!(file instanceof PyFile)) { + final PsiElement context = file.getContext(); + if (context != null) return getLanguageLevel(context); + return LanguageLevel.getDefault(); + } + return ((PyFile)file).getLanguageLevel(); + } +} diff --git a/python/src/com/jetbrains/python/psi/impl/PyFileImpl.java b/python/src/com/jetbrains/python/psi/impl/PyFileImpl.java index c76eeeb5230c..567f6e102606 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyFileImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyFileImpl.java @@ -1,19 +1,3 @@ -/* - * Copyright 2005 Pythonid Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS"; BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package com.jetbrains.python.psi.impl; import com.intellij.extapi.psi.PsiFileBase; @@ -27,6 +11,7 @@ import com.intellij.psi.scope.PsiScopeProcessor; import com.intellij.psi.stubs.StubElement; import com.intellij.psi.tree.IElementType; import com.intellij.psi.tree.TokenSet; +import com.intellij.util.indexing.FileBasedIndex; import com.jetbrains.python.PyElementTypes; import com.jetbrains.python.PythonDosStringFinder; import com.jetbrains.python.PythonFileType; @@ -78,6 +63,21 @@ public class PyFileImpl extends PsiFileBase implements PyFile, PyExpression { return null; } + public LanguageLevel getLanguageLevel() { + if (myOriginalFile != null) { + return ((PyFileImpl) myOriginalFile).getLanguageLevel(); + } + VirtualFile virtualFile = getVirtualFile(); + + if (virtualFile == null) { + virtualFile = getUserData(FileBasedIndex.VIRTUAL_FILE); + } + if (virtualFile != null) { + return LanguageLevel.forFile(virtualFile); + } + return LanguageLevel.getDefault(); + } + public Icon getIcon(int flags) { return PythonFileType.INSTANCE.getIcon(); } diff --git a/python/src/com/jetbrains/python/psi/impl/PythonLanguageLevelPusher.java b/python/src/com/jetbrains/python/psi/impl/PythonLanguageLevelPusher.java new file mode 100644 index 000000000000..c141653f2a3d --- /dev/null +++ b/python/src/com/jetbrains/python/psi/impl/PythonLanguageLevelPusher.java @@ -0,0 +1,165 @@ +/* + * Copyright 2000-2009 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.jetbrains.python.psi.impl; + +import com.intellij.openapi.application.ApplicationManager; +import com.intellij.openapi.module.Module; +import com.intellij.openapi.module.ModuleManager; +import com.intellij.openapi.module.ModuleUtil; +import com.intellij.openapi.project.Project; +import com.intellij.openapi.projectRoots.Sdk; +import com.intellij.openapi.roots.JdkOrderEntry; +import com.intellij.openapi.roots.ModuleRootManager; +import com.intellij.openapi.roots.OrderEntry; +import com.intellij.openapi.roots.ProjectRootManager; +import com.intellij.openapi.roots.impl.FilePropertyPusher; +import com.intellij.openapi.roots.impl.PushedFilePropertiesUpdater; +import com.intellij.openapi.util.Key; +import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.openapi.vfs.newvfs.FileAttribute; +import com.intellij.util.FileContentUtil; +import com.intellij.util.containers.WeakHashMap; +import com.intellij.util.indexing.FileBasedIndex; +import com.intellij.util.messages.MessageBus; +import com.jetbrains.python.PythonFileType; +import com.jetbrains.python.psi.LanguageLevel; +import com.jetbrains.python.sdk.PythonSdkType; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * @author yole + */ +public class PythonLanguageLevelPusher implements FilePropertyPusher { + public static LanguageLevel FORCE_LANGUAGE_LEVEL = null; + private final Map myModuleSdks = new WeakHashMap(); + + public static void pushLanguageLevel(final Project project) { + PushedFilePropertiesUpdater.getInstance(project).pushAll(new PythonLanguageLevelPusher()); + } + + public void initExtra(Project project, MessageBus bus, Engine languageLevelUpdater) { + final Module[] modules = ModuleManager.getInstance(project).getModules(); + for (Module module : modules) { + myModuleSdks.put(module, ModuleRootManager.getInstance(module).getSdk()); + } + } + + @NotNull + public Key getFileDataKey() { + return LanguageLevel.KEY; + } + + public boolean pushDirectoriesOnly() { + return true; + } + + public LanguageLevel getDefaultValue() { + return LanguageLevel.getDefault(); + } + + public LanguageLevel getImmediateValue(Project project, VirtualFile file) { + if (ApplicationManager.getApplication().isUnitTestMode() && FORCE_LANGUAGE_LEVEL != null) { + return FORCE_LANGUAGE_LEVEL; + } + + final Module module = ModuleUtil.findModuleForFile(file, project); + if (module != null) { + return getImmediateValue(module); + } + final Sdk sdk = findSdk(project, file); + if (sdk != null) { + return PythonSdkType.getLanguageLevelForSdk(sdk); + } + return null; + } + + @Nullable + private static Sdk findSdk(Project project, VirtualFile file) { + if (file != null) { + final List orderEntries = ProjectRootManager.getInstance(project).getFileIndex().getOrderEntriesForFile(file); + for (OrderEntry orderEntry : orderEntries) { + if (orderEntry instanceof JdkOrderEntry) { + return ((JdkOrderEntry)orderEntry).getJdk(); + } + } + } + return null; + } + + public LanguageLevel getImmediateValue(Module module) { + if (ApplicationManager.getApplication().isUnitTestMode() && FORCE_LANGUAGE_LEVEL != null) { + return FORCE_LANGUAGE_LEVEL; + } + + final Sdk sdk = ModuleRootManager.getInstance(module).getSdk(); + return PythonSdkType.getLanguageLevelForSdk(sdk); + } + + public boolean acceptsFile(VirtualFile file) { + return false; + } + + private static final FileAttribute PERSISTENCE = new FileAttribute("python_language_level_persistence", 1); + + public void persistAttribute(VirtualFile fileOrDir, LanguageLevel level) throws IOException { + final DataInputStream iStream = PERSISTENCE.readAttribute(fileOrDir); + if (iStream != null) { + try { + final int oldLevelOrdinal = iStream.readInt(); + if (oldLevelOrdinal == level.ordinal()) return; + } + finally { + iStream.close(); + } + } + + final DataOutputStream oStream = PERSISTENCE.writeAttribute(fileOrDir); + oStream.writeInt(level.ordinal()); + oStream.close(); + + for (VirtualFile child : fileOrDir.getChildren()) { + if (!child.isDirectory() && PythonFileType.INSTANCE.equals(child.getFileType())) { + FileBasedIndex.getInstance().requestReindex(child); + } + } + } + + public void afterRootsChanged(Project project) { + final Module[] modules = ModuleManager.getInstance(project).getModules(); + boolean needReparseOpenFiles = false; + for (Module module : modules) { + Sdk newSdk = PythonSdkType.findPythonSdk(module); + if (myModuleSdks.containsKey(module)) { + Sdk oldSdk = myModuleSdks.get(module); + if ((newSdk != null || oldSdk != null) && newSdk != oldSdk) { + needReparseOpenFiles = true; + } + } + myModuleSdks.put(module, newSdk); + } + if (needReparseOpenFiles) { + FileContentUtil.reparseFiles(project, Collections.emptyList(), true); + } + } +} \ No newline at end of file diff --git a/python/src/com/jetbrains/python/sdk/PythonSdkType.java b/python/src/com/jetbrains/python/sdk/PythonSdkType.java index ebb95ea67e2e..cb8820c44ead 100644 --- a/python/src/com/jetbrains/python/sdk/PythonSdkType.java +++ b/python/src/com/jetbrains/python/sdk/PythonSdkType.java @@ -29,6 +29,7 @@ import com.intellij.util.ArrayUtil; import com.jetbrains.python.PythonFileType; import com.jetbrains.python.PythonHelpersLocator; import com.jetbrains.python.facet.PythonFacetSettings; +import com.jetbrains.python.psi.LanguageLevel; import org.jdom.Element; import org.jetbrains.annotations.NonNls; import org.jetbrains.annotations.NotNull; @@ -452,4 +453,18 @@ public class PythonSdkType extends SdkType { } return null; } + + public static LanguageLevel getLanguageLevelForSdk(@Nullable Sdk sdk) { + if (sdk != null) { + String version = sdk.getVersionString(); + if (version != null) { + // HACK rewrite in some nicer way? + if (version.startsWith("Python ") || version.startsWith("Jython ")) { + String pythonVersion = version.substring("Python ".length()); + return LanguageLevel.fromPythonVersion(pythonVersion); + } + } + } + return LanguageLevel.getDefault(); + } } diff --git a/python/testData/psi/WithStatement26.py b/python/testData/psi/WithStatement26.py new file mode 100644 index 000000000000..34a2a4a9bd42 --- /dev/null +++ b/python/testData/psi/WithStatement26.py @@ -0,0 +1,2 @@ +with x as y: + pass \ No newline at end of file diff --git a/python/testData/psi/WithStatement26.txt b/python/testData/psi/WithStatement26.txt new file mode 100644 index 000000000000..9835c548757f --- /dev/null +++ b/python/testData/psi/WithStatement26.txt @@ -0,0 +1,16 @@ +PyFile:WithStatement26.py + PyWithStatement + PsiElement(Py:WITH_KEYWORD)('with') + PsiWhiteSpace(' ') + PyReferenceExpression: x + PsiElement(Py:IDENTIFIER)('x') + PsiWhiteSpace(' ') + PsiElement(Py:AS_KEYWORD)('as') + PsiWhiteSpace(' ') + PyTargetExpression: y + PsiElement(Py:IDENTIFIER)('y') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyPassStatement + PsiElement(Py:PASS_KEYWORD)('pass') \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/python/PythonParsingTest.java b/python/testSrc/com/jetbrains/python/PythonParsingTest.java index 2c46f70a2a5c..27e952890629 100644 --- a/python/testSrc/com/jetbrains/python/PythonParsingTest.java +++ b/python/testSrc/com/jetbrains/python/PythonParsingTest.java @@ -2,6 +2,8 @@ package com.jetbrains.python; import com.intellij.testFramework.ParsingTestCase; import com.jetbrains.python.fixtures.PyLightFixtureTestCase; +import com.jetbrains.python.psi.LanguageLevel; +import com.jetbrains.python.psi.impl.PythonLanguageLevelPusher; /** * @author yole @@ -17,80 +19,99 @@ public class PythonParsingTest extends ParsingTestCase { } public void testHelloWorld() throws Exception { - doTest(true); + doTest(); } public void testIfStatement() throws Exception { - doTest(true); + doTest(); } public void testConditionalExpression() throws Exception { - doTest(true); + doTest(); } public void testSubscribedAssignmentLHS() throws Exception { - doTest(true); + doTest(); } public void testConditionalParenLambda() throws Exception { - doTest(true); + doTest(); } public void testLambdaComprehension() throws Exception { - doTest(true); + doTest(); } public void testLambdaConditional() throws Exception { - doTest(true); + doTest(); } public void testTryExceptFinally() throws Exception { - doTest(true); + doTest(); } public void testTryFinally() throws Exception { - doTest(true); + doTest(); } public void testYieldStatement() throws Exception { - doTest(true); + doTest(); } public void testYieldInAssignment() throws Exception { - doTest(true); + doTest(); } public void testYieldInAugAssignment() throws Exception { - doTest(true); + doTest(); } public void testYieldInParentheses() throws Exception { - doTest(true); + doTest(); } public void _testYieldAsArgument() throws Exception { // this is a strange case: PEP 342 says this syntax is valid, but // Python 2.5 doesn't accept it. let's stick with Python behavior for now - doTest(true); + doTest(); } public void testWithStatement() throws Exception { - doTest(true); + doTest(); } - + public void testWithStatement2() throws Exception { - doTest(true); + doTest(); } public void testImportStmt() throws Exception { - doTest(true); + doTest(); } public void testDecoratedFunction() throws Exception { - doTest(true); + doTest(); } public void testTryExceptAs() throws Exception { // PY-293 - doTest(true); + doTest(); + } + + public void testWithStatement26() throws Exception { + doTest(LanguageLevel.PYTHON26); + } + + public void doTest() throws Exception { + doTest(LanguageLevel.PYTHON25); + } + + public void doTest(LanguageLevel languageLevel) throws Exception { + PythonLanguageLevelPusher.FORCE_LANGUAGE_LEVEL = languageLevel; + PythonLanguageLevelPusher.pushLanguageLevel(getProject()); + try { + doTest(true); + } + finally { + PythonLanguageLevelPusher.FORCE_LANGUAGE_LEVEL = null; + } } }