Python language level support (PY-261), correct parsing of with statements in Python 2.6 (PY-259)

python/src/META-INF/python-plugin-common.xml

@@ -1,4 +1,3 @@
-<!DOCTYPE idea-plugin PUBLIC "Plugin/DTD" "http://plugins.intellij.net/plugin.dtd">
 <idea-plugin version="2">
 
   <extensions defaultExtensionNs="com.intellij">
@@ -99,6 +98,8 @@
     <codeInsight.gotoSuper language="Python" implementationClass="com.jetbrains.python.codeInsight.PyGotoSuperHandler"/>
     <gotoDeclarationHandler implementation="com.jetbrains.python.codeInsight.PyBreakContinueGotoProvider" order="FIRST"/>
 
+    <filePropertyPusher implementation="com.jetbrains.python.psi.impl.PythonLanguageLevelPusher"/>
+
     <!-- Django -->
     <annotator language="Python" implementationClass="com.jetbrains.django.annotator.ViewMethodAnnotator"/>
     <configurationType implementation="com.jetbrains.django.run.DjangoServerRunConfigurationType"/>

python/src/com/jetbrains/python/PyHighlighter.java

@@ -1,47 +1,34 @@
-/*
- *  Copyright 2005 Pythonid Project
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS"; BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
 package com.jetbrains.python;
 
 import com.intellij.lexer.LayeredLexer;
 import com.intellij.lexer.Lexer;
 import com.intellij.openapi.editor.HighlighterColors;
-import static com.intellij.openapi.editor.SyntaxHighlighterColors.*;
 import com.intellij.openapi.editor.colors.TextAttributesKey;
 import com.intellij.openapi.editor.markup.TextAttributes;
 import com.intellij.openapi.fileTypes.SyntaxHighlighterBase;
 import com.intellij.psi.StringEscapesTokenTypes;
 import com.intellij.psi.tree.IElementType;
 import com.jetbrains.python.lexer.PyStringLiteralLexer;
-import com.jetbrains.python.lexer.PythonFutureAwareLexer;
+import com.jetbrains.python.lexer.PythonHighlightingLexer;
+import com.jetbrains.python.psi.LanguageLevel;
 import org.jetbrains.annotations.NotNull;
 
 import java.awt.*;
 import java.util.HashMap;
 import java.util.Map;
 
+import static com.intellij.openapi.editor.SyntaxHighlighterColors.*;
+
 /**
  * Colors and lexer(s) needed for highlighting.
  */
 public class PyHighlighter extends SyntaxHighlighterBase {
   private static Map<IElementType, TextAttributesKey> keys1;
+  private final LanguageLevel myLanguageLevel;
 
   @NotNull
   public Lexer getHighlightingLexer() {
-    LayeredLexer ret = new LayeredLexer(new PythonFutureAwareLexer());
+    LayeredLexer ret = new LayeredLexer(new PythonHighlightingLexer(myLanguageLevel));
     ret.registerSelfStoppingLayer(
       new PyStringLiteralLexer(PyTokenTypes.STRING_LITERAL, false), // TODO: set according to 2.x or 3.0 lang level
       new IElementType[]{PyTokenTypes.STRING_LITERAL}, IElementType.EMPTY_ARRAY
@@ -103,7 +90,8 @@ public class PyHighlighter extends SyntaxHighlighterBase {
   public static final TextAttributesKey PY_INVALID_STRING_ESCAPE = _copy("PY.INVALID_STRING_ESCAPE", INVALID_STRING_ESCAPE);
 
 
-  public PyHighlighter() {
+  public PyHighlighter(LanguageLevel languageLevel) {
+    myLanguageLevel = languageLevel;
     keys1 = new HashMap<IElementType, TextAttributesKey>();
 
     fillMap(keys1, PyTokenTypes.KEYWORDS, PY_KEYWORD);

python/src/com/jetbrains/python/PySyntaxHighlighterFactory.java

@@ -1,19 +1,28 @@
 package com.jetbrains.python;
 
-import com.intellij.openapi.fileTypes.SyntaxHighlighterFactory;
 import com.intellij.openapi.fileTypes.SyntaxHighlighter;
+import com.intellij.openapi.fileTypes.SyntaxHighlighterFactory;
 import com.intellij.openapi.project.Project;
 import com.intellij.openapi.vfs.VirtualFile;
+import com.intellij.util.containers.FactoryMap;
+import com.jetbrains.python.psi.LanguageLevel;
 import org.jetbrains.annotations.NotNull;
 
 /**
  * @author yole
  */
 public class PySyntaxHighlighterFactory extends SyntaxHighlighterFactory {
-  private final PyHighlighter myHighlighter = new PyHighlighter();
+  @SuppressWarnings({"MismatchedQueryAndUpdateOfCollection"})
+  private FactoryMap<LanguageLevel, PyHighlighter> myMap = new FactoryMap<LanguageLevel, PyHighlighter>() {
+    @Override
+    protected PyHighlighter create(LanguageLevel key) {
+      return new PyHighlighter(key);
+    }
+  };
 
   @NotNull
   public SyntaxHighlighter getSyntaxHighlighter(final Project project, final VirtualFile virtualFile) {
-    return myHighlighter;
+    LanguageLevel languageLevel = LanguageLevel.forFile(virtualFile);
+    return myMap.get(languageLevel);
   }
 }

python/src/com/jetbrains/python/PythonLanguage.java

@@ -5,8 +5,8 @@ import com.intellij.openapi.project.Project;
 import com.intellij.psi.PsiFile;
 import com.intellij.psi.PsiFileFactory;
 import com.intellij.psi.tree.IFileElementType;
-import com.intellij.psi.tree.IStubFileElementType;
 import com.jetbrains.python.psi.PyElementGenerator;
+import com.jetbrains.python.psi.PyFileElementType;
 import com.jetbrains.python.psi.impl.PyElementGeneratorImpl;
 import com.jetbrains.python.validation.*;
 
@@ -18,12 +18,7 @@ import java.util.concurrent.CopyOnWriteArraySet;
  */
 public class PythonLanguage extends Language {
   private final PyElementGenerator elementGenerator = new PyElementGeneratorImpl(this);
-  private final IFileElementType ELTYPE_FILE = new IStubFileElementType(this) {
-    @Override
-    public int getStubVersion() {
-      return 3;
-    }
-  };
+  private final IFileElementType ELTYPE_FILE;
   private final Set<Class<? extends PyAnnotator>> _annotators = new CopyOnWriteArraySet<Class<? extends PyAnnotator>>();
 
   public static PythonLanguage getInstance() {
@@ -47,6 +42,7 @@ public class PythonLanguage extends Language {
 
   protected PythonLanguage() {
     super("Python");
+    ELTYPE_FILE = new PyFileElementType(this);
   }
 
   public IFileElementType getFileElementType() {

python/src/com/jetbrains/python/PythonParserDefinition.java

@@ -1,19 +1,3 @@
-/*
- *  Copyright 2005 Pythonid Project
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS"; BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
 package com.jetbrains.python;
 
 import com.intellij.extapi.psi.ASTWrapperPsiElement;

python/src/com/jetbrains/python/lexer/PythonHighlightingLexer.java

@@ -0,0 +1,26 @@
+package com.jetbrains.python.lexer;
+
+import com.intellij.psi.tree.IElementType;
+import com.jetbrains.python.PyTokenTypes;
+import com.jetbrains.python.psi.LanguageLevel;
+
+/**
+ * @author yole
+ */
+public class PythonHighlightingLexer extends PythonLexer {
+  private final LanguageLevel myLanguageLevel;
+
+  public PythonHighlightingLexer(LanguageLevel languageLevel) {
+    myLanguageLevel = languageLevel;
+  }
+
+  @Override
+  public IElementType getTokenType() {
+    if (myLanguageLevel.hasWithStatement()) {
+      final String tokenText = getTokenText();
+      if (tokenText.equals("with")) return PyTokenTypes.WITH_KEYWORD;
+      if (tokenText.equals("as")) return PyTokenTypes.AS_KEYWORD;
+    }
+    return super.getTokenType();
+  }
+}

python/src/com/jetbrains/python/parsing/ParsingContext.java

@@ -16,15 +16,18 @@
 package com.jetbrains.python.parsing;
 
 import com.intellij.lang.PsiBuilder;
+import com.jetbrains.python.psi.LanguageLevel;
 
 public class ParsingContext {
   private final StatementParsing stmtParser;
   private final ExpressionParsing expressionParser;
   private final FunctionParsing functionParser;
   private final PsiBuilder myBuilder;
+  private final LanguageLevel myLanguageLevel;
 
-  public ParsingContext(final PsiBuilder builder) {
+  public ParsingContext(final PsiBuilder builder, LanguageLevel languageLevel) {
     myBuilder = builder;
+    myLanguageLevel = languageLevel;
     stmtParser = new StatementParsing(this);
     expressionParser = new ExpressionParsing(this);
     functionParser = new FunctionParsing(this);
@@ -45,4 +48,8 @@ public class ParsingContext {
   public PsiBuilder getBuilder() {
     return myBuilder;
   }
+
+  public LanguageLevel getLanguageLevel() {
+    return myLanguageLevel;
+  }
 }

python/src/com/jetbrains/python/parsing/PyParser.java

@@ -1,19 +1,3 @@
-/*
- *  Copyright 2005 Pythonid Project
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS"; BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
 package com.jetbrains.python.parsing;
 
 import com.intellij.lang.ASTNode;
@@ -21,25 +5,31 @@ import com.intellij.lang.PsiBuilder;
 import com.intellij.lang.PsiParser;
 import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.psi.tree.IElementType;
+import com.jetbrains.python.psi.LanguageLevel;
 import org.jetbrains.annotations.NotNull;
 
 /**
- * Created by IntelliJ IDEA.
- * User: yole
- * Date: 28.05.2005
- * Time: 10:06:10
- * To change this template use File | Settings | File Templates.
+ * @author yole
  */
 public class PyParser implements PsiParser {
   private static final Logger LOGGER = Logger.getInstance(PyParser.class.getName());
 
-  public
+  private final LanguageLevel myLanguageLevel;
+
+  public PyParser() {
+    myLanguageLevel = LanguageLevel.getDefault();
+  }
+
+  public PyParser(LanguageLevel languageLevel) {
+    myLanguageLevel = languageLevel;
+  }
+
   @NotNull
-  ASTNode parse(IElementType root, PsiBuilder builder) {
+  public ASTNode parse(IElementType root, PsiBuilder builder) {
     builder.setDebugMode(false);
     long start = System.currentTimeMillis();
     final PsiBuilder.Marker rootMarker = builder.mark();
-    ParsingContext context = new ParsingContext(builder);
+    ParsingContext context = new ParsingContext(builder, myLanguageLevel);
     StatementParsing stmt_parser = context.getStatementParser();
     builder.setTokenTypeRemapper(stmt_parser); // must be done before touching the caching lexer with eof() call.
     while (!builder.eof()) {

python/src/com/jetbrains/python/parsing/StatementParsing.java

@@ -630,6 +630,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper {
     final PsiBuilder.Marker statement = myBuilder.mark();
     myBuilder.advanceLexer();
     getExpressionParser().parseExpression();
+    myExpectAsKeyword = true;
     if (myBuilder.getTokenType() == PyTokenTypes.AS_KEYWORD) {
       myBuilder.advanceLexer();
       getExpressionParser().parseExpression(true, true); // 'as' is followed by a target
@@ -702,27 +703,34 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper {
   }
   public IElementType filter(final IElementType source, final int start, final int end, final CharSequence text) {
     if (
-      (myFutureFlags.contains(FUTURE.WITH_STATEMENT) || myExpectAsKeyword) &&
-      source == PyTokenTypes.IDENTIFIER &&
-      CharArrayUtil.regionMatches(text, start, end, TOK_AS)
+      (myExpectAsKeyword || myContext.getLanguageLevel().hasWithStatement()) &&
+      source == PyTokenTypes.IDENTIFIER && isWordAtPosition(text, start, end, TOK_AS)
     ) {
       return PyTokenTypes.AS_KEYWORD;
     }
     else if ( // filter
         (myFutureImportPhase == Phase.FROM) &&
         source == PyTokenTypes.IDENTIFIER &&
-        CharArrayUtil.regionMatches(text, start, end, TOK_FUTURE_IMPORT)
+        isWordAtPosition(text, start, end, TOK_FUTURE_IMPORT)
     ) {
       myFutureImportPhase = Phase.FUTURE;
       return source;
     }
     else if (
-        myFutureFlags.contains(FUTURE.WITH_STATEMENT) &&
+        hasWithStatement() &&
         source == PyTokenTypes.IDENTIFIER &&
-        CharArrayUtil.regionMatches(text, start, end, TOK_WITH)
+        isWordAtPosition(text, start, end, TOK_WITH)
     ) {
       return PyTokenTypes.WITH_KEYWORD;
     }
     return source;
   }
+
+  private static boolean isWordAtPosition(CharSequence text, int start, int end, final String tokenText) {
+    return CharArrayUtil.regionMatches(text, start, end, tokenText) && end - start == tokenText.length();
+  }
+
+  private boolean hasWithStatement() {
+    return myContext.getLanguageLevel().hasWithStatement() || myFutureFlags.contains(FUTURE.WITH_STATEMENT);
+  }
 }

python/src/com/jetbrains/python/psi/LanguageLevel.java

@@ -0,0 +1,57 @@
+package com.jetbrains.python.psi;
+
+import com.intellij.openapi.application.ApplicationManager;
+import com.intellij.openapi.util.Key;
+import com.intellij.openapi.vfs.VirtualFile;
+import com.jetbrains.python.psi.impl.PythonLanguageLevelPusher;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * @author yole
+ */
+public enum LanguageLevel {
+  PYTHON24(false), PYTHON25(false), PYTHON26(true);
+
+  public static LanguageLevel getDefault() {
+    return PYTHON26;
+  }
+
+  private boolean myHasWithStatement;
+
+  LanguageLevel(boolean hasWithStatement) {
+    myHasWithStatement = hasWithStatement;
+  }
+
+  public boolean hasWithStatement() {
+    return myHasWithStatement;
+  }
+
+  public static LanguageLevel fromPythonVersion(String pythonVersion) {
+    if (pythonVersion.startsWith("2.6")) {
+      return PYTHON26;
+    }
+    if (pythonVersion.startsWith("2.5")) {
+      return PYTHON25;
+    }
+    return PYTHON24;
+  }
+
+  public static final Key<LanguageLevel> KEY = new Key<LanguageLevel>("python.language.level");
+
+  @NotNull
+  public static LanguageLevel forFile(VirtualFile virtualFile) {
+    final VirtualFile folder = virtualFile.getParent();
+    if (folder != null) {
+      final LanguageLevel level = folder.getUserData(KEY);
+      if (level != null) return level;
+    }
+    else if (ApplicationManager.getApplication().isUnitTestMode()) {
+      final LanguageLevel languageLevel = PythonLanguageLevelPusher.FORCE_LANGUAGE_LEVEL;
+      if (languageLevel != null) {
+        return languageLevel;
+      }
+    }
+
+    return getDefault();
+  }
+}

python/src/com/jetbrains/python/psi/PyFile.java

@@ -40,4 +40,6 @@ public interface PyFile extends PyElement, PsiFile, PyDocStringOwner {
 
   @Nullable
   PyFunction findTopLevelFunction(String name);
+
+  LanguageLevel getLanguageLevel();
 }

python/src/com/jetbrains/python/psi/PyFileElementType.java

@@ -0,0 +1,50 @@
+package com.jetbrains.python.psi;
+
+import com.intellij.lang.*;
+import com.intellij.lexer.Lexer;
+import com.intellij.openapi.project.Project;
+import com.intellij.psi.PsiElement;
+import com.intellij.psi.PsiFile;
+import com.intellij.psi.impl.source.tree.FileElement;
+import com.intellij.psi.tree.IStubFileElementType;
+import com.jetbrains.python.lexer.PythonIndentingLexer;
+import com.jetbrains.python.parsing.PyParser;
+
+/**
+* @author yole
+*/
+public class PyFileElementType extends IStubFileElementType {
+  public PyFileElementType(Language language) {
+    super(language);
+  }
+
+  @Override
+  public int getStubVersion() {
+    return 4;
+  }
+
+  @Override
+  public ASTNode parseContents(ASTNode chameleon) {
+    final FileElement node = (FileElement)chameleon;
+    final LanguageLevel languageLevel = getLanguageLevel(node.getPsi());
+    final Lexer lexer = new PythonIndentingLexer();
+
+    final Project project = chameleon.getPsi().getProject();
+    final PsiBuilderFactory factory = PsiBuilderFactory.getInstance();
+
+    final PsiBuilder builder = factory.createBuilder(project, chameleon, lexer, getLanguage(), chameleon.getChars());
+
+    final PsiParser parser = new PyParser(languageLevel);
+    return parser.parse(this, builder).getFirstChildNode();
+  }
+
+  private static LanguageLevel getLanguageLevel(PsiElement psi) {
+    final PsiFile file = psi.getContainingFile();
+    if (!(file instanceof PyFile)) {
+      final PsiElement context = file.getContext();
+      if (context != null) return getLanguageLevel(context);
+      return LanguageLevel.getDefault();
+    }
+    return ((PyFile)file).getLanguageLevel();
+  }
+}

python/src/com/jetbrains/python/psi/impl/PyFileImpl.java

@@ -1,19 +1,3 @@
-/*
- *  Copyright 2005 Pythonid Project
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS"; BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
 package com.jetbrains.python.psi.impl;
 
 import com.intellij.extapi.psi.PsiFileBase;
@@ -27,6 +11,7 @@ import com.intellij.psi.scope.PsiScopeProcessor;
 import com.intellij.psi.stubs.StubElement;
 import com.intellij.psi.tree.IElementType;
 import com.intellij.psi.tree.TokenSet;
+import com.intellij.util.indexing.FileBasedIndex;
 import com.jetbrains.python.PyElementTypes;
 import com.jetbrains.python.PythonDosStringFinder;
 import com.jetbrains.python.PythonFileType;
@@ -78,6 +63,21 @@ public class PyFileImpl extends PsiFileBase implements PyFile, PyExpression {
     return null;
   }
 
+  public LanguageLevel getLanguageLevel() {
+    if (myOriginalFile != null) {
+      return ((PyFileImpl) myOriginalFile).getLanguageLevel();
+    }
+    VirtualFile virtualFile = getVirtualFile();
+
+    if (virtualFile == null) {
+      virtualFile = getUserData(FileBasedIndex.VIRTUAL_FILE);
+    }
+    if (virtualFile != null) {
+      return LanguageLevel.forFile(virtualFile);
+    }
+    return LanguageLevel.getDefault();
+  }
+
   public Icon getIcon(int flags) {
     return PythonFileType.INSTANCE.getIcon();
   }

python/src/com/jetbrains/python/psi/impl/PythonLanguageLevelPusher.java

@@ -0,0 +1,165 @@
+/*
+ * Copyright 2000-2009 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.jetbrains.python.psi.impl;
+
+import com.intellij.openapi.application.ApplicationManager;
+import com.intellij.openapi.module.Module;
+import com.intellij.openapi.module.ModuleManager;
+import com.intellij.openapi.module.ModuleUtil;
+import com.intellij.openapi.project.Project;
+import com.intellij.openapi.projectRoots.Sdk;
+import com.intellij.openapi.roots.JdkOrderEntry;
+import com.intellij.openapi.roots.ModuleRootManager;
+import com.intellij.openapi.roots.OrderEntry;
+import com.intellij.openapi.roots.ProjectRootManager;
+import com.intellij.openapi.roots.impl.FilePropertyPusher;
+import com.intellij.openapi.roots.impl.PushedFilePropertiesUpdater;
+import com.intellij.openapi.util.Key;
+import com.intellij.openapi.vfs.VirtualFile;
+import com.intellij.openapi.vfs.newvfs.FileAttribute;
+import com.intellij.util.FileContentUtil;
+import com.intellij.util.containers.WeakHashMap;
+import com.intellij.util.indexing.FileBasedIndex;
+import com.intellij.util.messages.MessageBus;
+import com.jetbrains.python.PythonFileType;
+import com.jetbrains.python.psi.LanguageLevel;
+import com.jetbrains.python.sdk.PythonSdkType;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author yole
+ */
+public class PythonLanguageLevelPusher implements FilePropertyPusher<LanguageLevel> {
+  public static LanguageLevel FORCE_LANGUAGE_LEVEL = null;
+  private final Map<Module, Sdk> myModuleSdks = new WeakHashMap<Module, Sdk>();
+
+  public static void pushLanguageLevel(final Project project) {
+    PushedFilePropertiesUpdater.getInstance(project).pushAll(new PythonLanguageLevelPusher());
+  }
+
+  public void initExtra(Project project, MessageBus bus, Engine languageLevelUpdater) {
+    final Module[] modules = ModuleManager.getInstance(project).getModules();
+    for (Module module : modules) {
+      myModuleSdks.put(module, ModuleRootManager.getInstance(module).getSdk());
+    }
+  }
+
+  @NotNull
+  public Key<LanguageLevel> getFileDataKey() {
+    return LanguageLevel.KEY;
+  }
+
+  public boolean pushDirectoriesOnly() {
+    return true;
+  }
+
+  public LanguageLevel getDefaultValue() {
+    return LanguageLevel.getDefault();
+  }
+
+  public LanguageLevel getImmediateValue(Project project, VirtualFile file) {
+    if (ApplicationManager.getApplication().isUnitTestMode() && FORCE_LANGUAGE_LEVEL != null) {
+      return FORCE_LANGUAGE_LEVEL;
+    }
+
+    final Module module = ModuleUtil.findModuleForFile(file, project);
+    if (module != null) {
+      return getImmediateValue(module);
+    }
+    final Sdk sdk = findSdk(project, file);
+    if (sdk != null) {
+      return PythonSdkType.getLanguageLevelForSdk(sdk);
+    }
+    return null;
+  }
+
+  @Nullable
+  private static Sdk findSdk(Project project, VirtualFile file) {
+    if (file != null) {
+      final List<OrderEntry> orderEntries = ProjectRootManager.getInstance(project).getFileIndex().getOrderEntriesForFile(file);
+      for (OrderEntry orderEntry : orderEntries) {
+        if (orderEntry instanceof JdkOrderEntry) {
+          return ((JdkOrderEntry)orderEntry).getJdk();
+        }
+      }
+    }
+    return null;
+  }
+
+  public LanguageLevel getImmediateValue(Module module) {
+    if (ApplicationManager.getApplication().isUnitTestMode() && FORCE_LANGUAGE_LEVEL != null) {
+      return FORCE_LANGUAGE_LEVEL;
+    }
+
+    final Sdk sdk = ModuleRootManager.getInstance(module).getSdk();
+    return PythonSdkType.getLanguageLevelForSdk(sdk);
+  }
+
+  public boolean acceptsFile(VirtualFile file) {
+    return false;
+  }
+
+  private static final FileAttribute PERSISTENCE = new FileAttribute("python_language_level_persistence", 1);
+
+  public void persistAttribute(VirtualFile fileOrDir, LanguageLevel level) throws IOException {
+    final DataInputStream iStream = PERSISTENCE.readAttribute(fileOrDir);
+    if (iStream != null) {
+      try {
+        final int oldLevelOrdinal = iStream.readInt();
+        if (oldLevelOrdinal == level.ordinal()) return;
+      }
+      finally {
+        iStream.close();
+      }
+    }
+
+    final DataOutputStream oStream = PERSISTENCE.writeAttribute(fileOrDir);
+    oStream.writeInt(level.ordinal());
+    oStream.close();
+
+    for (VirtualFile child : fileOrDir.getChildren()) {
+      if (!child.isDirectory() && PythonFileType.INSTANCE.equals(child.getFileType())) {
+        FileBasedIndex.getInstance().requestReindex(child);
+      }
+    }
+  }
+
+  public void afterRootsChanged(Project project) {
+    final Module[] modules = ModuleManager.getInstance(project).getModules();
+    boolean needReparseOpenFiles = false;
+    for (Module module : modules) {
+      Sdk newSdk = PythonSdkType.findPythonSdk(module);
+      if (myModuleSdks.containsKey(module)) {
+        Sdk oldSdk = myModuleSdks.get(module);
+        if ((newSdk != null || oldSdk != null) && newSdk != oldSdk) {
+          needReparseOpenFiles = true;
+        }
+      }
+      myModuleSdks.put(module, newSdk);
+    }
+    if (needReparseOpenFiles) {
+      FileContentUtil.reparseFiles(project, Collections.<VirtualFile>emptyList(), true);
+    }
+  }
+}

python/src/com/jetbrains/python/sdk/PythonSdkType.java

@@ -29,6 +29,7 @@ import com.intellij.util.ArrayUtil;
 import com.jetbrains.python.PythonFileType;
 import com.jetbrains.python.PythonHelpersLocator;
 import com.jetbrains.python.facet.PythonFacetSettings;
+import com.jetbrains.python.psi.LanguageLevel;
 import org.jdom.Element;
 import org.jetbrains.annotations.NonNls;
 import org.jetbrains.annotations.NotNull;
@@ -452,4 +453,18 @@ public class PythonSdkType extends SdkType {
     }
     return null;
   }
+
+  public static LanguageLevel getLanguageLevelForSdk(@Nullable Sdk sdk) {
+    if (sdk != null) {
+      String version = sdk.getVersionString();
+      if (version != null) {
+        // HACK rewrite in some nicer way?
+        if (version.startsWith("Python ") || version.startsWith("Jython ")) {
+          String pythonVersion = version.substring("Python ".length());
+          return LanguageLevel.fromPythonVersion(pythonVersion);
+        }
+      }
+    }
+    return LanguageLevel.getDefault();
+  }
 }

python/testData/psi/WithStatement26.py

@@ -0,0 +1,2 @@
+with x as y:
+  pass    

python/testData/psi/WithStatement26.txt

@@ -0,0 +1,16 @@
+PyFile:WithStatement26.py
+  PyWithStatement
+    PsiElement(Py:WITH_KEYWORD)('with')
+    PsiWhiteSpace(' ')
+    PyReferenceExpression: x
+      PsiElement(Py:IDENTIFIER)('x')
+    PsiWhiteSpace(' ')
+    PsiElement(Py:AS_KEYWORD)('as')
+    PsiWhiteSpace(' ')
+    PyTargetExpression: y
+      PsiElement(Py:IDENTIFIER)('y')
+    PsiElement(Py:COLON)(':')
+    PsiWhiteSpace('\n  ')
+    PyStatementList
+      PyPassStatement
+        PsiElement(Py:PASS_KEYWORD)('pass')

python/testSrc/com/jetbrains/python/PythonParsingTest.java

@@ -2,6 +2,8 @@ package com.jetbrains.python;
 
 import com.intellij.testFramework.ParsingTestCase;
 import com.jetbrains.python.fixtures.PyLightFixtureTestCase;
+import com.jetbrains.python.psi.LanguageLevel;
+import com.jetbrains.python.psi.impl.PythonLanguageLevelPusher;
 
 /**
  * @author yole
@@ -17,80 +19,99 @@ public class PythonParsingTest extends ParsingTestCase {
   }
 
   public void testHelloWorld() throws Exception {
-    doTest(true);
+    doTest();
   }
 
   public void testIfStatement() throws Exception {
-    doTest(true);
+    doTest();
   }
 
   public void testConditionalExpression() throws Exception {
-    doTest(true);
+    doTest();
   }
 
   public void testSubscribedAssignmentLHS() throws Exception {
-    doTest(true);
+    doTest();
   }
 
   public void testConditionalParenLambda() throws Exception {
-    doTest(true);
+    doTest();
   }
 
   public void testLambdaComprehension() throws Exception {
-    doTest(true);
+    doTest();
   }
 
   public void testLambdaConditional() throws Exception {
-    doTest(true);
+    doTest();
   }
 
   public void testTryExceptFinally() throws Exception {
-    doTest(true);
+    doTest();
   }
 
   public void testTryFinally() throws Exception {
-    doTest(true);
+    doTest();
   }
 
   public void testYieldStatement() throws Exception {
-    doTest(true);
+    doTest();
   }
 
   public void testYieldInAssignment() throws Exception {
-    doTest(true);
+    doTest();
   }
 
   public void testYieldInAugAssignment() throws Exception {
-    doTest(true);
+    doTest();
   }
 
   public void testYieldInParentheses() throws Exception {
-    doTest(true);
+    doTest();
   }
 
   public void _testYieldAsArgument() throws Exception {
     // this is a strange case: PEP 342 says this syntax is valid, but
     // Python 2.5 doesn't accept it. let's stick with Python behavior for now
-    doTest(true);
+    doTest();
   }
 
   public void testWithStatement() throws Exception {
-    doTest(true);
+    doTest();
   }
-  
+
   public void testWithStatement2() throws Exception {
-    doTest(true);
+    doTest();
   }
 
   public void testImportStmt() throws Exception {
-    doTest(true);
+    doTest();
   }
 
   public void testDecoratedFunction() throws Exception {
-    doTest(true);
+    doTest();
   }
 
   public void testTryExceptAs() throws Exception {   // PY-293
-    doTest(true);
+    doTest();
+  }
+
+  public void testWithStatement26() throws Exception {
+    doTest(LanguageLevel.PYTHON26);    
+  }
+
+  public void doTest() throws Exception {
+    doTest(LanguageLevel.PYTHON25);
+  }
+
+  public void doTest(LanguageLevel languageLevel) throws Exception {
+    PythonLanguageLevelPusher.FORCE_LANGUAGE_LEVEL = languageLevel;
+    PythonLanguageLevelPusher.pushLanguageLevel(getProject());
+    try {
+      doTest(true);
+    }
+    finally {
+      PythonLanguageLevelPusher.FORCE_LANGUAGE_LEVEL = null;
+    }
   }
 }