Support 'yield from' syntax (PY-6702)

python/src/com/jetbrains/python/parsing/ExpressionParsing.java

@@ -531,6 +531,9 @@ public class ExpressionParsing extends Parsing {
     if (myBuilder.getTokenType() == PyTokenTypes.YIELD_KEYWORD) {
       PsiBuilder.Marker yieldExpr = myBuilder.mark();
       myBuilder.advanceLexer();
+      if (myBuilder.getTokenType() == PyTokenTypes.FROM_KEYWORD) {
+        myBuilder.advanceLexer();
+      }
       parseTupleExpression(false, isTargetExpression, false);
       yieldExpr.done(PyElementTypes.YIELD_EXPRESSION);
       return true;

python/src/com/jetbrains/python/psi/PyYieldExpression.java

@@ -8,4 +8,6 @@ import org.jetbrains.annotations.Nullable;
 public interface PyYieldExpression extends PyExpression {
   @Nullable
   PyExpression getExpression();
+
+  boolean isDelegating();
 }

python/src/com/jetbrains/python/psi/impl/PyYieldExpressionImpl.java

@@ -1,8 +1,8 @@
 package com.jetbrains.python.psi.impl;
 
 import com.intellij.lang.ASTNode;
-import com.jetbrains.python.PyElementTypes;
-import com.jetbrains.python.PythonDialectsTokenSetProvider;
+import com.intellij.psi.util.PsiTreeUtil;
+import com.jetbrains.python.PyTokenTypes;
 import com.jetbrains.python.psi.PyElementVisitor;
 import com.jetbrains.python.psi.PyExpression;
 import com.jetbrains.python.psi.PyYieldExpression;
@@ -24,7 +24,13 @@ public class PyYieldExpressionImpl extends PyElementImpl implements PyYieldExpre
 
   @Override
   public PyExpression getExpression() {
-    return childToPsi(PythonDialectsTokenSetProvider.INSTANCE.getExpressionTokens(), 0);
+    final PyExpression[] expressions = PsiTreeUtil.getChildrenOfType(this, PyExpression.class);
+    return (expressions != null && expressions.length > 0) ? expressions[0] : null;
+  }
+
+  @Override
+  public boolean isDelegating() {
+    return getNode().findChildByType(PyTokenTypes.FROM_KEYWORD) != null;
   }
 
   @Override

python/src/com/jetbrains/python/validation/CompatibilityVisitor.java

@@ -460,6 +460,20 @@ public abstract class CompatibilityVisitor extends PyAnnotator {
                           len, node, null);
   }
 
+  @Override
+  public void visitPyYieldExpression(PyYieldExpression node) {
+    super.visitPyYieldExpression(node);
+    if (!node.isDelegating()) {
+      return;
+    }
+    for (LanguageLevel level : myVersionsToProcess) {
+      if (level.isOlderThan(LanguageLevel.PYTHON33)) {
+        registerProblem(node, "Python versions < 3.3 do not support this syntax. Delegating to a subgenerator is available since " +
+                              "Python 3.3; use explicit iteration over subgenerator instead.");
+      }
+    }
+  }
+
   private boolean shouldBeCompatibleWithPy3() {
     if (myVersionsToProcess.contains(LanguageLevel.PYTHON30) || myVersionsToProcess.contains(LanguageLevel.PYTHON31)
         || myVersionsToProcess.contains(LanguageLevel.PYTHON32))

python/testData/highlighting/yieldFromBefore33.py

@@ -0,0 +1,2 @@
+def f(g):
+    <error descr="Python versions < 3.3 do not support this syntax. Delegating to a subgenerator is available since Python 3.3; use explicit iteration over subgenerator instead.">yield from g()</error>

python/testData/psi/YieldFrom.py

@@ -0,0 +1,2 @@
+def f(g):
+    yield from g()

python/testData/psi/YieldFrom.txt

@@ -0,0 +1,25 @@
+PyFile:YieldFrom.py
+  PyFunction('f')
+    PsiElement(Py:DEF_KEYWORD)('def')
+    PsiWhiteSpace(' ')
+    PsiElement(Py:IDENTIFIER)('f')
+    PyParameterList
+      PsiElement(Py:LPAR)('(')
+      PyNamedParameter('g')
+        PsiElement(Py:IDENTIFIER)('g')
+      PsiElement(Py:RPAR)(')')
+    PsiElement(Py:COLON)(':')
+    PsiWhiteSpace('\n    ')
+    PyStatementList
+      PyExpressionStatement
+        PyYieldExpression
+          PsiElement(Py:YIELD_KEYWORD)('yield')
+          PsiWhiteSpace(' ')
+          PsiElement(Py:FROM_KEYWORD)('from')
+          PsiWhiteSpace(' ')
+          PyCallExpression: g
+            PyReferenceExpression: g
+              PsiElement(Py:IDENTIFIER)('g')
+            PyArgumentList
+              PsiElement(Py:LPAR)('(')
+              PsiElement(Py:RPAR)(')')

python/testSrc/com/jetbrains/python/PythonHighlightingTest.java

@@ -152,6 +152,11 @@ public class PythonHighlightingTest extends PyTestCase {
     doTest(LanguageLevel.PYTHON33, true, false);
   }
 
+  // PY-6702
+  public void testYieldFromBefore33() {
+    doTest(LanguageLevel.PYTHON32, true, false);
+  }
+
   public void testYieldInNestedFunction() {
     // highlight func declaration first, lest we get an "Extra fragment highlighted" error.
     EditorColorsManager manager = EditorColorsManager.getInstance();

python/testSrc/com/jetbrains/python/PythonParsingTest.java

@@ -318,6 +318,11 @@ public class PythonParsingTest extends ParsingTestCase {
     doTest();
   }
 
+  // PY-6702
+  public void testYieldFrom() {
+    doTest(LanguageLevel.PYTHON33);
+  }
+
   public void doTest() {
     doTest(LanguageLevel.PYTHON25);
   }