initial work in progress support for Py3K keyword-only arguments

python/src/com/jetbrains/python/PyBundle.properties

@@ -146,8 +146,9 @@ ANN.$0.both.global.and.param=Name ''{0}'' used both as a parameter and as a glob
 ANN.$0.assigned.before.global.decl=Name ''{0}'' is assigned before global declaration
 
 ANN.duplicate.param.name=duplicate parameter name
-ANN.starred.param.after.kwparam=* parameter after ** paremeter
-ANN.regular.param.after.starred=regular parameter after * or ** parameter
+ANN.starred.param.after.kwparam=* parameter after ** parameter
+ANN.regular.param.after.vararg=regular parameter after * parameter
+ANN.regular.param.after.keyword=regular parameter after ** parameter
 ANN.non.default.param.after.default=non-default parameter follows default parameter
 
 ANN.star.import.at.top.only='import *' only allowed at module level

python/src/com/jetbrains/python/PyElementTypes.java

@@ -5,7 +5,6 @@ import com.jetbrains.python.psi.*;
 import com.jetbrains.python.psi.impl.*;
 import com.jetbrains.python.psi.impl.stubs.*;
 import com.jetbrains.python.psi.stubs.*;
-import com.jetbrains.python.psi.PyDecorator;
 
 public interface PyElementTypes {
 
@@ -19,8 +18,9 @@ public interface PyElementTypes {
 
   PyStubElementType<PyNamedParameterStub, PyNamedParameter> NAMED_PARAMETER = new PyNamedParameterElementType();
   PyStubElementType<PyTupleParameterStub, PyTupleParameter> TUPLE_PARAMETER = new PyTupleParameterElementType();
+  PyStubElementType<PySingleStarParameterStub, PySingleStarParameter> SINGLE_STAR_PARAMETER = new PySingleStarParameterElementType();
 
-  TokenSet PARAMETERS = TokenSet.create(NAMED_PARAMETER, TUPLE_PARAMETER);
+  TokenSet PARAMETERS = TokenSet.create(NAMED_PARAMETER, TUPLE_PARAMETER, SINGLE_STAR_PARAMETER);
 
   PyStubElementType<PyDecoratorStub, PyDecorator> DECORATOR_CALL = new PyDecoratorCallElementType();
 

python/src/com/jetbrains/python/PyParameterInfoHandler.java

@@ -7,12 +7,13 @@ import com.intellij.psi.PsiFile;
 import com.intellij.util.ArrayUtil;
 import com.intellij.util.text.CharArrayUtil;
 import com.jetbrains.python.psi.*;
-import static com.jetbrains.python.psi.PyCallExpression.PyMarkedFunction;
 import com.jetbrains.python.psi.impl.ParamHelper;
 import org.jetbrains.annotations.NotNull;
 
 import java.util.*;
 
+import static com.jetbrains.python.psi.PyCallExpression.PyMarkedFunction;
+
 /**
  * @author yole
  */
@@ -131,6 +132,12 @@ public class PyParameterInfoHandler implements ParameterInfoHandler<PyArgumentLi
           hint_flags.put(hint_index, EnumSet.noneOf(ParameterInfoUIContextEx.Flag.class));
           hint_texts.add(strb.toString());
         }
+
+        public void visitSingleStarParameter(PySingleStarParameter param, boolean first, boolean last) {
+          hint_flags.put(hint_texts.size(), EnumSet.noneOf(ParameterInfoUIContextEx.Flag.class));
+          if (last) hint_texts.add("*");
+          else hint_texts.add("*,");
+        }
       }
     );
 

python/src/com/jetbrains/python/parsing/FunctionParsing.java

@@ -111,7 +111,16 @@ public class FunctionParsing extends Parsing {
 
       final PsiBuilder.Marker parameter = myBuilder.mark();
       boolean isStarParameter = false;
-      if (myBuilder.getTokenType() == PyTokenTypes.MULT || myBuilder.getTokenType() == PyTokenTypes.EXP) {
+      if (myBuilder.getTokenType() == PyTokenTypes.MULT) {
+        myBuilder.advanceLexer();
+        if (myContext.getLanguageLevel().isPy3K() &&
+            (myBuilder.getTokenType() == PyTokenTypes.COMMA) || myBuilder.getTokenType() == endToken) {
+          parameter.done(PyElementTypes.SINGLE_STAR_PARAMETER);
+          continue;
+        }
+        isStarParameter = true;
+      }
+      else if (myBuilder.getTokenType() == PyTokenTypes.EXP) {
         myBuilder.advanceLexer();
         isStarParameter = true;
       }

python/src/com/jetbrains/python/psi/LanguageLevel.java

@@ -10,7 +10,10 @@ import org.jetbrains.annotations.NotNull;
  * @author yole
  */
 public enum LanguageLevel {
-  PYTHON24(false, true), PYTHON25(false, true), PYTHON26(true, true), PYTHON30(true, false);
+  PYTHON24(false, true, false),
+  PYTHON25(false, true, false),
+  PYTHON26(true, true, false),
+  PYTHON30(true, false, true);
 
   public static LanguageLevel getDefault() {
     return PYTHON26;
@@ -18,10 +21,12 @@ public enum LanguageLevel {
 
   private boolean myHasWithStatement;
   private boolean myHasPrintStatement;
+  private boolean myIsPy3K;
 
-  LanguageLevel(boolean hasWithStatement, boolean hasPrintStatement) {
+  LanguageLevel(boolean hasWithStatement, boolean hasPrintStatement, boolean isPy3K) {
     myHasWithStatement = hasWithStatement;
     myHasPrintStatement = hasPrintStatement;
+    myIsPy3K = isPy3K;
   }
 
   public boolean hasWithStatement() {
@@ -32,6 +37,10 @@ public enum LanguageLevel {
     return myHasPrintStatement;
   }
 
+  public boolean isPy3K() {
+    return myIsPy3K;
+  }
+
   public static LanguageLevel fromPythonVersion(String pythonVersion) {
     if (pythonVersion.startsWith("2.6")) {
       return PYTHON26;

python/src/com/jetbrains/python/psi/PySingleStarParameter.java

@@ -0,0 +1,13 @@
+package com.jetbrains.python.psi;
+
+import com.intellij.psi.StubBasedPsiElement;
+import com.jetbrains.python.psi.stubs.PySingleStarParameterStub;
+
+/**
+ * Represents a single star (keyword-only parameter delimiter) appearing in the
+ * parameter list of a Py3K function.
+ *
+ * @author yole
+ */
+public interface PySingleStarParameter extends PyParameter, StubBasedPsiElement<PySingleStarParameterStub> {
+}

python/src/com/jetbrains/python/psi/impl/ParamHelper.java

@@ -1,9 +1,6 @@
 package com.jetbrains.python.psi.impl;
 
-import com.jetbrains.python.psi.PyNamedParameter;
-import com.jetbrains.python.psi.PyParameter;
-import com.jetbrains.python.psi.PyParameterList;
-import com.jetbrains.python.psi.PyTupleParameter;
+import com.jetbrains.python.psi.*;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -31,12 +28,20 @@ public class ParamHelper {
         walkDownParamArray(nested_params, walker);
         walker.leaveTupleParameter(tpar, (i==0), (i == last));
       }
-      else walker.visitNamedParameter(param.getAsNamed(), (i==0), (i == last));
+      else {
+        final PyNamedParameter namedParameter = param.getAsNamed();
+        if (namedParameter != null) {
+          walker.visitNamedParameter(namedParameter, (i==0), (i == last));
+        }
+        else {
+          walker.visitSingleStarParameter((PySingleStarParameter) param, (i == 0), (i == last));
+        }
+      }
       i += 1;
     }
   }
 
-  public static interface ParamWalker {
+  public interface ParamWalker {
     /**
      * Is called when a tuple parameter is encountered, before visiting any parameters nested in it.
      * @param param the parameter
@@ -60,6 +65,8 @@ public class ParamHelper {
      * @param last true it is the last in the list
      */
     void visitNamedParameter(PyNamedParameter param, boolean first, boolean last);
+
+    void visitSingleStarParameter(PySingleStarParameter param, boolean first, boolean last);
   }
 
   public static abstract class ParamVisitor implements ParamWalker {
@@ -68,6 +75,8 @@ public class ParamHelper {
     public void leaveTupleParameter(PyTupleParameter param, boolean first, boolean last) { }
 
     public void visitNamedParameter(PyNamedParameter param, boolean first, boolean last) { }
+
+    public void visitSingleStarParameter(PySingleStarParameter param, boolean first, boolean last) { }
   }
 
   public static StringBuilder appendParameterList(PyParameterList plist, final StringBuilder target) {
@@ -89,6 +98,11 @@ public class ParamHelper {
           target.append(param.getRepr(true));
           if (! last) target.append(COMMA);
         }
+
+        public void visitSingleStarParameter(PySingleStarParameter param, boolean first, boolean last) {
+          target.append('*');
+          if (!last) target.append(COMMA);
+        }
       }
     );
     target.append(")");
@@ -99,11 +113,7 @@ public class ParamHelper {
     final List<PyNamedParameter> result = new ArrayList<PyNamedParameter>(10); // a random 'enough'
     walkDownParamArray(
       plist.getParameters(),
-      new ParamWalker() {
-        public void enterTupleParameter(PyTupleParameter param, boolean first, boolean last) { }
-
-        public void leaveTupleParameter(PyTupleParameter param, boolean first, boolean last) { }
-
+      new ParamVisitor() {
         public void visitNamedParameter(PyNamedParameter param, boolean first, boolean last) {
           result.add(param);
         }

python/src/com/jetbrains/python/psi/impl/PyArgumentListImpl.java

@@ -312,8 +312,12 @@ public class PyArgumentListImpl extends PyElementImpl implements PyArgumentList
         ListIterator<PyExpression> unmatched_arg_iter = unmatched_args.listIterator();
         // check positional args
         while (unmatched_arg_iter.hasNext() && (param_index < params.length)) {
-          final PyExpression arg = unmatched_arg_iter.next(); // current arg
           PyParameter a_param = params[param_index];      // its matching param
+          if (a_param instanceof PySingleStarParameter) {
+            param_index++;
+            continue;
+          }
+          final PyExpression arg = unmatched_arg_iter.next(); // current arg
           PyNamedParameter n_param = a_param.getAsNamed();
           if (n_param != null) { // named
             if (
@@ -329,10 +333,13 @@ public class PyArgumentListImpl extends PyElementImpl implements PyArgumentList
             ret.my_plain_mapped_params.put(arg, n_param);
           }
           else { // tuple: it may contain only positionals or other tuples.
-            unmatched_arg_iter.previous(); // step back so that the visitor takes this arg again
-            MyParamVisitor visitor = new MyParamVisitor(unmatched_arg_iter, ret);
-            visitor.enterTuple(a_param.getAsTuple()); // will recurse as needed
-            unmatched_subargs.addAll(visitor.getUnmatchedSubargs()); // what it's seen 
+            PyTupleParameter tupleParameter = a_param.getAsTuple();
+            if (tupleParameter != null) {
+              unmatched_arg_iter.previous(); // step back so that the visitor takes this arg again
+              MyParamVisitor visitor = new MyParamVisitor(unmatched_arg_iter, ret);
+              visitor.enterTuple(a_param.getAsTuple()); // will recurse as needed
+              unmatched_subargs.addAll(visitor.getUnmatchedSubargs()); // what it's seen
+            }
           }
           unmatched_arg_iter.remove(); // it has been matched
           param_index += 1;

python/src/com/jetbrains/python/psi/impl/PySingleStarParameterImpl.java

@@ -0,0 +1,37 @@
+package com.jetbrains.python.psi.impl;
+
+import com.intellij.lang.ASTNode;
+import com.intellij.psi.PsiElement;
+import com.intellij.util.IncorrectOperationException;
+import com.jetbrains.python.PyElementTypes;
+import com.jetbrains.python.psi.PyNamedParameter;
+import com.jetbrains.python.psi.PySingleStarParameter;
+import com.jetbrains.python.psi.PyTupleParameter;
+import com.jetbrains.python.psi.stubs.PySingleStarParameterStub;
+import org.jetbrains.annotations.NonNls;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * @author yole
+ */
+public class PySingleStarParameterImpl extends PyPresentableElementImpl<PySingleStarParameterStub> implements PySingleStarParameter {
+  public PySingleStarParameterImpl(ASTNode astNode) {
+    super(astNode);
+  }
+
+  public PySingleStarParameterImpl(PySingleStarParameterStub stub) {
+    super(stub, PyElementTypes.SINGLE_STAR_PARAMETER);
+  }
+
+  public PsiElement setName(@NonNls @NotNull String name) throws IncorrectOperationException {
+    throw new UnsupportedOperationException();
+  }
+
+  public PyNamedParameter getAsNamed() {
+    return null;
+  }
+
+  public PyTupleParameter getAsTuple() {
+    return null;
+  }
+}

python/src/com/jetbrains/python/psi/impl/stubs/PySingleStarParameterElementType.java

@@ -0,0 +1,44 @@
+package com.jetbrains.python.psi.impl.stubs;
+
+import com.intellij.lang.ASTNode;
+import com.intellij.psi.PsiElement;
+import com.intellij.psi.stubs.StubElement;
+import com.intellij.psi.stubs.StubInputStream;
+import com.intellij.psi.stubs.StubOutputStream;
+import com.jetbrains.python.psi.PySingleStarParameter;
+import com.jetbrains.python.psi.PyStubElementType;
+import com.jetbrains.python.psi.impl.PySingleStarParameterImpl;
+import com.jetbrains.python.psi.stubs.PySingleStarParameterStub;
+
+import java.io.IOException;
+
+/**
+ * @author yole
+ */
+public class PySingleStarParameterElementType extends PyStubElementType<PySingleStarParameterStub, PySingleStarParameter> {
+  public PySingleStarParameterElementType() {
+    super("SINGLE_STAR_PARAMETER");
+  }
+
+  @Override
+  public PsiElement createElement(ASTNode node) {
+    return new PySingleStarParameterImpl(node);
+  }
+
+  @Override
+  public PySingleStarParameter createPsi(PySingleStarParameterStub stub) {
+    return new PySingleStarParameterImpl(stub);
+  }
+
+  @Override
+  public PySingleStarParameterStub createStub(PySingleStarParameter psi, StubElement parentStub) {
+    return new PySingleStarParameterStubImpl(parentStub);
+  }
+
+  public void serialize(PySingleStarParameterStub stub, StubOutputStream dataStream) throws IOException {
+  }
+
+  public PySingleStarParameterStub deserialize(StubInputStream dataStream, StubElement parentStub) throws IOException {
+    return new PySingleStarParameterStubImpl(parentStub);
+  }
+}

python/src/com/jetbrains/python/psi/impl/stubs/PySingleStarParameterStubImpl.java

@@ -0,0 +1,16 @@
+package com.jetbrains.python.psi.impl.stubs;
+
+import com.intellij.psi.stubs.StubBase;
+import com.intellij.psi.stubs.StubElement;
+import com.jetbrains.python.PyElementTypes;
+import com.jetbrains.python.psi.PySingleStarParameter;
+import com.jetbrains.python.psi.stubs.PySingleStarParameterStub;
+
+/**
+ * @author yole
+ */
+public class PySingleStarParameterStubImpl extends StubBase<PySingleStarParameter> implements PySingleStarParameterStub {
+  protected PySingleStarParameterStubImpl(final StubElement parent) {
+    super(parent, PyElementTypes.SINGLE_STAR_PARAMETER);
+  }
+}

python/src/com/jetbrains/python/psi/stubs/PySingleStarParameterStub.java

@@ -0,0 +1,10 @@
+package com.jetbrains.python.psi.stubs;
+
+import com.intellij.psi.stubs.StubElement;
+import com.jetbrains.python.psi.PySingleStarParameter;
+
+/**
+ * @author yole
+ */
+public interface PySingleStarParameterStub extends StubElement<PySingleStarParameter> {
+}

python/src/com/jetbrains/python/validation/ParameterListAnnotator.java

@@ -1,25 +1,8 @@
-/*
- *  Copyright 2005 Pythonid Project
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS"; BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
 package com.jetbrains.python.validation;
 
 import com.intellij.util.containers.HashSet;
 import com.jetbrains.python.PyBundle;
-import com.jetbrains.python.psi.PyNamedParameter;
-import com.jetbrains.python.psi.PyParameterList;
+import com.jetbrains.python.psi.*;
 import com.jetbrains.python.psi.impl.ParamHelper;
 
 import java.util.Set;
@@ -30,12 +13,15 @@ import java.util.Set;
 public class ParameterListAnnotator extends PyAnnotator {
   @Override
   public void visitPyParameterList(final PyParameterList paramlist) {
+    final LanguageLevel languageLevel = ((PyFile)paramlist.getContainingFile()).getLanguageLevel();
     ParamHelper.walkDownParamArray(
       paramlist.getParameters(),
       new ParamHelper.ParamVisitor() {
         Set<String> parameterNames = new HashSet<String>();
-        boolean hadPositionalContainer = false, hadKeywordContainer = false;
+        boolean hadPositionalContainer = false;
+        boolean hadKeywordContainer = false;
         boolean hadDefaultValue = false;
+        boolean hadSingleStar = false;
         @Override
         public void visitNamedParameter(PyNamedParameter parameter, boolean first, boolean last) {
           if (parameterNames.contains(parameter.getName())) {
@@ -52,19 +38,27 @@ public class ParameterListAnnotator extends PyAnnotator {
             hadKeywordContainer = true;
           }
           else {
-            if (hadPositionalContainer || hadKeywordContainer) {
-              getHolder().createErrorAnnotation(parameter, PyBundle.message("ANN.regular.param.after.starred"));
+            if (hadPositionalContainer && !languageLevel.isPy3K()) {
+              getHolder().createErrorAnnotation(parameter, PyBundle.message("ANN.regular.param.after.vararg"));
+            }
+            else if (hadKeywordContainer) {
+              getHolder().createErrorAnnotation(parameter, PyBundle.message("ANN.regular.param.after.keyword"));
             }
             if (parameter.getDefaultValue() != null) {
               hadDefaultValue = true;
             }
             else {
-              if (hadDefaultValue) {
+              if (hadDefaultValue && !hadSingleStar && (!languageLevel.isPy3K() || !hadPositionalContainer)) {
                 getHolder().createErrorAnnotation(parameter, PyBundle.message("ANN.non.default.param.after.default"));
               }
             }
           }
         }
+
+        @Override
+        public void visitSingleStarParameter(PySingleStarParameter param, boolean first, boolean last) {
+          hadSingleStar = true;
+        }
       }
     );
   }

python/testData/highlighting/keywordOnlyArguments.py

@@ -0,0 +1,2 @@
+def keywordonly_sum(*, k1=0, k2):
+    return k1 + k2

python/testData/highlighting/regularAfterVarArgs.py

@@ -0,0 +1,2 @@
+def sortwords(*wordlist, case_sensitive=False): pass
+def mixedargs_sum(a, b=0, *arg, k1, k2=0): pass

python/testData/psi/KeywordOnlyArgument.py

@@ -0,0 +1,2 @@
+def compare(a, b, *, key=None): pass
+def keywordonly_sum(*, k1=0, k2): pass

python/testData/psi/KeywordOnlyArgument.txt

@@ -0,0 +1,30 @@
+PyFile:KeywordOnlyArgument.py
+  PyFunction('compare')
+    PsiElement(Py:DEF_KEYWORD)('def')
+    PsiWhiteSpace(' ')
+    PsiElement(Py:IDENTIFIER)('compare')
+    PyParameterList
+      PsiElement(Py:LPAR)('(')
+      PyNamedParameter('a')
+        PsiElement(Py:IDENTIFIER)('a')
+      PsiElement(Py:COMMA)(',')
+      PsiWhiteSpace(' ')
+      PyNamedParameter('b')
+        PsiElement(Py:IDENTIFIER)('b')
+      PsiElement(Py:COMMA)(',')
+      PsiWhiteSpace(' ')
+      PySingleStarParameter
+        PsiElement(Py:MULT)('*')
+      PsiElement(Py:COMMA)(',')
+      PsiWhiteSpace(' ')
+      PyNamedParameter('key')
+        PsiElement(Py:IDENTIFIER)('key')
+        PsiElement(Py:EQ)('=')
+        PyReferenceExpression: None
+          PsiElement(Py:IDENTIFIER)('None')
+      PsiElement(Py:RPAR)(')')
+    PsiElement(Py:COLON)(':')
+    PsiWhiteSpace(' ')
+    PyStatementList
+      PyPassStatement
+        PsiElement(Py:PASS_KEYWORD)('pass')

python/testSrc/com/jetbrains/python/PythonHighlightingTest.java

@@ -93,17 +93,17 @@ public class PythonHighlightingTest extends PyLightFixtureTestCase {
   }
 
   public void testStringBytesLiteralOK() throws Exception {
-    PythonLanguageLevelPusher.FORCE_LANGUAGE_LEVEL = LanguageLevel.PYTHON26;
-    PythonLanguageLevelPusher.pushLanguageLevel(myFixture.getProject());
-    try {
-      doTest();
-    }
-    finally {
-      PythonLanguageLevelPusher.FORCE_LANGUAGE_LEVEL = null;
-    }
+    doTest(LanguageLevel.PYTHON26, true, true);
   }
 
+  public void testRegularAfterVarArgs() throws Exception {
+    doTest(LanguageLevel.PYTHON30, true, false);
+  }
 
+  public void testKeywordOnlyArguments() throws Exception {
+    doTest(LanguageLevel.PYTHON30, true, false);
+  }
+  
   public void testMalformedStringTripleQuoteUnterminated() throws Exception {
     doTest();
   }
@@ -122,6 +122,17 @@ public class PythonHighlightingTest extends PyLightFixtureTestCase {
     doTest();
   }
 
+  private void doTest(final LanguageLevel languageLevel, final boolean checkWarnings, final boolean checkInfos) throws Exception {
+    PythonLanguageLevelPusher.FORCE_LANGUAGE_LEVEL = languageLevel;
+    PythonLanguageLevelPusher.pushLanguageLevel(myFixture.getProject());
+    try {
+      doTest(checkWarnings, checkInfos);
+    }
+    finally {
+      PythonLanguageLevelPusher.FORCE_LANGUAGE_LEVEL = null;
+    }
+  }
+
   private void doTest() throws Exception {
     myFixture.testHighlighting(true, true, false, getTestName(true) + PyNames.DOT_PY);
   }

python/testSrc/com/jetbrains/python/PythonInspectionsTest.java

@@ -4,6 +4,8 @@ import com.intellij.codeInspection.LocalInspectionTool;
 import com.intellij.codeInspection.ex.LocalInspectionToolWrapper;
 import com.jetbrains.python.fixtures.PyLightFixtureTestCase;
 import com.jetbrains.python.inspections.*;
+import com.jetbrains.python.psi.LanguageLevel;
+import com.jetbrains.python.psi.impl.PythonLanguageLevelPusher;
 
 /**
  * @author yole
@@ -38,6 +40,18 @@ public class PythonInspectionsTest extends PyLightFixtureTestCase {
     doTest(getTestName(false), inspection);
   }
 
+  public void testPyArgumentListInspection3K() throws Throwable {
+    PythonLanguageLevelPusher.FORCE_LANGUAGE_LEVEL = LanguageLevel.PYTHON30;
+    PythonLanguageLevelPusher.pushLanguageLevel(myFixture.getProject());
+    try {
+      LocalInspectionTool inspection = new PyArgumentListInspection();
+      doTest(getTestName(false), inspection);
+    }
+    finally {
+      PythonLanguageLevelPusher.FORCE_LANGUAGE_LEVEL = null;
+    }
+  }
+
   public void testPyRedeclarationInspection() throws Throwable {
     LocalInspectionTool inspection = new PyRedeclarationInspection();
     doTest(getTestName(false), inspection);

python/testSrc/com/jetbrains/python/PythonParsingTest.java

@@ -116,6 +116,10 @@ public class PythonParsingTest extends ParsingTestCase {
     doTest();
   }
 
+  public void testKeywordOnlyArgument() throws Exception {   // PEP 3102
+    doTest(LanguageLevel.PYTHON30);
+  }
+
   public void doTest() throws Exception {
     doTest(LanguageLevel.PYTHON25);
   }