PY-14844 Add integer suffix support for Cython

Cython supports C-style integer suffix (u, l, ll). I added them to Python lexer and annotator checker to highlight them in Python language.

More information:

- https://cython.readthedocs.io/en/latest/src/userguide/language_basics.html#differences-between-c-and-cython-expressions

- https://en.cppreference.com/w/cpp/language/integer_literal

GitOrigin-RevId: 97d7bcb19239f931d9ed5e5746aaed84ac09cbc8

python/pluginResources/messages/PyBundle.properties

@@ -226,7 +226,7 @@ INTN.replace.noteq.operator=Replace not equal operator
 INTN.remove.leading.$0=Remove leading {0}
 INTN.remove.leading.prefix=Remove prefix
 
-INTN.remove.trailing.l=Remove trailing L
+INTN.remove.trailing.suffix=Remove trailing suffix
 
 INTN.replace.list.comprehensions=Convert list comprehensions to supported form
 
@@ -636,6 +636,9 @@ INSP.NAME.coroutine.is.not.awaited=Coroutine ''{0}'' is not awaited
 # PyTestParametrizedInspection
 INSP.NAME.pytest-parametrized=Checks that functions decorated by pytest parametrize have correct arguments
 
+# PyHighlightingAnnotator
+INSP.python.trailing.suffix.not.support=Python does not support a trailing ''{0}''
+
 INSP.abstract.class.set.as.metaclass=Set ''{0}'' as metaclass
 INSP.abstract.class.add.to.superclasses=Add ''{0}'' to superclasses
 INSP.compatibility.this.syntax.available.only.since.py3=This syntax available only since py3

python/python-psi-api/src/com/jetbrains/python/psi/PyNumericLiteralExpression.java

@@ -43,4 +43,12 @@ public interface PyNumericLiteralExpression extends PyLiteralExpression {
   BigDecimal getBigDecimalValue();
 
   boolean isIntegerLiteral();
+
+  /**
+   * Returns possible suffix of integer literal like {@code uL}.
+   *
+   * @return null if this is not integer literal or the suffix is missed; string suffix otherwise
+   */
+  @Nullable
+  String getIntegerLiteralSuffix();
 }

python/python-psi-impl/src/com/jetbrains/python/lexer/Python.flex

@@ -24,7 +24,8 @@ OCTINTEGER = 0[Oo]?("_"?{OCTDIGIT})+
 BININTEGER = 0[Bb]("_"?{BINDIGIT})+
 DECIMALINTEGER = (({NONZERODIGIT}("_"?{DIGIT})*)|0)
 INTEGER = {DECIMALINTEGER}|{OCTINTEGER}|{HEXINTEGER}|{BININTEGER}
-LONGINTEGER = {INTEGER}[Ll]
+INTEGER_SUFFIX = u|l|ll|U|L|LL|ul|ull|lu|llu|uL|Ul|UL|uLL|Ull|ULL|lU|Lu|LU|llU|LLu|LLU
+LONGINTEGER = {INTEGER}{INTEGER_SUFFIX}
 
 END_OF_LINE_COMMENT="#"[^\r\n]*
 

python/python-psi-impl/src/com/jetbrains/python/psi/impl/PyNumericLiteralExpressionImpl.java

@@ -87,6 +87,12 @@ public class PyNumericLiteralExpressionImpl extends PyElementImpl implements PyN
     return getNode().getElementType() == PyElementTypes.INTEGER_LITERAL_EXPRESSION;
   }
 
+  @Nullable
+  @Override
+  public String getIntegerLiteralSuffix() {
+    return isIntegerLiteral() ? StringUtil.nullize(retrieveSuffix(getText())) : null;
+  }
+
   @Override
   @Nullable
   public PyType getType(@NotNull TypeEvalContext context, @NotNull TypeEvalContext.Key key) {
@@ -107,7 +113,7 @@ public class PyNumericLiteralExpressionImpl extends PyElementImpl implements PyN
 
   @Nullable
   private static BigInteger getBigIntegerValue(@NotNull String text) {
-    if (text.equals("0") || text.equalsIgnoreCase("0l")) {
+    if (text.equalsIgnoreCase("0" + retrieveSuffix(text))) {
       return BigInteger.ZERO;
     }
 
@@ -141,9 +147,18 @@ public class PyNumericLiteralExpressionImpl extends PyElementImpl implements PyN
 
   @NotNull
   private static String prepareLiteralForJava(@NotNull String text, int beginIndex) {
-    final int endIndex =
-      StringUtil.endsWithIgnoreCase(text, "l") || StringUtil.endsWithIgnoreCase(text, "j") ? text.length() - 1 : text.length();
-
+    int endIndex = text.length() - retrieveSuffix(text).length();
     return text.substring(beginIndex, endIndex).replaceAll("_", "");
   }
+
+  @NotNull
+  private static String retrieveSuffix(@NotNull String text) {
+    int lastIndex = text.length();
+    while (lastIndex > 0) {
+      char last = text.charAt(lastIndex - 1);
+      if (last != 'u' && last != 'U' && last != 'l' && last != 'L' && last != 'j' && last != 'J') break;
+      --lastIndex;
+    }
+    return text.substring(lastIndex);
+  }
 }

python/src/com/jetbrains/python/inspections/quickfix/RemoveTrailingSuffixQuickFix.java

@@ -29,21 +29,25 @@ import org.jetbrains.annotations.NotNull;
  * Created by IntelliJ IDEA.
  * Author: Alexey.Ivanov
  */
-public class RemoveTrailingLQuickFix implements LocalQuickFix {
+public class RemoveTrailingSuffixQuickFix implements LocalQuickFix {
+
   @NotNull
   @Override
   public String getFamilyName() {
-    return PyBundle.message("INTN.remove.trailing.l");
+    return PyBundle.message("INTN.remove.trailing.suffix");
   }
 
   @Override
   public void applyFix(@NotNull Project project, @NotNull ProblemDescriptor descriptor) {
     PsiElement numericLiteralExpression = descriptor.getPsiElement();
     if (numericLiteralExpression instanceof PyNumericLiteralExpression) {
-      PyElementGenerator elementGenerator = PyElementGenerator.getInstance(project);
-      String text = numericLiteralExpression.getText();
-      final LanguageLevel level = LanguageLevel.forElement(numericLiteralExpression);
-      numericLiteralExpression.replace(elementGenerator.createExpressionFromText(level, text.substring(0, text.length() - 1)));
+      PyNumericLiteralExpression numeric = (PyNumericLiteralExpression)numericLiteralExpression;
+      String suffix = numeric.getIntegerLiteralSuffix();
+      if (suffix == null) return;
+      String text = numeric.getText();
+      String newText = text.substring(0, text.length() - suffix.length());
+      numeric.replace(
+        PyElementGenerator.getInstance(project).createExpressionFromText(LanguageLevel.forElement(numeric), newText));
     }
   }
 }

python/src/com/jetbrains/python/validation/CompatibilityVisitor.java

@@ -216,11 +216,12 @@ public abstract class CompatibilityVisitor extends PyAnnotator {
     final String text = node.getText();
 
     if (node.isIntegerLiteral()) {
-      if (text.endsWith("l") || text.endsWith("L")) {
+      String suffix = node.getIntegerLiteralSuffix();
+      if ("l".equalsIgnoreCase(suffix)) {
         registerForAllMatchingVersions(level -> level.isPy3K() && registerForLanguageLevel(level),
-                                       " not support a trailing \'l\' or \'L\'.",
+                                       " not support a trailing '" + suffix + "'.",
                                        node,
-                                       new RemoveTrailingLQuickFix());
+                                       new RemoveTrailingSuffixQuickFix());
       }
 
       if (text.length() > 1 && text.charAt(0) == '0') {

python/src/com/jetbrains/python/validation/PyHighlightingAnnotator.java

@@ -23,6 +23,7 @@ import com.intellij.psi.PsiFile;
 import com.intellij.psi.tree.TokenSet;
 import com.jetbrains.python.PyBundle;
 import com.jetbrains.python.PyTokenTypes;
+import com.jetbrains.python.PythonLanguage;
 import com.jetbrains.python.highlighting.PyHighlighter;
 import com.jetbrains.python.psi.*;
 import org.jetbrains.annotations.NotNull;
@@ -49,6 +50,15 @@ public class PyHighlightingAnnotator extends PyAnnotator implements HighlightRan
     }
   }
 
+  @Override
+  public void visitPyNumericLiteralExpression(PyNumericLiteralExpression node) {
+    String suffix = node.getIntegerLiteralSuffix();
+    if (suffix == null || "l".equalsIgnoreCase(suffix)) return;
+    if (node.getContainingFile().getLanguage() != PythonLanguage.getInstance()) return;
+    getHolder().newAnnotation(HighlightSeverity.ERROR, PyBundle.message("INSP.python.trailing.suffix.not.support", suffix))
+      .range(node).create();
+  }
+
   @Override
   public void visitPyForStatement(PyForStatement node) {
     highlightKeyword(node, PyTokenTypes.ASYNC_KEYWORD);

python/testData/highlighting/implicitOctLongInteger.py

@@ -1 +1 @@
-<error descr="Python version 3.5 does not support a trailing 'l' or 'L'."><error descr="Python version 3.5 does not support this syntax. It requires '0o' prefix for octal literals">01234L</error></error>
+<error descr="Python version 3.5 does not support a trailing 'L'."><error descr="Python version 3.5 does not support this syntax. It requires '0o' prefix for octal literals">01234L</error></error>

python/testData/highlighting/unsupportedFeaturesInPython3.py

@@ -1,8 +1,13 @@
 print(<error descr="Python version 3.4 does not support <>, use != instead.">a <> 3</error>)
 <error descr="Python version 3.4 does not support backquotes, use repr() instead">`foo()`</error>
-a = <error descr="Python version 3.4 does not support a trailing 'l' or 'L'.">123l</error>
+a = <error descr="Python version 3.4 does not support a trailing 'l'.">123l</error>
 a = <error descr="Python version 3.4 does not support this syntax. It requires '0o' prefix for octal literals">043</error>
 a = 0X43
+a = 0X43
+a = 0x43
+a = 0O43
+a = 0o43
+a = 0B1
 a = 0b1
 a = 0.0
 s = u"text"

python/testData/inspections/PyCompatibilityInspection/numericLiteralExpression.py

@@ -1,2 +1,23 @@
-a = <warning descr="Python version 3.4, 3.5, 3.6, 3.7, 3.8 do not support a trailing 'l' or 'L'.">12l</warning>
-v = <warning descr="Python version 3.4, 3.5, 3.6, 3.7, 3.8 do not support this syntax. It requires '0o' prefix for octal literals">04</warning><error descr="End of statement expected">8</error>
+a = <error descr="Python does not support a trailing 'u'">12u</error>
+b = <warning descr="Python version 3.4, 3.5, 3.6, 3.7, 3.8 do not support a trailing 'l'.">12l</warning>
+c = <error descr="Python does not support a trailing 'll'">12ll</error>
+d = <error descr="Python does not support a trailing 'U'">12U</error>
+e = <warning descr="Python version 3.4, 3.5, 3.6, 3.7, 3.8 do not support a trailing 'L'.">12L</warning>
+f = <error descr="Python does not support a trailing 'LL'">12LL</error>
+g = <error descr="Python does not support a trailing 'ul'">0x12ful</error>
+h = <error descr="Python does not support a trailing 'uL'">0X12fuL</error>
+i = <error descr="Python does not support a trailing 'Ul'">12Ul</error>
+j = <error descr="Python does not support a trailing 'UL'">12UL</error>
+k = <error descr="Python does not support a trailing 'ull'">0o12ull</error>
+l = <error descr="Python does not support a trailing 'uLL'">0O12uLL</error>
+m = <error descr="Python does not support a trailing 'Ull'">0b1Ull</error>
+n = <error descr="Python does not support a trailing 'ULL'">0B1ULL</error>
+o = <error descr="Python does not support a trailing 'lu'">12lu</error>
+p = <error descr="Python does not support a trailing 'lU'">12lU</error>
+q = <error descr="Python does not support a trailing 'Lu'">12Lu</error>
+r = <error descr="Python does not support a trailing 'LU'">12LU</error>
+s = <error descr="Python does not support a trailing 'llu'">12llu</error>
+t = <error descr="Python does not support a trailing 'llU'">12llU</error>
+u = <error descr="Python does not support a trailing 'LLu'">12LLu</error>
+v = <error descr="Python does not support a trailing 'LLU'">12LLU</error>
+w = <warning descr="Python version 3.4, 3.5, 3.6, 3.7, 3.8 do not support this syntax. It requires '0o' prefix for octal literals">04</warning><error descr="End of statement expected">8</error>

python/testData/quickFixes/PyRemoveTrailingSuffixQuickFixTest/fixl.py

@@ -0,0 +1 @@
+<error bundleMsg="bad-trail|l">1000l</error>

python/testData/quickFixes/PyRemoveTrailingSuffixQuickFixTest/fixl_after.py

@@ -0,0 +1 @@
+1000

python/testSrc/com/jetbrains/python/PyNumericLiteralTest.java

@@ -369,6 +369,33 @@ public class PyNumericLiteralTest extends PyTestCase {
     doTestMoreThanLongComplexLiteral("9_22_337_2036_85477_5808J", expectedInt2, new BigDecimal("9223372036854775808"));
   }
 
+  // PY-14844
+  public void testIntegerLiteralSuffix() {
+    doTestSuffix("1000", "u");
+    doTestSuffix("1000", "l");
+    doTestSuffix("1000", "ll");
+    doTestSuffix("1000", "U");
+    doTestSuffix("1000", "L");
+    doTestSuffix("1000", "LL");
+    doTestSuffix("1000", "ul");
+    doTestSuffix("1000", "Ul");
+    doTestSuffix("1000", "uL");
+    doTestSuffix("1000", "UL");
+    doTestSuffix("1000", "lu");
+    doTestSuffix("1000", "Lu");
+    doTestSuffix("1000", "lU");
+    doTestSuffix("1000", "LU");
+    doTestSuffix("1000", "ull");
+    doTestSuffix("1000", "Ull");
+    doTestSuffix("1000", "uLL");
+    doTestSuffix("1000", "ULL");
+    doTestSuffix("1000", "llu");
+    doTestSuffix("1000", "LLu");
+    doTestSuffix("1000", "llU");
+    doTestSuffix("1000", "LLU");
+    doTestSuffix("1000", null);
+  }
+
   private void doTestIntegerLiteral(@NotNull String text, int expected) {
     doTestLiteral(text, true, Long.valueOf(expected), BigInteger.valueOf(expected), BigDecimal.valueOf(expected), "int");
   }
@@ -411,6 +438,11 @@ public class PyNumericLiteralTest extends PyTestCase {
     assertEquals(expectedType, type.getName());
   }
 
+  private void doTestSuffix(@NotNull String text, @Nullable String suffix) {
+    final PyNumericLiteralExpression literal = configureByText(text + (suffix == null ? "" : suffix));
+    assertEquals(suffix, literal.getIntegerLiteralSuffix());
+  }
+
   @NotNull
   private PyNumericLiteralExpression configureByText(@NotNull String text) {
     final PsiElement element = myFixture.configureByText(PythonFileType.INSTANCE, text).findElementAt(0);

python/testSrc/com/jetbrains/python/PyQuickFixTestCase.java

@@ -37,6 +37,20 @@ public abstract class PyQuickFixTestCase extends PyTestCase {
     myFixture.checkResultByFile(testFileName + "_after.py", true);
   }
 
+  protected void doQuickFixTest(final String hint, LanguageLevel languageLevel) {
+    runWithLanguageLevel(languageLevel, () -> doQuickFixTest(hint));
+  }
+
+  protected void doQuickFixTest(final String hint) {
+    String testName = getTestName(true);
+    myFixture.configureByFile(testName + ".py");
+    myFixture.checkHighlighting(true, false, false);
+    final IntentionAction intentionAction = myFixture.findSingleIntention(hint);
+    assertNotNull(intentionAction);
+    myFixture.launchAction(intentionAction);
+    myFixture.checkResultByFile(testName + "_after.py");
+  }
+
   protected void doInspectionTest(final Class inspectionClass) {
     final String testFileName = getTestName(true);
     myFixture.enableInspections(inspectionClass);

python/testSrc/com/jetbrains/python/intentions/PyIntentionTest.java

@@ -93,7 +93,7 @@ public class PyIntentionTest extends PyTestCase {
 
   // PY-18972
   public void testRemoveTrailingL() {
-    doTest(PyBundle.message("INTN.remove.trailing.l"), LanguageLevel.PYTHON34);
+    doTest(PyBundle.message("INTN.remove.trailing.suffix"), LanguageLevel.PYTHON34);
   }
 
   public void testReplaceOctalNumericLiteral() {

python/testSrc/com/jetbrains/python/quickFixes/PyRemoveTrailingSuffixQuickFixTest.java

@@ -0,0 +1,31 @@
+// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.jetbrains.python.quickFixes;
+
+import com.intellij.testFramework.fixtures.impl.CodeInsightTestFixtureImpl;
+import com.jetbrains.python.PyBundle;
+import com.jetbrains.python.PyQuickFixTestCase;
+import com.jetbrains.python.psi.LanguageLevel;
+
+import java.util.ListResourceBundle;
+
+public class PyRemoveTrailingSuffixQuickFixTest extends PyQuickFixTestCase {
+
+  private static final ListResourceBundle MESSAGE_BUNDLE = new ListResourceBundle() {
+    @Override
+    protected Object[][] getContents() {
+      return new Object[][] {
+        new Object[] { "bad-trail", "Python version " + LanguageLevel.getLatest() + " does not support a trailing ''{0}''." }
+      };
+    }
+  };
+
+  @Override
+  public void setUp() throws Exception {
+    super.setUp();
+    ((CodeInsightTestFixtureImpl)myFixture).setMessageBundles(MESSAGE_BUNDLE);
+  }
+
+  public void testFixl() {
+    doQuickFixTest(PyBundle.message("INTN.remove.trailing.suffix"), LanguageLevel.getLatest());
+  }
+}