Update CompatibilityVisitor to highlight and suggest quick fix for underscores in numeric literals in Pythons < 3.6

As a result compatibility inspection warns about underscores for Pythons < 3.6

python/src/com/jetbrains/python/PyBundle.properties

@@ -159,6 +159,9 @@ QFIX.NAME.wrap.in.exception=Wrap with Exception call
 
 QFIX.NAME.make.list=Replace tuple with list
 
+#PyRemoveUnderscoresInNumericLiteralsQuickFix
+QFIX.NAME.remove.underscores.in.numeric=Remove underscores in numeric literals
+
 # Intentions: INTN
 INTN.Family.convert.import.unqualify=Convert 'import module' to 'from module import'
 INTN.Family.convert.import.qualify=Convert 'from module import' to 'import module'

python/src/com/jetbrains/python/inspections/quickfix/PyRemoveUnderscoresInNumericLiteralsQuickFix.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2000-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.jetbrains.python.inspections.quickfix;
+
+import com.intellij.codeInspection.LocalQuickFix;
+import com.intellij.codeInspection.ProblemDescriptor;
+import com.intellij.openapi.project.Project;
+import com.intellij.psi.PsiElement;
+import com.jetbrains.python.PyBundle;
+import com.jetbrains.python.psi.LanguageLevel;
+import com.jetbrains.python.psi.PyElementGenerator;
+import com.jetbrains.python.psi.PyNumericLiteralExpression;
+import org.jetbrains.annotations.Nls;
+import org.jetbrains.annotations.NotNull;
+
+public class PyRemoveUnderscoresInNumericLiteralsQuickFix implements LocalQuickFix {
+
+  @Nls
+  @NotNull
+  @Override
+  public String getName() {
+    return PyBundle.message("QFIX.NAME.remove.underscores.in.numeric");
+  }
+
+  @Nls
+  @NotNull
+  @Override
+  public String getFamilyName() {
+    return getName();
+  }
+
+  @Override
+  public void applyFix(@NotNull Project project, @NotNull ProblemDescriptor descriptor) {
+    final PsiElement element = descriptor.getPsiElement();
+    if (element instanceof PyNumericLiteralExpression) {
+      final PyElementGenerator elementGenerator = PyElementGenerator.getInstance(project);
+      final String text = element.getText();
+
+      element.replace(elementGenerator.createExpressionFromText(LanguageLevel.forElement(element), text.replaceAll("_", "")));
+    }
+  }
+}

python/src/com/jetbrains/python/validation/CompatibilityVisitor.java

@@ -239,9 +239,9 @@ public abstract class CompatibilityVisitor extends PyAnnotator {
   public void visitPyNumericLiteralExpression(final PyNumericLiteralExpression node) {
     super.visitPyNumericLiteralExpression(node);
 
-    if (node.isIntegerLiteral()) {
-      final String text = node.getText();
+    final String text = node.getText();
 
+    if (node.isIntegerLiteral()) {
       if (text.endsWith("l") || text.endsWith("L")) {
         final StringBuilder message = new StringBuilder(myCommonMessage);
         final String suffix = " not support a trailing \'l\' or \'L\'.";
@@ -273,6 +273,20 @@ public abstract class CompatibilityVisitor extends PyAnnotator {
         }
       }
     }
+
+    if (text.contains("_")) {
+      final StringBuilder message = new StringBuilder(myCommonMessage);
+      final String suffix = " not support underscores in numeric literals";
+      int len = 0;
+
+      for (LanguageLevel languageLevel : myVersionsToProcess) {
+        if (languageLevel.isOlderThan(LanguageLevel.PYTHON36)) {
+          len = appendLanguageLevel(message, len, languageLevel);
+        }
+      }
+
+      commonRegisterProblem(message, suffix, len, node, new PyRemoveUnderscoresInNumericLiteralsQuickFix());
+    }
   }
 
   @Override

python/testData/highlighting/underscoresInNumericLiterals.py

@@ -0,0 +1,58 @@
+# hex
+<error descr="Python version 3.5 does not support underscores in numeric literals">0xCAFE_F00D</error>
+
+# oct
+<error descr="Python version 3.5 does not support underscores in numeric literals">0o1_23</error>
+<error descr="Python version 3.5 does not support this syntax. It requires '0o' prefix for octal literals"><error descr="Python version 3.5 does not support underscores in numeric literals">01_23</error></error>
+
+# bin
+<error descr="Python version 3.5 does not support underscores in numeric literals">0b_0011_1111_0100_1110</error>
+
+# dec
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_000_000</error>
+
+# pointfloat
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_00.00_23</error>
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_00.</error>
+
+# exponentfloat
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_00.00_23e1_2</error>
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_00.00_23E1_2</error>
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_00.e1_2</error>
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_00.E1_2</error>
+
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_00.00_23e+1_2</error>
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_00.00_23E+1_2</error>
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_00.e+1_2</error>
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_00.E+1_2</error>
+
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_00.00_23e-1_2</error>
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_00.00_23E-1_2</error>
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_00.e-1_2</error>
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_00.E-1_2</error>
+
+
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_0000_23e1_2</error>
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_0000_23E1_2</error>
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_00e1_2</error>
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_00E1_2</error>
+
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_0000_23e+1_2</error>
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_0000_23E+1_2</error>
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_00e+1_2</error>
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_00E+1_2</error>
+
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_0000_23e-1_2</error>
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_0000_23E-1_2</error>
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_00e-1_2</error>
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_00E-1_2</error>
+
+# imag
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_00.00_23j</error>
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_00.00_23J</error>
+
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_00.00_23e1_2j</error>
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_00.00_23e1_2J</error>
+
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_000_000j</error>
+<error descr="Python version 3.5 does not support underscores in numeric literals">10_000_000J</error>

python/testData/inspections/PyCompatibilityInspection/underscoresInNumericLiterals.py

@@ -0,0 +1,58 @@
+# hex
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">0xCAFE_F00D</warning>
+
+# oct
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">0o1_23</warning>
+<error descr="Python version 3.6 does not support this syntax. It requires '0o' prefix for octal literals"><warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals"><warning descr="Python version 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6 do not support this syntax. It requires '0o' prefix for octal literals">01_23</warning></warning></error>
+
+# bin
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">0b_0011_1111_0100_1110</warning>
+
+# dec
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_000_000</warning>
+
+# pointfloat
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_00.00_23</warning>
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_00.</warning>
+
+# exponentfloat
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_00.00_23e1_2</warning>
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_00.00_23E1_2</warning>
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_00.e1_2</warning>
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_00.E1_2</warning>
+
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_00.00_23e+1_2</warning>
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_00.00_23E+1_2</warning>
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_00.e+1_2</warning>
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_00.E+1_2</warning>
+
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_00.00_23e-1_2</warning>
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_00.00_23E-1_2</warning>
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_00.e-1_2</warning>
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_00.E-1_2</warning>
+
+
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_0000_23e1_2</warning>
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_0000_23E1_2</warning>
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_00e1_2</warning>
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_00E1_2</warning>
+
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_0000_23e+1_2</warning>
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_0000_23E+1_2</warning>
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_00e+1_2</warning>
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_00E+1_2</warning>
+
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_0000_23e-1_2</warning>
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_0000_23E-1_2</warning>
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_00e-1_2</warning>
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_00E-1_2</warning>
+
+# imag
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_00.00_23j</warning>
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_00.00_23J</warning>
+
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_00.00_23e1_2j</warning>
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_00.00_23e1_2J</warning>
+
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_000_000j</warning>
+<warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5 do not support underscores in numeric literals">10_000_000J</warning>

python/testSrc/com/jetbrains/python/PyQuickFixTest.java

@@ -588,6 +588,15 @@ public class PyQuickFixTest extends PyTestCase {
                      true, true);
   }
 
+  public void testRemovingUnderscoresInNumericLiterals() {
+    myFixture.configureByText(PythonFileType.INSTANCE, "1_0_0");
+
+    final IntentionAction action = myFixture.findSingleIntention(PyBundle.message("QFIX.NAME.remove.underscores.in.numeric"));
+    myFixture.launchAction(action);
+
+    myFixture.checkResult("100");
+  }
+
   @Override
   @NonNls
   protected String getTestDataPath() {

python/testSrc/com/jetbrains/python/PythonHighlightingTest.java

@@ -289,6 +289,10 @@ public class PythonHighlightingTest extends PyTestCase {
     doTest(LanguageLevel.PYTHON35, true, false);
   }
 
+  public void testUnderscoresInNumericLiterals() {
+    doTest(LanguageLevel.PYTHON35, true, false);
+  }
+
   // ---
   private void doTest(final LanguageLevel languageLevel, final boolean checkWarnings, final boolean checkInfos) {
     PythonLanguageLevelPusher.setForcedLanguageLevel(myFixture.getProject(), languageLevel);

python/testSrc/com/jetbrains/python/inspections/PyCompatibilityInspectionTest.java

@@ -186,6 +186,10 @@ public class PyCompatibilityInspectionTest extends PyTestCase {
     doTest();
   }
 
+  public void testUnderscoresInNumericLiterals() {
+    doTest(LanguageLevel.PYTHON36);
+  }
+
   private void doTest(@NotNull LanguageLevel level) {
     runWithLanguageLevel(level, this::doTest);
   }