Python also has its own regexp dialect; correct parsing of character classes for Python (PY-2908)

python/src/META-INF/python-plugin-common.xml

@@ -198,7 +198,8 @@
     <fileStructureGroupRuleProvider implementation="com.jetbrains.python.findUsages.PyFunctionGroupingRuleProvider" id="py-function"/>
     <usageTypeProvider implementation="com.jetbrains.python.findUsages.PyUsageTypeProvider"/>
 
-    <languageInjector implementation="com.jetbrains.python.codeInsight.PythonRegexpInjector"/>
+    <languageInjector implementation="com.jetbrains.python.codeInsight.regexp.PythonRegexpInjector"/>
+    <lang.parserDefinition language="PythonRegExp" implementationClass="com.jetbrains.python.codeInsight.regexp.PythonRegexpParserDefinition"/>
 
     <applicationService serviceInterface="com.jetbrains.python.codeInsight.PyCodeInsightSettings"
                         serviceImplementation="com.jetbrains.python.codeInsight.PyCodeInsightSettings"/>

python/src/com/jetbrains/python/codeInsight/regexp/PythonRegexpInjector.java

@@ -1,4 +1,4 @@
-package com.jetbrains.python.codeInsight;
+package com.jetbrains.python.codeInsight.regexp;
 
 import com.intellij.openapi.util.TextRange;
 import com.intellij.psi.InjectedLanguagePlaces;
@@ -57,7 +57,7 @@ public class PythonRegexpInjector implements LanguageInjector {
           if (element != null && element.getContainingFile().getName().equals("re.py") && isRegexpMethod(element, index)) {
             List<TextRange> ranges = ((PyStringLiteralExpression)host).getStringValueTextRanges();
             if (ranges.size() == 1) {
-              injectionPlacesRegistrar.addPlace(RegExpLanguage.INSTANCE, ranges.get(0), null, null);
+              injectionPlacesRegistrar.addPlace(PythonRegexpLanguage.INSTANCE, ranges.get(0), null, null);
             }
           }
         }

python/src/com/jetbrains/python/codeInsight/regexp/PythonRegexpLanguage.java

@@ -0,0 +1,15 @@
+package com.jetbrains.python.codeInsight.regexp;
+
+import com.intellij.lang.Language;
+import org.intellij.lang.regexp.RegExpLanguage;
+
+/**
+ * @author yole
+ */
+public class PythonRegexpLanguage extends Language {
+  public static final PythonRegexpLanguage INSTANCE = new PythonRegexpLanguage();
+
+  public PythonRegexpLanguage() {
+    super(RegExpLanguage.INSTANCE, "PythonRegExp");
+  }
+}

python/src/com/jetbrains/python/codeInsight/regexp/PythonRegexpParserDefinition.java

@@ -0,0 +1,33 @@
+package com.jetbrains.python.codeInsight.regexp;
+
+import com.intellij.lexer.Lexer;
+import com.intellij.openapi.project.Project;
+import com.intellij.psi.FileViewProvider;
+import com.intellij.psi.PsiFile;
+import com.intellij.psi.tree.IFileElementType;
+import org.intellij.lang.regexp.RegExpFile;
+import org.intellij.lang.regexp.RegExpLexer;
+import org.intellij.lang.regexp.RegExpParserDefinition;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * @author yole
+ */
+public class PythonRegexpParserDefinition extends RegExpParserDefinition {
+  public static final IFileElementType PYTHON_REGEXP_FILE = new IFileElementType("PYTHON_REGEXP_FILE", PythonRegexpLanguage.INSTANCE);
+
+  @NotNull
+  public Lexer createLexer(Project project) {
+    return new RegExpLexer(false, true, false);
+  }
+
+  @Override
+  public IFileElementType getFileNodeType() {
+    return PYTHON_REGEXP_FILE;
+  }
+
+  @Override
+  public PsiFile createFile(FileViewProvider viewProvider) {
+    return new RegExpFile(viewProvider, PythonRegexpLanguage.INSTANCE);
+  }
+}

python/testData/regexp/nestedCharacterClasses.py

@@ -0,0 +1,3 @@
+import re
+
+re.compile('[][]')

python/testData/regexp/nestedCharacterClasses2.py

@@ -0,0 +1,3 @@
+import re
+
+re.compile(r'''[[\]]''')

python/testSrc/com/jetbrains/python/PyRegexpTest.java

@@ -0,0 +1,20 @@
+package com.jetbrains.python;
+
+import com.jetbrains.python.fixtures.PyLightFixtureTestCase;
+
+/**
+ * @author yole
+ */
+public class PyRegexpTest extends PyLightFixtureTestCase {
+  public void testNestedCharacterClasses() {  // PY-2908
+    doTest();
+  }
+
+  public void testNestedCharacterClasses2() {  // PY-2908
+    doTest();
+  }
+
+  private void doTest() {
+    myFixture.testHighlighting(true, false, false, "regexp/" + getTestName(true) + ".py");
+  }
+}