RegExp: handle unicode character values outside of the BMP correctly (PY-24172)

RegExpSupport/src/org/intellij/lang/regexp/psi/impl/RegExpCharImpl.java

@@ -65,8 +65,9 @@ public class RegExpCharImpl extends RegExpElementImpl implements RegExpChar {
         assert length > 0;
 
         boolean escaped = false;
-        for (int idx = 0; idx < length; idx++) {
-            final char ch = s.charAt(idx);
+        int idx = 0;
+        while (idx < length) {
+            final int ch = s.codePointAt(idx);
             if (!escaped) {
                 if (ch == '\\') {
                     escaped = true;
@@ -133,6 +134,7 @@ public class RegExpCharImpl extends RegExpElementImpl implements RegExpChar {
                         return ch;
                 }
             }
+            idx = Character.charCount(ch);
         }
 
         return -1;

python/testData/regexp/unicodePy3.py

@@ -0,0 +1,2 @@
+import re
+re.compile(r"[\U0001f570\U0001f573\U0001f57a]")

python/testSrc/com/jetbrains/python/PyRegexpTest.java

@@ -1,18 +1,4 @@
-/*
- * Copyright 2000-2017 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
 package com.jetbrains.python;
 
 import com.intellij.lang.injection.InjectedLanguageManager;
@@ -37,6 +23,10 @@ import java.util.List;
  */
 public class PyRegexpTest extends PyTestCase {
 
+  public void testUnicodePy3() {
+    doTestHighlighting();
+  }
+
   public void testCommentModeWhitespace() {
     doTestHighlighting();
   }
@@ -224,7 +214,7 @@ public class PyRegexpTest extends PyTestCase {
   }
 
   // PY-16404
-  public void testFullmatch() {
+  public void testFullmatchPy3() {
     doTestInjectedText("import re\n" +
                        "re.fullmatch(\"<caret>\\w+\", \"string\"",
                        "\\w+");
@@ -233,7 +223,7 @@ public class PyRegexpTest extends PyTestCase {
   @Nullable
   @Override
   protected LightProjectDescriptor getProjectDescriptor() {
-    return getName().equals("testFullmatch") ? ourPy3Descriptor : super.getProjectDescriptor();
+    return getName().endsWith("Py3") ? ourPy3Descriptor : super.getProjectDescriptor();
   }
 
   @NotNull