From a453fede94984a5371920adf90ccf46c34092ac3 Mon Sep 17 00:00:00 2001 From: Morgan Bartholomew Date: Sun, 2 Nov 2025 12:13:58 +1000 Subject: [PATCH] [python] PY-35730 use `NonAsciiCharactersInspection` to report non-ascii characters in python regular expressions (cherry picked from commit d80469c6b698863b8986a7c9486fb7e69c418fcf) GitOrigin-RevId: 2400ea51b7b3b3d72011195ca336e4cf69d17931 --- .../intellij/lang/regexp/psi/RegExpGroup.java | 4 +-- .../lang/regexp/psi/impl/RegExpGroupImpl.java | 7 ++++ .../RegexNamedGroupWithNonAscii.java | 29 +++++++++++++++++ .../daemon/NonAsciiCharactersTest.java | 13 ++++++++ .../NonAsciiCharactersInspection.java | 9 +++++- .../python/codeInsight/PythonRegExpHost.java | 32 ++++++++++++++++--- .../com/jetbrains/python/PyRegexpTest.java | 18 +++++++++++ 7 files changed, 105 insertions(+), 7 deletions(-) create mode 100644 java/java-tests/testData/codeInsight/daemonCodeAnalyzer/nonAsciiCharacters/RegexNamedGroupWithNonAscii.java diff --git a/RegExpSupport/src/org/intellij/lang/regexp/psi/RegExpGroup.java b/RegExpSupport/src/org/intellij/lang/regexp/psi/RegExpGroup.java index 6a35954322f8..f45f467071f5 100644 --- a/RegExpSupport/src/org/intellij/lang/regexp/psi/RegExpGroup.java +++ b/RegExpSupport/src/org/intellij/lang/regexp/psi/RegExpGroup.java @@ -15,11 +15,11 @@ */ package org.intellij.lang.regexp.psi; -import com.intellij.psi.PsiNamedElement; +import com.intellij.psi.PsiNameIdentifierOwner; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -public interface RegExpGroup extends RegExpAtom, PsiNamedElement { +public interface RegExpGroup extends RegExpAtom, PsiNameIdentifierOwner { boolean isCapturing(); diff --git a/RegExpSupport/src/org/intellij/lang/regexp/psi/impl/RegExpGroupImpl.java b/RegExpSupport/src/org/intellij/lang/regexp/psi/impl/RegExpGroupImpl.java index 7c2d6139c1a7..265d26a6aecd 100644 --- a/RegExpSupport/src/org/intellij/lang/regexp/psi/impl/RegExpGroupImpl.java +++ b/RegExpSupport/src/org/intellij/lang/regexp/psi/impl/RegExpGroupImpl.java @@ -26,6 +26,7 @@ import org.intellij.lang.regexp.psi.RegExpGroup; import org.intellij.lang.regexp.psi.RegExpPattern; import org.jetbrains.annotations.NonNls; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; public class RegExpGroupImpl extends RegExpElementImpl implements RegExpGroup { public RegExpGroupImpl(ASTNode astNode) { @@ -98,6 +99,12 @@ public class RegExpGroupImpl extends RegExpElementImpl implements RegExpGroup { throw new AssertionError(); } + @Override + public @Nullable PsiElement getNameIdentifier() { + final ASTNode nameNode = getNode().findChildByType(RegExpTT.NAME); + return nameNode == null ? null : nameNode.getPsi(); + } + public static boolean isPcreConditionalGroup(ASTNode node) { return node != null && node.findChildByType(RegExpTT.PCRE_CONDITIONS) != null; } diff --git a/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/nonAsciiCharacters/RegexNamedGroupWithNonAscii.java b/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/nonAsciiCharacters/RegexNamedGroupWithNonAscii.java new file mode 100644 index 000000000000..e7682cc7bcd9 --- /dev/null +++ b/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/nonAsciiCharacters/RegexNamedGroupWithNonAscii.java @@ -0,0 +1,29 @@ +class RegexTest { + void testRegexNamedGroups() { + // Non-ASCII characters in named group names should be highlighted + // language=regexp + String a = "(?<水>a)"; + // language=regexp + String b = "(?<group_水>a)"; + // language=regexp + String c = "(?<水_group>a)"; + // language=regexp + String d = "(?<my_группа_123>a)"; + + // Multiple named groups with non-ASCII + // language=regexp + String e = "(?<первая>a)(?<вторая>b)"; + + // ASCII only group names should not be highlighted + // language=regexp + String f = "(?a)"; + // language=regexp + String g = "(?<my_group_123>a)"; + + // Non-ASCII in pattern body (not in group name) - these are not checked by this test + // language=regexp + String h = "(?水)"; + // language=regexp + String i = "水"; + } +} diff --git a/java/java-tests/testSrc/com/intellij/java/codeInsight/daemon/NonAsciiCharactersTest.java b/java/java-tests/testSrc/com/intellij/java/codeInsight/daemon/NonAsciiCharactersTest.java index 4572d19cd915..24bf1bf871a9 100644 --- a/java/java-tests/testSrc/com/intellij/java/codeInsight/daemon/NonAsciiCharactersTest.java +++ b/java/java-tests/testSrc/com/intellij/java/codeInsight/daemon/NonAsciiCharactersTest.java @@ -128,4 +128,17 @@ public class NonAsciiCharactersTest extends DaemonAnalyzerTestCase { myInspection.CHECK_FOR_FILES_CONTAINING_BOM = false; doTest(".txt"); } + + public void testRegexNamedGroupWithNonAscii() throws Exception { + myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_ANY_OTHER_WORD = false; + myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_IDENTIFIER_NAME = false; + myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_COMMENTS = false; + myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_STRING = false; + myInspection.CHECK_FOR_NOT_ASCII_IN_ANY_OTHER_WORD = false; + myInspection.CHECK_FOR_NOT_ASCII_IDENTIFIER_NAME = true; + myInspection.CHECK_FOR_NOT_ASCII_COMMENT = false; + myInspection.CHECK_FOR_NOT_ASCII_STRING_LITERAL = false; + myInspection.CHECK_FOR_FILES_CONTAINING_BOM = false; + doTest(".java"); + } } diff --git a/platform/lang-impl/src/com/intellij/codeInspection/NonAsciiCharactersInspection.java b/platform/lang-impl/src/com/intellij/codeInspection/NonAsciiCharactersInspection.java index cf7429babbfd..eb78fc52ccb8 100644 --- a/platform/lang-impl/src/com/intellij/codeInspection/NonAsciiCharactersInspection.java +++ b/platform/lang-impl/src/com/intellij/codeInspection/NonAsciiCharactersInspection.java @@ -25,6 +25,7 @@ import com.intellij.usages.ChunkExtractor; import org.jetbrains.annotations.*; import java.nio.charset.Charset; +import java.util.Objects; import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -203,7 +204,13 @@ public final class NonAsciiCharactersInspection extends LocalInspectionTool { } private static boolean isFileWorthIt(@NotNull PsiFile file) { - if (InjectedLanguageManager.getInstance(file.getProject()).isInjectedFragment(file)) return false; + if (InjectedLanguageManager.getInstance(file.getProject()).isInjectedFragment(file)) { + Language language = file.getLanguage(); + language = Objects.requireNonNullElse(language.getBaseLanguage(), language); + if (!language.getID().equals("RegExp")) { + return false; + } + } VirtualFile virtualFile = file.getVirtualFile(); if (virtualFile == null) return false; CharSequence text = file.getViewProvider().getContents(); diff --git a/python/src/com/jetbrains/python/codeInsight/PythonRegExpHost.java b/python/src/com/jetbrains/python/codeInsight/PythonRegExpHost.java index 563c4a9baaae..15c9cbfe8033 100644 --- a/python/src/com/jetbrains/python/codeInsight/PythonRegExpHost.java +++ b/python/src/com/jetbrains/python/codeInsight/PythonRegExpHost.java @@ -3,11 +3,9 @@ package com.jetbrains.python.codeInsight; import com.intellij.lang.injection.InjectedLanguageManager; import com.intellij.psi.PsiLanguageInjectionHost; +import com.jetbrains.python.PyNames; import com.jetbrains.python.psi.LanguageLevel; -import org.intellij.lang.regexp.DefaultRegExpPropertiesProvider; -import org.intellij.lang.regexp.RegExpLanguageHost; -import org.intellij.lang.regexp.RegExpTT; -import org.intellij.lang.regexp.UnicodeCharacterNames; +import org.intellij.lang.regexp.*; import org.intellij.lang.regexp.psi.*; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -117,4 +115,30 @@ public final class PythonRegExpHost implements RegExpLanguageHost { public boolean isValidNamedCharacter(RegExpNamedCharacter namedCharacter) { return UnicodeCharacterNames.getCodePoint(namedCharacter.getName()) >= 0; } + + @Override + public boolean isValidGroupName(String name, @NotNull RegExpGroup group) { + // non-ascii characters are allowed as group names in Python 3 + // the specification is ` *` + int offset = 0; + int codePoint = name.codePointAt(offset); + + // First character must be XID_Start + if (!Character.isUnicodeIdentifierStart(codePoint)) { + return false; + } + + offset += Character.charCount(codePoint); + + // Remaining characters must be XID_Continue + while (offset < name.length()) { + codePoint = name.codePointAt(offset); + if (!Character.isUnicodeIdentifierPart(codePoint)) { + return false; + } + offset += Character.charCount(codePoint); + } + + return true; + } } diff --git a/python/testSrc/com/jetbrains/python/PyRegexpTest.java b/python/testSrc/com/jetbrains/python/PyRegexpTest.java index 12b9fbbbbb14..7ce2ec498537 100644 --- a/python/testSrc/com/jetbrains/python/PyRegexpTest.java +++ b/python/testSrc/com/jetbrains/python/PyRegexpTest.java @@ -289,6 +289,24 @@ public class PyRegexpTest extends PyTestCase { ); } + @TestFor(issues="PY-35730") + public void testGroupNameIsValidIdentifier() { + //noinspection NonAsciiCharacters + testHighlighting( + """ + import re + re.compile("(?P<水>水)") # non-ascii is a valid group name, a warning will be reported by `NonAsciiCharactersInspection` + re.compile("(?P<𝕏>水)") # ok character outside the BMP + + # broken case IJPL-217664 + # re.compile("(?Perror descr="Invalid group name">😀>水)") # bad character outside the BMP + + re.compile("(?P< a>水)") + re.compile("(?P<0a>水)") + re.compile("(?P<a$b>水)") + """); + } + @Nullable @Override protected LightProjectDescriptor getProjectDescriptor() {