MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

[python] PY-35730 use NonAsciiCharactersInspection to report non-ascii characters in python regular expressions

Browse Source 
(cherry picked from commit d80469c6b698863b8986a7c9486fb7e69c418fcf)

GitOrigin-RevId: 2400ea51b7b3b3d72011195ca336e4cf69d17931
This commit is contained in:
Morgan Bartholomew
2025-11-12 10:47:17 +00:00
committed by intellij-monorepo-bot
parent 4067221a8f
commit a453fede94
7 changed files with 105 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
RegExpSupport/src/org/intellij/lang/regexp/psi/RegExpGroup.java
+2 -2
View File
@@ -15,11 +15,11 @@
*/
package org.intellij.lang.regexp.psi;
import com.intellij.psi.PsiNamedElement;
import com.intellij.psi.PsiNameIdentifierOwner;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
public interface RegExpGroup extends RegExpAtom, PsiNamedElement {
public interface RegExpGroup extends RegExpAtom, PsiNameIdentifierOwner {
boolean isCapturing();
RegExpSupport/src/org/intellij/lang/regexp/psi/impl/RegExpGroupImpl.java
+7
View File
@@ -26,6 +26,7 @@ import org.intellij.lang.regexp.psi.RegExpGroup;
import org.intellij.lang.regexp.psi.RegExpPattern;
import org.jetbrains.annotations.NonNls;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
public class RegExpGroupImpl extends RegExpElementImpl implements RegExpGroup {
public RegExpGroupImpl(ASTNode astNode) {
@@ -98,6 +99,12 @@ public class RegExpGroupImpl extends RegExpElementImpl implements RegExpGroup {
throw new AssertionError();
}
@Override
public @Nullable PsiElement getNameIdentifier() {
final ASTNode nameNode = getNode().findChildByType(RegExpTT.NAME);
return nameNode == null ? null : nameNode.getPsi();
}
public static boolean isPcreConditionalGroup(ASTNode node) {
return node != null && node.findChildByType(RegExpTT.PCRE_CONDITIONS) != null;
}
java/java-tests/testData/codeInsight/daemonCodeAnalyzer/nonAsciiCharacters/RegexNamedGroupWithNonAscii.java
+29
View File
@@ -0,0 +1,29 @@
class RegexTest {
void testRegexNamedGroups() {
// Non-ASCII characters in named group names should be highlighted
// language=regexp
String a = "(?<<error descr="Invalid group name"><warning descr="Non-ASCII characters">水</warning></error>>a)";
// language=regexp
String b = "(?<<error descr="Invalid group name">group_<warning descr="Non-ASCII characters">水</warning></error>>a)";
// language=regexp
String c = "(?<<error descr="Invalid group name"><warning descr="Non-ASCII characters">水</warning>_group</error>>a)";
// language=regexp
String d = "(?<<error descr="Invalid group name">my_<warning descr="Non-ASCII characters">группа</warning>_123</error>>a)";
// Multiple named groups with non-ASCII
// language=regexp
String e = "(?<<error descr="Invalid group name"><warning descr="Non-ASCII characters">первая</warning></error>>a)(?<<error descr="Invalid group name"><warning descr="Non-ASCII characters">вторая</warning></error>>b)";
// ASCII only group names should not be highlighted
// language=regexp
String f = "(?<group>a)";
// language=regexp
String g = "(?<<error descr="Invalid group name">my_group_123</error>>a)";
// Non-ASCII in pattern body (not in group name) - these are not checked by this test
// language=regexp
String h = "(?<group>水)";
// language=regexp
String i = "水";
}
}
java/java-tests/testSrc/com/intellij/java/codeInsight/daemon/NonAsciiCharactersTest.java
+13
View File
@@ -128,4 +128,17 @@ public class NonAsciiCharactersTest extends DaemonAnalyzerTestCase {
myInspection.CHECK_FOR_FILES_CONTAINING_BOM = false;
doTest(".txt");
}
public void testRegexNamedGroupWithNonAscii() throws Exception {
myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_ANY_OTHER_WORD = false;
myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_IDENTIFIER_NAME = false;
myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_COMMENTS = false;
myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_STRING = false;
myInspection.CHECK_FOR_NOT_ASCII_IN_ANY_OTHER_WORD = false;
myInspection.CHECK_FOR_NOT_ASCII_IDENTIFIER_NAME = true;
myInspection.CHECK_FOR_NOT_ASCII_COMMENT = false;
myInspection.CHECK_FOR_NOT_ASCII_STRING_LITERAL = false;
myInspection.CHECK_FOR_FILES_CONTAINING_BOM = false;
doTest(".java");
}
}
platform/lang-impl/src/com/intellij/codeInspection/NonAsciiCharactersInspection.java
+8 -1
View File
@@ -25,6 +25,7 @@ import com.intellij.usages.ChunkExtractor;
import org.jetbrains.annotations.*;
import java.nio.charset.Charset;
import java.util.Objects;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
@@ -203,7 +204,13 @@ public final class NonAsciiCharactersInspection extends LocalInspectionTool {
}
private static boolean isFileWorthIt(@NotNull PsiFile file) {
if (InjectedLanguageManager.getInstance(file.getProject()).isInjectedFragment(file)) return false;
if (InjectedLanguageManager.getInstance(file.getProject()).isInjectedFragment(file)) {
Language language = file.getLanguage();
language = Objects.requireNonNullElse(language.getBaseLanguage(), language);
if (!language.getID().equals("RegExp")) {
return false;
}
}
VirtualFile virtualFile = file.getVirtualFile();
if (virtualFile == null) return false;
CharSequence text = file.getViewProvider().getContents();
python/src/com/jetbrains/python/codeInsight/PythonRegExpHost.java
+28 -4
View File
@@ -3,11 +3,9 @@ package com.jetbrains.python.codeInsight;
import com.intellij.lang.injection.InjectedLanguageManager;
import com.intellij.psi.PsiLanguageInjectionHost;
import com.jetbrains.python.PyNames;
import com.jetbrains.python.psi.LanguageLevel;
import org.intellij.lang.regexp.DefaultRegExpPropertiesProvider;
import org.intellij.lang.regexp.RegExpLanguageHost;
import org.intellij.lang.regexp.RegExpTT;
import org.intellij.lang.regexp.UnicodeCharacterNames;
import org.intellij.lang.regexp.*;
import org.intellij.lang.regexp.psi.*;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -117,4 +115,30 @@ public final class PythonRegExpHost implements RegExpLanguageHost {
public boolean isValidNamedCharacter(RegExpNamedCharacter namedCharacter) {
return UnicodeCharacterNames.getCodePoint(namedCharacter.getName()) >= 0;
}
@Override
public boolean isValidGroupName(String name, @NotNull RegExpGroup group) {
// non-ascii characters are allowed as group names in Python 3
// the specification is `<XID_Start> <XID_Continue>*`
int offset = 0;
int codePoint = name.codePointAt(offset);
// First character must be XID_Start
if (!Character.isUnicodeIdentifierStart(codePoint)) {
return false;
}
offset += Character.charCount(codePoint);
// Remaining characters must be XID_Continue
while (offset < name.length()) {
codePoint = name.codePointAt(offset);
if (!Character.isUnicodeIdentifierPart(codePoint)) {
return false;
}
offset += Character.charCount(codePoint);
}
return true;
}
}
python/testSrc/com/jetbrains/python/PyRegexpTest.java
+18
View File
@@ -289,6 +289,24 @@ public class PyRegexpTest extends PyTestCase {
);
}
@TestFor(issues="PY-35730")
public void testGroupNameIsValidIdentifier() {
//noinspection NonAsciiCharacters
testHighlighting(
"""
import re
re.compile("(?P<水>水)") # non-ascii is a valid group name, a warning will be reported by `NonAsciiCharactersInspection`
re.compile("(?P<𝕏>水)") # ok character outside the BMP
# broken case IJPL-217664
# re.compile("(?Perror descr="Invalid group name">😀</error>>水)") # bad character outside the BMP
re.compile("(?P<<error descr="Group name expected"> </error>a>水)")
re.compile("(?P<<error descr="Group name expected">0</error>a>水)")
re.compile("(?P<<error descr="Invalid group name">a$b</error>>水)")
""");
}
@Nullable
@Override
protected LightProjectDescriptor getProjectDescriptor() {