Non-ASCII characters inspection: introduce new option "mixed languages in any word" to report this kind of problem in any language, fix incorrect mapping UI<-> CHECK field, to fix IDEA-291262 Report words with mixed charsets

GitOrigin-RevId: e22ebd3b1d11683b38280d254da8e8083d0770f1
This commit is contained in:
Alexey Kudravtsev
2022-04-06 12:18:10 +00:00
committed by intellij-monorepo-bot
parent f3ac494fcd
commit 3caa07b67a
13 changed files with 454 additions and 140 deletions
@@ -1,9 +1,9 @@
class X {
int <warning descr="Non-ASCII characters in an identifier">Ж</warning> = 0;
class <warning descr="Non-ASCII characters in an identifier"><warning descr="Symbols from different languages found: [LATIN, CYRILLIC]">InnerП</warning></warning> {}
// comment<warning descr="Non-ASCII characters in a comment">жп</warning> 234
String s = "12<warning descr="Non-ASCII characters in a string literal">л</warning>3<warning descr="Non-ASCII characters in a string literal">орыва</warning>0";
void <warning descr="Non-ASCII characters in an identifier">жжж</warning>() {
жжж();
int <warning descr="Non-ASCII characters">Ж</warning> = 0;
class Inner<warning descr="Mixed languages: CYRILLIC symbols found in LATIN word"><warning descr="Non-ASCII characters">П</warning></warning> {}
// comment<warning descr="Mixed languages: CYRILLIC symbols found in LATIN word"><warning descr="Non-ASCII characters">жп</warning></warning> 234
String s = "12<warning descr="Mixed languages: CYRILLIC symbols found in LATIN word"><warning descr="Non-ASCII characters">л</warning></warning>TO<warning descr="Non-ASCII characters">орыва</warning>0";
void <warning descr="Non-ASCII characters">жжж</warning>() {
<warning descr="Non-ASCII characters">жжж</warning>();
}
}
@@ -0,0 +1,13 @@
class X {
int Ж = 0;
class Inner<warning descr="Mixed languages: CYRILLIC symbols found in LATIN word">П</warning> {}
// comment<warning descr="Mixed languages: CYRILLIC symbols found in LATIN word">жп</warning> 234
String sameLang = "12л3орыва0";
String mixed = "12<warning descr="Mixed languages: CYRILLIC symbols found in LATIN word">че</warning>to3";
void жжж() {
жжж();
String core = "<warning descr="Mixed languages: CYRILLIC symbols found in LATIN word">С</warning>ore";
System.out.println("<warning descr="Mixed languages: CYRILLIC symbols found in LATIN word">С</warning>ore");
String u = "\5\0\51\2\3\0\136\2\21\0\33\2\65\0\20\2\u0200\0\u19b6\2"; // ignore slash u
}
}
@@ -0,0 +1,13 @@
class X {
int Ж = 0;
class Inner<warning descr="Mixed languages: CYRILLIC symbols found in LATIN word">П</warning> {}
// comment<warning descr="Mixed languages: CYRILLIC symbols found in LATIN word">жп</warning> 234
String sameLang = "12л3орыва0";
String mixed = "12<warning descr="Mixed languages: CYRILLIC symbols found in LATIN word">че</warning>to3";
void жжж() {
жжж();
String core = "<warning descr="Mixed languages: CYRILLIC symbols found in LATIN word">С</warning>ore";
System.out.println("<warning descr="Mixed languages: CYRILLIC symbols found in LATIN word">С</warning>ore");
String u = "\5\0\51\2\3\0\136\2\21\0\33\2\65\0\20\2\u0200\0\u19b6\2"; // ignore slash u
}
}
@@ -0,0 +1,6 @@
<x<warning descr="Mixed languages: CYRILLIC symbols found in LATIN word">Ш</warning>>
блабла
<t<warning descr="Mixed languages: CYRILLIC symbols found in LATIN word">Ы</warning> attr="ИИИИ23" attr2="<warning descr="Mixed languages: CYRILLIC symbols found in LATIN word">ИИИ</warning>foo">
<warning descr="Mixed languages: CYRILLIC symbols found in LATIN word">бла</warning>foo
</t<warning descr="Mixed languages: CYRILLIC symbols found in LATIN word">Ы</warning>>
</x<warning descr="Mixed languages: CYRILLIC symbols found in LATIN word">Ш</warning>>
@@ -0,0 +1,6 @@
<x<warning descr="Mixed languages: CYRILLIC symbols found in LATIN word">Ш</warning>>
блабла
<t<warning descr="Mixed languages: CYRILLIC symbols found in LATIN word">Ы</warning> attr="ИИИИ23" attr2="ИИИfoo">
<warning descr="Mixed languages: CYRILLIC symbols found in LATIN word">бла</warning>foo
</t<warning descr="Mixed languages: CYRILLIC symbols found in LATIN word">Ы</warning>>
</x<warning descr="Mixed languages: CYRILLIC symbols found in LATIN word">Ш</warning>>
@@ -0,0 +1,12 @@
class X {
int <warning descr="Non-ASCII characters">Ж</warning> = 0;
class Inner<warning descr="Non-ASCII characters">П</warning> {}
// comment<warning descr="Non-ASCII characters">жп</warning> 234
String s = "12<warning descr="Non-ASCII characters">л</warning>3<warning descr="Non-ASCII characters">орыва</warning>0";
void <warning descr="Non-ASCII characters">жжж</warning>() {
<warning descr="Non-ASCII characters">жжж</warning>();
String s = "<warning descr="Non-ASCII characters">С</warning>ore";
System.out.println("<warning descr="Non-ASCII characters">С</warning>ore");
String u = "\5\0\51\2\3\0\136\2\21\0\33\2\65\0\20\2\u0200\0\u19b6\2"; // ignore slash u
}
}
@@ -0,0 +1,12 @@
class X {
int <warning descr="Non-ASCII characters">Ж</warning> = 0;
class Inner<warning descr="Non-ASCII characters">П</warning> {}
// comment<warning descr="Non-ASCII characters">жп</warning> 234
String s = "12<warning descr="Non-ASCII characters">л</warning>3<warning descr="Non-ASCII characters">орыва</warning>0";
void <warning descr="Non-ASCII characters">жжж</warning>() {
<warning descr="Non-ASCII characters">жжж</warning>();
String s = "<warning descr="Non-ASCII characters">С</warning>ore";
System.out.println("<warning descr="Non-ASCII characters">С</warning>ore");
String u = "\5\0\51\2\3\0\136\2\21\0\33\2\65\0\20\2\u0200\0\u19b6\2"; // ignore slash u
}
}
@@ -1,12 +0,0 @@
class X {
int <warning descr="Non-ASCII characters in an identifier">Ж</warning> = 0;
class <warning descr="Non-ASCII characters in an identifier"><warning descr="Symbols from different languages found: [LATIN, CYRILLIC]">InnerП</warning></warning> {}
// comment<warning descr="Non-ASCII characters in a comment">жп</warning> 234
String s = "12<warning descr="Non-ASCII characters in a string literal">л</warning>3<warning descr="Non-ASCII characters in a string literal">орыва</warning>0";
void <warning descr="Non-ASCII characters in an identifier">жжж</warning>() {
жжж();
String s = <warning descr="Symbols from different languages found: [LATIN, CYRILLIC]">"<warning descr="Non-ASCII characters in a string literal">С</warning>ore"</warning>;
System.out.println(<warning descr="Symbols from different languages found: [LATIN, CYRILLIC]">"<warning descr="Non-ASCII characters in a string literal">С</warning>ore"</warning>);
String u = "\5\0\51\2\3\0\136\2\21\0\33\2\65\0\20\2\u0200\0\u19b6\2"; // ignore slash u
}
}
@@ -9,17 +9,17 @@ import org.jetbrains.annotations.NotNull;
public class NonAsciiCharactersTest extends DaemonAnalyzerTestCase {
private static final String BASE_PATH = "/codeInsight/daemonCodeAnalyzer/nonAsciiCharacters";
private NonAsciiCharactersInspection myInspection = new NonAsciiCharactersInspection();
@Override
protected LocalInspectionTool @NotNull [] configureLocalInspectionTools() {
NonAsciiCharactersInspection inspection = new NonAsciiCharactersInspection();
inspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_IDENTIFIER_NAME = true;
inspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_STRING = true;
inspection.CHECK_FOR_NOT_ASCII_IDENTIFIER_NAME = true;
inspection.CHECK_FOR_NOT_ASCII_COMMENT = true;
inspection.CHECK_FOR_NOT_ASCII_STRING_LITERAL = true;
inspection.CHECK_FOR_FILES_CONTAINING_BOM = true;
return new LocalInspectionTool[]{inspection};
return new LocalInspectionTool[]{myInspection};
}
@Override
protected void tearDown() throws Exception {
myInspection = null;
super.tearDown();
}
private void doTest(String extension) throws Exception {
@@ -27,10 +27,92 @@ public class NonAsciiCharactersTest extends DaemonAnalyzerTestCase {
UIUtil.dispatchAllInvocationEvents();
}
public void testSimple() throws Exception {
public void testNotAsciiJavaInVariousContexts() throws Exception {
myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_ANY_OTHER_WORD = false;
myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_IDENTIFIER_NAME = false;
myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_COMMENTS = false;
myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_STRING = false;
myInspection.CHECK_FOR_NOT_ASCII_IN_ANY_OTHER_WORD = false;
myInspection.CHECK_FOR_NOT_ASCII_IDENTIFIER_NAME = true;
myInspection.CHECK_FOR_NOT_ASCII_COMMENT = true;
myInspection.CHECK_FOR_NOT_ASCII_STRING_LITERAL = true;
myInspection.CHECK_FOR_FILES_CONTAINING_BOM = false;
doTest(".java");
}
public void testNotAsciiJavaInAnyWord() throws Exception {
myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_ANY_OTHER_WORD = false;
myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_IDENTIFIER_NAME = false;
myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_COMMENTS = false;
myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_STRING = false;
myInspection.CHECK_FOR_NOT_ASCII_IN_ANY_OTHER_WORD = true;
myInspection.CHECK_FOR_NOT_ASCII_IDENTIFIER_NAME = true;
myInspection.CHECK_FOR_NOT_ASCII_COMMENT = true;
myInspection.CHECK_FOR_NOT_ASCII_STRING_LITERAL = true;
myInspection.CHECK_FOR_FILES_CONTAINING_BOM = false;
doTest(".java");
}
public void testMixedLanguagesJavaInAnyWord() throws Exception {
myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_ANY_OTHER_WORD = true;
myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_IDENTIFIER_NAME = true;
myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_COMMENTS = true;
myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_STRING = true;
myInspection.CHECK_FOR_NOT_ASCII_IN_ANY_OTHER_WORD = false;
myInspection.CHECK_FOR_NOT_ASCII_IDENTIFIER_NAME = false;
myInspection.CHECK_FOR_NOT_ASCII_COMMENT = false;
myInspection.CHECK_FOR_NOT_ASCII_STRING_LITERAL = false;
myInspection.CHECK_FOR_FILES_CONTAINING_BOM = false;
doTest(".java");
}
public void testMixedLanguagesXMLInAnyWord() throws Exception {
myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_ANY_OTHER_WORD = true;
myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_IDENTIFIER_NAME = true;
myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_COMMENTS = true;
myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_STRING = true;
myInspection.CHECK_FOR_NOT_ASCII_IN_ANY_OTHER_WORD = false;
myInspection.CHECK_FOR_NOT_ASCII_IDENTIFIER_NAME = false;
myInspection.CHECK_FOR_NOT_ASCII_COMMENT = false;
myInspection.CHECK_FOR_NOT_ASCII_STRING_LITERAL = false;
myInspection.CHECK_FOR_FILES_CONTAINING_BOM = false;
doTest(".xml");
}
public void testMixedLanguagesXMLInAnyWordExceptString() throws Exception {
myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_ANY_OTHER_WORD = true;
myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_IDENTIFIER_NAME = true;
myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_COMMENTS = true;
myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_STRING = false;
myInspection.CHECK_FOR_NOT_ASCII_IN_ANY_OTHER_WORD = false;
myInspection.CHECK_FOR_NOT_ASCII_IDENTIFIER_NAME = false;
myInspection.CHECK_FOR_NOT_ASCII_COMMENT = false;
myInspection.CHECK_FOR_NOT_ASCII_STRING_LITERAL = false;
myInspection.CHECK_FOR_FILES_CONTAINING_BOM = false;
doTest(".xml");
}
public void testMixedLanguagesJavaInVariousContexts() throws Exception {
myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_ANY_OTHER_WORD = false;
myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_IDENTIFIER_NAME = true;
myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_COMMENTS = true;
myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_STRING = true;
myInspection.CHECK_FOR_NOT_ASCII_IN_ANY_OTHER_WORD = false;
myInspection.CHECK_FOR_NOT_ASCII_IDENTIFIER_NAME = false;
myInspection.CHECK_FOR_NOT_ASCII_COMMENT = false;
myInspection.CHECK_FOR_NOT_ASCII_STRING_LITERAL = false;
myInspection.CHECK_FOR_FILES_CONTAINING_BOM = false;
doTest(".java");
}
public void testGroovy() throws Exception {
myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_ANY_OTHER_WORD = true;
myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_IDENTIFIER_NAME = true;
myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_COMMENTS = true;
myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_STRING = true;
myInspection.CHECK_FOR_NOT_ASCII_IN_ANY_OTHER_WORD = true;
myInspection.CHECK_FOR_NOT_ASCII_IDENTIFIER_NAME = true;
myInspection.CHECK_FOR_NOT_ASCII_COMMENT = true;
myInspection.CHECK_FOR_NOT_ASCII_STRING_LITERAL = true;
myInspection.CHECK_FOR_FILES_CONTAINING_BOM = false;
doTest(".groovy");
}
}
@@ -512,19 +512,31 @@ actions.on.save.page.popup.title.arrangement.settings=Open Arrangement Settings
actions.on.save.page.checkbox.run.code.cleanup=Run code cleanup
actions.on.save.page.code.cleanup.comment=Applies fixes from the code cleanup inspections
actions.on.save.page.link.configure.inspections=Configure inspections...
non.ascii.warn.of=Warn of:
label.non.ascii.chars.example.files.containing.bom=E.g. files starting with <code>FEFF</code>
label.non.ascii.chars.example.characters.in.comments=E.g.: <code>// hello \u10D2\u10D4\u10DC\u10D0\u10EA\u10D5\u10D0\u10DA\u10D8</code>
label.non.ascii.chars.example.characters.in.identifiers=E.g.: <code>int \u041F\u043E\u043B\u0435 = 0;</code>
label.non.ascii.chars.example.characters.in.strings=E.g.: <code>println("\u10D2\u10D4\u10DC\u10D0\u10EA\u10D5\u10D0\u10DA\u10D8");</code>
label.non.ascii.chars.example.different.languages.in.identifiers=E.g.: <code>void print\u0415\u0440\u0443\u043D\u0434\u0430()</code>
label.non.ascii.chars.example.different.languages.in.string=E.g.: <code>foo("\u0421ore"); //cyrillic \u0421</code>
checkbox.non.ascii.option.different.languages.in.identifiers=Different languages in identifiers
checkbox.non.ascii.option.different.languages.in.string=Different languages in strings
checkbox.non.ascii.option.files.containing.bom=Files containing BOM
checkbox.non.ascii.option.characters.in.identifiers=Non-ASCII characters in identifiers
checkbox.non.ascii.option.characters.in.comments=Non-ASCII characters in comments
checkbox.non.ascii.option.characters.in.strings=Non-ASCII characters in strings
non.ascii.chars.inspection.non.ascii.top.label=Warn of non-ASCII characters in:
non.ascii.chars.inspection.option.files.containing.bom.checkbox=Warn of files containing BOM (e.g., files starting with 'FEFF' bytes)
non.ascii.chars.inspection.option.characters.in.any.other.word.checkbox=any other word
non.ascii.chars.inspection.example.characters.in.any.other.word.label=(e.g.: <code>\\&lt;<font color=red>\u0422\u042D\u0413-\u0428\u041C\u042D\u0413</font>\\&gt;0\\&lt;/<font color=red>\u0422\u042D\u0413-\u0428\u041C\u042D\u0413</font>\\&gt;</code>)
non.ascii.chars.inspection.option.characters.in.identifiers.checkbox=identifiers
non.ascii.chars.inspection.example.characters.in.identifiers.label=(e.g.: <code>int <font color=red>\u041F\u043E\u043B\u0435</font> = 0;</code>)
non.ascii.chars.inspection.option.characters.in.comments.checkbox=comments
non.ascii.chars.inspection.example.characters.in.comments.label=(e.g.: <code>// <font color=red>\u10D2\u10D4\u10DC\u10D0\u10EA\u10D5\u10D0\u10DA\u10D8</font></code>)
non.ascii.chars.inspection.option.characters.in.strings.checkbox=strings
non.ascii.chars.inspection.example.characters.in.strings.label=(e.g.: <code>println("<font color=red>\u10D2\u10D4\u10DC\u10D0\u10EA\u10D5\u10D0\u10DA\u10D8</font>")</code>)
non.ascii.chars.inspection.mixed.chars.top.label=Warn of mixed language characters in:
non.ascii.chars.inspection.option.mixed.languages.in.any.other.word.checkbox=any other word
non.ascii.chars.inspection.example.mixed.languages.in.any.other.word.label=(e.g.: <code>\\&lt;t<font color=red>\u0430</font>g\\&gt;0\\&lt;/t<font color=red>\u0430</font>g\\&gt;</code> (cyrillic '<font color=red>\u0430</font>'))
non.ascii.chars.inspection.option.mixed.languages.in.identifiers.checkbox=identifiers
non.ascii.chars.inspection.example.mixed.languages.in.identifiers.label=(e.g.: <code>void print<font color=red>\u0415\u0440\u0443\u043D\u0434\u0430</font>()</code>)
non.ascii.chars.inspection.option.mixed.languages.in.comments.checkbox=comments
non.ascii.chars.inspection.example.mixed.languages.in.comments.label=(e.g.: <code>//c<font color=red>\u043E</font>unts bugs</code> (cyrillic '<font color=red>o</font>'))
non.ascii.chars.inspection.option.mixed.languages.in.strings.checkbox=strings
non.ascii.chars.inspection.example.mixed.languages.in.string.label=(e.g.: <code>foo("<font color=red>\u0421</font>ore"); //cyrillic '<font color=red>\u0421</font>' </code>)
non.ascii.chars.inspection.message.file.contains.bom=File contains BOM: ''{0}''{1}
non.ascii.chars.inspection.message.charset.signature=\ (charset ''{0}'' signature)
non.ascii.chars.inspection.message.non.ascii.characters=Non-ASCII characters
non.ascii.chars.inspection.message.symbols.from.different.languages.found=Mixed languages: {0} symbols found in {1} word
label.file.included.into=Included into:
checkbox.reformat.project.file.masks=&File mask(s)
checkbox.reformat.project.include.subdirectories=&Include subdirectories
@@ -100,10 +100,6 @@ dialog.message.live.template.with.empty.abbreviation=A live template with an emp
dialog.message.live.template.with.empty.text=A live template ''{0}'' with an empty text has been found in ''{1}'' group, such live templates make no sense
dialog.message.duplicate.live.templates.in.group=Duplicate `{0}` live templates in {1} group
popup.advertisement.press.or.to.replace=Press {0} or {1} to replace
inspection.message.file.contains.bom=File contains BOM: ''{0}''{1}
inspection.message.charset.signature=\ (charset ''{0}'' signature)
inspection.message.non.ascii.characters.in=Non-ASCII characters in {0, choice, 0#an identifier|1#a comment|2#a string literal}
inspection.message.symbols.from.different.languages.found=Symbols from different languages found: {0}
hint.text.unfortunately.currently.available.for.batch.mode=Unfortunately ''{0}'' is currently not available for batch mode\n User interaction is required for each problem found
progress.title.inspect.code=Running Code Cleanup with profile ''{0}''\u2026
hint.text.can.t.modify.decompiled.code=Can't modify decompiled code
@@ -2,26 +2,31 @@
package com.intellij.codeInspection;
import com.intellij.lang.LangBundle;
import com.intellij.codeInsight.CodeInsightBundle;
import com.intellij.lang.Commenter;
import com.intellij.lang.Language;
import com.intellij.lang.LanguageCommenters;
import com.intellij.lang.injection.InjectedLanguageManager;
import com.intellij.lang.properties.charset.Native2AsciiCharset;
import com.intellij.openapi.editor.colors.TextAttributesKey;
import com.intellij.openapi.fileEditor.impl.LoadTextUtil;
import com.intellij.openapi.fileTypes.SyntaxHighlighter;
import com.intellij.openapi.fileTypes.SyntaxHighlighterFactory;
import com.intellij.openapi.util.TextRange;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.openapi.vfs.CharsetToolkit;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.psi.*;
import com.intellij.util.io.IOUtil;
import com.intellij.psi.impl.source.tree.ForeignLeafPsiElement;
import com.intellij.psi.impl.source.tree.LeafPsiElement;
import com.intellij.usages.ChunkExtractor;
import org.jetbrains.annotations.Nls;
import org.jetbrains.annotations.NonNls;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import javax.swing.*;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
@@ -29,9 +34,12 @@ public class NonAsciiCharactersInspection extends LocalInspectionTool {
public boolean CHECK_FOR_NOT_ASCII_IDENTIFIER_NAME = true;
public boolean CHECK_FOR_NOT_ASCII_STRING_LITERAL;
public boolean CHECK_FOR_NOT_ASCII_COMMENT;
public boolean CHECK_FOR_NOT_ASCII_IN_ANY_OTHER_WORD;
public boolean CHECK_FOR_DIFFERENT_LANGUAGES_IN_IDENTIFIER_NAME = true;
public boolean CHECK_FOR_DIFFERENT_LANGUAGES_IN_STRING;
public boolean CHECK_FOR_DIFFERENT_LANGUAGES_IN_COMMENTS;
public boolean CHECK_FOR_DIFFERENT_LANGUAGES_IN_ANY_OTHER_WORD = true;
public boolean CHECK_FOR_FILES_CONTAINING_BOM;
@Override
@@ -51,40 +59,63 @@ public class NonAsciiCharactersInspection extends LocalInspectionTool {
@NotNull
@Override
public PsiElementVisitor buildVisitor(@NotNull ProblemsHolder holder, boolean isOnTheFly, @NotNull LocalInspectionToolSession session) {
if (!isFileWorthIt(session.getFile())) return PsiElementVisitor.EMPTY_VISITOR;
PsiFile file = session.getFile();
if (!isFileWorthIt(file)) return PsiElementVisitor.EMPTY_VISITOR;
SyntaxHighlighter syntaxHighlighter = SyntaxHighlighterFactory.getSyntaxHighlighter(file.getFileType(), file.getProject(), file.getVirtualFile());
return new PsiElementVisitor() {
@Override
public void visitElement(@NotNull PsiElement element) {
if (CHECK_FOR_NOT_ASCII_IDENTIFIER_NAME || CHECK_FOR_DIFFERENT_LANGUAGES_IN_IDENTIFIER_NAME) {
PsiElement parent = element.getParent();
PsiElement identifier;
if (parent instanceof PsiNameIdentifierOwner &&
(identifier = ((PsiNameIdentifierOwner)parent).getNameIdentifier()) != null) {
// Groovy has this twisted PSI where method.geNameIdentifier() is some random light element
String text = element.getText();
if (identifier == element || text.equals(identifier.getText())) {
if (CHECK_FOR_NOT_ASCII_IDENTIFIER_NAME) {
checkAscii(element, text, holder);
if (!(element instanceof LeafPsiElement)
// optimization: ignore very frequent white space element
|| element instanceof PsiWhiteSpace) {
return;
}
PsiElementKind kind = getKind(element, syntaxHighlighter);
TextRange valueRange = null; // the range inside element with the actual contents with quotes/comment prefixes stripped out
switch (kind) {
case STRING:
if (CHECK_FOR_NOT_ASCII_STRING_LITERAL || CHECK_FOR_DIFFERENT_LANGUAGES_IN_STRING) {
String text = element.getText();
valueRange = StringUtil.isQuotedString(text) ? new TextRange(1, text.length() - 1) : null;
if (CHECK_FOR_DIFFERENT_LANGUAGES_IN_STRING) {
reportMixedLanguages(element, text, holder, valueRange);
}
if (CHECK_FOR_DIFFERENT_LANGUAGES_IN_IDENTIFIER_NAME) {
checkSameLanguage(element, text, holder);
if (CHECK_FOR_NOT_ASCII_STRING_LITERAL) {
reportNonAsciiRange(element, text, holder, valueRange);
}
}
}
}
if (CHECK_FOR_NOT_ASCII_COMMENT) {
if (element instanceof PsiComment) {
checkAsciiRange(element, element.getText(), holder, Kind.COMMENT);
}
}
if (element instanceof PsiLiteralValue) {
String text = element.getText();
if (CHECK_FOR_NOT_ASCII_STRING_LITERAL) {
checkAsciiRange(element, text, holder, Kind.LITERAL);
}
if (CHECK_FOR_DIFFERENT_LANGUAGES_IN_STRING) {
checkSameLanguage(element, text, holder);
}
break;
case IDENTIFIER:
if (CHECK_FOR_NOT_ASCII_IDENTIFIER_NAME) {
reportNonAsciiRange(element, element.getText(), holder, null);
}
if (CHECK_FOR_DIFFERENT_LANGUAGES_IN_IDENTIFIER_NAME) {
reportMixedLanguages(element, element.getText(), holder, null);
}
break;
case COMMENT:
if (CHECK_FOR_NOT_ASCII_COMMENT || CHECK_FOR_DIFFERENT_LANGUAGES_IN_COMMENTS) {
String text = element.getText();
valueRange = getCommentRange(element, text);
if (CHECK_FOR_NOT_ASCII_COMMENT) {
reportNonAsciiRange(element, text, holder, valueRange);
}
if (CHECK_FOR_DIFFERENT_LANGUAGES_IN_COMMENTS) {
reportMixedLanguages(element, text, holder, valueRange);
}
}
break;
case OTHER:
if (CHECK_FOR_NOT_ASCII_IN_ANY_OTHER_WORD) {
String text = element.getText();
reportNonAsciiRange(element, text, holder, valueRange);
}
if (CHECK_FOR_DIFFERENT_LANGUAGES_IN_ANY_OTHER_WORD) {
String text = element.getText();
reportMixedLanguages(element, text, holder, valueRange);
}
break;
}
}
@@ -92,28 +123,72 @@ public class NonAsciiCharactersInspection extends LocalInspectionTool {
public void visitFile(@NotNull PsiFile file) {
super.visitFile(file);
if (CHECK_FOR_FILES_CONTAINING_BOM) {
if (file.getViewProvider().getBaseLanguage() != file.getLanguage()) {
// don't warn multiple times on files which have multiple views like PHP and JSP
return;
}
VirtualFile virtualFile = file.getVirtualFile();
byte[] bom = virtualFile == null ? null : virtualFile.getBOM();
if (bom != null) {
String hex = IntStream.range(0, bom.length)
.map(i -> bom[i])
.mapToObj(b -> StringUtil.toUpperCase(Integer.toString(b & 0x00ff, 16)))
.collect(Collectors.joining());
Charset charsetFromBOM = CharsetToolkit.guessFromBOM(bom);
final String signature = charsetFromBOM == null
? ""
: LangBundle.message("inspection.message.charset.signature", charsetFromBOM.displayName());
holder.registerProblem(file, LangBundle.message("inspection.message.file.contains.bom", hex, signature));
}
checkBOM(file, holder);
}
}
};
}
// null means natural range
private static TextRange getCommentRange(@NotNull PsiElement comment, @NotNull String text) {
Language language = comment.getLanguage();
Commenter commenter = LanguageCommenters.INSTANCE.forLanguage(language);
if (commenter == null) {
return null;
}
for (String prefix : commenter.getLineCommentPrefixes()) {
if (StringUtil.startsWith(text, prefix)) {
return new TextRange(prefix.length(), text.length());
}
}
String blockCommentPrefix = commenter.getBlockCommentPrefix();
if (blockCommentPrefix != null && StringUtil.startsWith(text, blockCommentPrefix)) {
String suffix = commenter.getBlockCommentSuffix();
int endOffset = text.length() - (suffix != null && StringUtil.endsWith(text, blockCommentPrefix.length(), text.length(), suffix) ? suffix.length() : 0);
return new TextRange(blockCommentPrefix.length(), endOffset);
}
return null;
}
private static void checkBOM(@NotNull PsiFile file, @NotNull ProblemsHolder holder) {
if (file.getViewProvider().getBaseLanguage() != file.getLanguage()) {
// don't warn multiple times on files which have multiple views like PHP and JSP
return;
}
VirtualFile virtualFile = file.getVirtualFile();
byte[] bom = virtualFile == null ? null : virtualFile.getBOM();
if (bom != null) {
String hex = IntStream.range(0, bom.length)
.map(i -> bom[i])
.mapToObj(b -> StringUtil.toUpperCase(Integer.toString(b & 0x00ff, 16)))
.collect(Collectors.joining());
Charset charsetFromBOM = CharsetToolkit.guessFromBOM(bom);
final String signature = charsetFromBOM == null
? ""
: CodeInsightBundle.message("non.ascii.chars.inspection.message.charset.signature", charsetFromBOM.displayName());
holder.registerProblem(file, CodeInsightBundle.message("non.ascii.chars.inspection.message.file.contains.bom", hex, signature));
}
}
// if element is an identifier, return its text (its non-trivial in case of Groovy)
private static boolean isIdentifier(@NotNull PsiElement element) {
if (element instanceof ForeignLeafPsiElement) return false;
PsiElement parent = element.getParent();
PsiElement identifier;
if (parent instanceof PsiNameIdentifierOwner &&
(identifier = ((PsiNameIdentifierOwner)parent).getNameIdentifier()) != null) {
// Groovy has this twisted PSI where method.getNameIdentifier() is some random light element
String text = element.getText();
return identifier == element || text.equals(identifier.getText());
}
// or it maybe the reference name
if (parent instanceof PsiReference) {
PsiElement refElement = ((PsiReference)parent).getElement();
return refElement == parent || refElement == element;
}
return false;
}
private static boolean isFileWorthIt(@NotNull PsiFile file) {
if (InjectedLanguageManager.getInstance(file.getProject()).isInjectedFragment(file)) return false;
VirtualFile virtualFile = file.getVirtualFile();
@@ -126,38 +201,91 @@ public class NonAsciiCharactersInspection extends LocalInspectionTool {
return !(charset instanceof Native2AsciiCharset);
}
private static void checkSameLanguage(@NotNull PsiElement element, @NotNull String text, @NotNull ProblemsHolder holder) {
Set<Character.UnicodeScript> scripts = text.codePoints()
.mapToObj(Character.UnicodeScript::of)
.filter(script -> !script.equals(Character.UnicodeScript.COMMON))
.collect(Collectors.toSet());
if (scripts.size() > 1) {
List<Character.UnicodeScript> list = new ArrayList<>(scripts);
Collections.sort(list); // a little bit of stability
holder.registerProblem(element, LangBundle.message("inspection.message.symbols.from.different.languages.found", list),
ProblemHighlightType.GENERIC_ERROR_OR_WARNING);
private static void reportMixedLanguages(@NotNull PsiElement element,
@NotNull String text,
@NotNull ProblemsHolder holder,
@Nullable("null means natural range") TextRange elementRange) {
Character.UnicodeScript first = null;
Character.UnicodeScript second = null;
int i;
int codePoint = -1;
int endOffset = elementRange == null ? text.length() : elementRange.getEndOffset();
int startOffset = elementRange == null ? 0 : elementRange.getStartOffset();
for (i = startOffset; i < endOffset; i++) {
codePoint = text.codePointAt(i);
Character.UnicodeScript currentScript = Character.UnicodeScript.of(codePoint);
if (ignoreScript(currentScript)) {
if (i == startOffset) startOffset++;
continue; // ignore '123.(&$'...
}
second = currentScript;
if (first == null) {
first = second;
}
else if (first != second) {
break;
}
// advance to the next codepoint
if (Character.isLowSurrogate((char)codePoint)) {
i++;
}
}
if (first == null || first == second) {
return;
}
// found two scripts
// now [startOffset..i) are of 'first' script
int j;
for (j = Character.isLowSurrogate((char)codePoint) ? i + 1 : i; j < endOffset; j++) {
codePoint = text.codePointAt(j);
Character.UnicodeScript currentScript = Character.UnicodeScript.of(codePoint);
if (ignoreScript(currentScript)) continue;
if (currentScript != second) {
break;
}
// advance to the next codepoint
if (Character.isLowSurrogate((char)codePoint)) {
j++;
}
}
// ignore trailing COMMON script characters
for (; j > i; j--) {
codePoint = text.codePointAt(j-1);
if (!ignoreScript(Character.UnicodeScript.of(codePoint))) break;
}
// now [i..j) are of 'second' script
// try to report the range which is the least latin
TextRange toReport;
if (first == Character.UnicodeScript.LATIN) {
toReport = new TextRange(i, j);
}
else {
toReport = new TextRange(startOffset, i);
Character.UnicodeScript t = second;
second = first;
first = t;
}
holder.registerProblem(element, toReport, CodeInsightBundle.message("non.ascii.chars.inspection.message.symbols.from.different.languages.found", second, first));
}
private static void checkAscii(@NotNull PsiElement element, @NotNull String text, @NotNull ProblemsHolder holder) {
if (!IOUtil.isAscii(text)) {
holder.registerProblem(element, LangBundle.message("inspection.message.non.ascii.characters.in", Kind.IDENTIFIER.ordinal()));
}
private static boolean ignoreScript(@NotNull Character.UnicodeScript script) {
return script == Character.UnicodeScript.COMMON || script == Character.UnicodeScript.INHERITED;
}
private static void checkAsciiRange(@NotNull PsiElement element,
@NotNull String text,
@NotNull ProblemsHolder holder,
@NotNull Kind kind) {
private static void reportNonAsciiRange(@NotNull PsiElement element,
@NotNull String text,
@NotNull ProblemsHolder holder,
@Nullable("null means natural range") TextRange elementRange) {
int errorCount = 0;
int start = -1;
for (int i = 0; i <= text.length(); i++) {
char c = i >= text.length() ? 0 : text.charAt(i);
if (i == text.length() || c<128) {
int startOffset = elementRange == null ? 0 : elementRange.getStartOffset();
int endOffset = elementRange == null ? text.length() : elementRange.getEndOffset();
for (int i = startOffset; i <= endOffset; i++) {
char c = i >= endOffset ? 0 : text.charAt(i);
if (i == endOffset || c < 128) {
if (start != -1) {
TextRange range = new TextRange(start, i);
holder.registerProblem(element, range, LangBundle.message("inspection.message.non.ascii.characters.in", kind.ordinal()));
holder.registerProblem(element, range, CodeInsightBundle.message("non.ascii.chars.inspection.message.non.ascii.characters"));
start = -1;
//do not report too many errors
if (errorCount++ > 200) break;
@@ -175,7 +303,19 @@ public class NonAsciiCharactersInspection extends LocalInspectionTool {
return new NonAsciiCharactersInspectionFormUi(this).getPanel();
}
private enum Kind {
IDENTIFIER, COMMENT, LITERAL
enum PsiElementKind { IDENTIFIER, STRING, COMMENT, OTHER}
@NotNull
private static PsiElementKind getKind(@NotNull PsiElement element, SyntaxHighlighter syntaxHighlighter) {
TextAttributesKey[] keys;
if (element.getParent() instanceof PsiLiteralValue || ChunkExtractor.isHighlightedAsString(keys = syntaxHighlighter.getTokenHighlights(((LeafPsiElement)element).getElementType()))) {
return PsiElementKind.STRING;
}
if (isIdentifier(element)) {
return PsiElementKind.IDENTIFIER;
}
if (element instanceof PsiComment || ChunkExtractor.isHighlightedAsComment(keys)) {
return PsiElementKind.COMMENT;
}
return PsiElementKind.OTHER;
}
}
@@ -7,67 +7,101 @@ import com.intellij.util.ui.CheckBox
class NonAsciiCharactersInspectionFormUi(entry: InspectionProfileEntry) {
val panel = panel {
buttonsGroup(CodeInsightBundle.message("non.ascii.warn.of")) {
row {
cell(
CheckBox(
CodeInsightBundle.message("non.ascii.chars.inspection.option.files.containing.bom.checkbox"),
entry,
"CHECK_FOR_FILES_CONTAINING_BOM"
)
)
}
buttonsGroup(CodeInsightBundle.message("non.ascii.chars.inspection.non.ascii.top.label")) {
row {
cell(
CheckBox(
CodeInsightBundle.message("checkbox.non.ascii.option.characters.in.identifiers"),
CodeInsightBundle.message("non.ascii.chars.inspection.option.characters.in.identifiers.checkbox"),
entry,
"CHECK_FOR_NOT_ASCII_IDENTIFIER_NAME"
)
).comment(CodeInsightBundle.message("label.non.ascii.chars.example.characters.in.identifiers"))
)
comment(CodeInsightBundle.message("non.ascii.chars.inspection.example.characters.in.identifiers.label"))
}
row {
cell(
CheckBox(
CodeInsightBundle.message("checkbox.non.ascii.option.characters.in.comments"),
CodeInsightBundle.message("non.ascii.chars.inspection.option.characters.in.strings.checkbox"),
entry,
"CHECK_FOR_NOT_ASCII_STRING_LITERAL"
)
).comment(CodeInsightBundle.message("label.non.ascii.chars.example.characters.in.comments"))
)
comment(CodeInsightBundle.message("non.ascii.chars.inspection.example.characters.in.strings.label"))
}
row {
cell(
CheckBox(
CodeInsightBundle.message("checkbox.non.ascii.option.characters.in.strings"),
CodeInsightBundle.message("non.ascii.chars.inspection.option.characters.in.comments.checkbox"),
entry,
"CHECK_FOR_NOT_ASCII_COMMENT"
)
).comment(CodeInsightBundle.message("label.non.ascii.chars.example.characters.in.strings"))
)
comment(CodeInsightBundle.message("non.ascii.chars.inspection.example.characters.in.comments.label"))
}
row {
cell(
CheckBox(
CodeInsightBundle.message("checkbox.non.ascii.option.different.languages.in.identifiers"),
CodeInsightBundle.message("non.ascii.chars.inspection.option.characters.in.any.other.word.checkbox"),
entry,
"CHECK_FOR_NOT_ASCII_IN_ANY_OTHER_WORD"
)
)
comment(CodeInsightBundle.message("non.ascii.chars.inspection.example.characters.in.any.other.word.label"))
}
}
buttonsGroup(CodeInsightBundle.message("non.ascii.chars.inspection.mixed.chars.top.label")) {
row {
cell(
CheckBox(
CodeInsightBundle.message("non.ascii.chars.inspection.option.mixed.languages.in.identifiers.checkbox"),
entry,
"CHECK_FOR_DIFFERENT_LANGUAGES_IN_IDENTIFIER_NAME"
)
).comment(CodeInsightBundle.message("label.non.ascii.chars.example.different.languages.in.identifiers"))
)
comment(CodeInsightBundle.message("non.ascii.chars.inspection.example.mixed.languages.in.identifiers.label"))
}
row {
cell(
CheckBox(
CodeInsightBundle.message("checkbox.non.ascii.option.different.languages.in.string"),
CodeInsightBundle.message("non.ascii.chars.inspection.option.mixed.languages.in.strings.checkbox"),
entry,
"CHECK_FOR_DIFFERENT_LANGUAGES_IN_STRING"
)
).comment(CodeInsightBundle.message("label.non.ascii.chars.example.different.languages.in.string"))
)
comment(CodeInsightBundle.message("non.ascii.chars.inspection.example.mixed.languages.in.string.label"))
}
row {
cell(
CheckBox(
CodeInsightBundle.message("checkbox.non.ascii.option.files.containing.bom"),
entry,
"CHECK_FOR_FILES_CONTAINING_BOM"
)
).comment(CodeInsightBundle.message("label.non.ascii.chars.example.files.containing.bom"))
CodeInsightBundle.message("non.ascii.chars.inspection.option.mixed.languages.in.comments.checkbox"),
entry,
"CHECK_FOR_DIFFERENT_LANGUAGES_IN_COMMENTS"
)
)
comment(CodeInsightBundle.message("non.ascii.chars.inspection.example.mixed.languages.in.comments.label"))
}
row {
cell(
CheckBox(
CodeInsightBundle.message("non.ascii.chars.inspection.option.mixed.languages.in.any.other.word.checkbox"),
entry,
"CHECK_FOR_DIFFERENT_LANGUAGES_IN_ANY_OTHER_WORD"
)
)
comment(CodeInsightBundle.message("non.ascii.chars.inspection.example.mixed.languages.in.any.other.word.label"))
}
}
}