diff --git a/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/nonAsciiCharacters/Groovy.groovy b/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/nonAsciiCharacters/Groovy.groovy index 6b81f6869e01..a458a5124a24 100644 --- a/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/nonAsciiCharacters/Groovy.groovy +++ b/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/nonAsciiCharacters/Groovy.groovy @@ -1,9 +1,9 @@ class X { - int Ж = 0; - class InnerП {} - // commentжп 234 - String s = "12л3орыва0"; - void жжж() { - жжж(); + int Ж = 0; + class InnerП {} + // commentжп 234 + String s = "12лTOорыва0"; + void жжж() { + жжж(); } } \ No newline at end of file diff --git a/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/nonAsciiCharacters/MixedLanguagesJavaInAnyWord.java b/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/nonAsciiCharacters/MixedLanguagesJavaInAnyWord.java new file mode 100644 index 000000000000..2981bf1232c5 --- /dev/null +++ b/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/nonAsciiCharacters/MixedLanguagesJavaInAnyWord.java @@ -0,0 +1,13 @@ +class X { + int Ж = 0; + class InnerП {} + // commentжп 234 + String sameLang = "12л3орыва0"; + String mixed = "12чеto3"; + void жжж() { + жжж(); + String core = "Сore"; + System.out.println("Сore"); + String u = "\5\0\51\2\3\0\136\2\21\0\33\2\65\0\20\2\u0200\0\u19b6\2"; // ignore slash u + } +} \ No newline at end of file diff --git a/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/nonAsciiCharacters/MixedLanguagesJavaInVariousContexts.java b/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/nonAsciiCharacters/MixedLanguagesJavaInVariousContexts.java new file mode 100644 index 000000000000..2981bf1232c5 --- /dev/null +++ b/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/nonAsciiCharacters/MixedLanguagesJavaInVariousContexts.java @@ -0,0 +1,13 @@ +class X { + int Ж = 0; + class InnerП {} + // commentжп 234 + String sameLang = "12л3орыва0"; + String mixed = "12чеto3"; + void жжж() { + жжж(); + String core = "Сore"; + System.out.println("Сore"); + String u = "\5\0\51\2\3\0\136\2\21\0\33\2\65\0\20\2\u0200\0\u19b6\2"; // ignore slash u + } +} \ No newline at end of file diff --git a/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/nonAsciiCharacters/MixedLanguagesXMLInAnyWord.xml b/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/nonAsciiCharacters/MixedLanguagesXMLInAnyWord.xml new file mode 100644 index 000000000000..c7ea407844c7 --- /dev/null +++ b/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/nonAsciiCharacters/MixedLanguagesXMLInAnyWord.xml @@ -0,0 +1,6 @@ +Ш> + блабла + Ы attr="ИИИИ23" attr2="ИИИfoo"> + блаfoo + Ы> +Ш> \ No newline at end of file diff --git a/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/nonAsciiCharacters/MixedLanguagesXMLInAnyWordExceptString.xml b/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/nonAsciiCharacters/MixedLanguagesXMLInAnyWordExceptString.xml new file mode 100644 index 000000000000..307a7e14e242 --- /dev/null +++ b/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/nonAsciiCharacters/MixedLanguagesXMLInAnyWordExceptString.xml @@ -0,0 +1,6 @@ +Ш> + блабла + Ы attr="ИИИИ23" attr2="ИИИfoo"> + блаfoo + Ы> +Ш> \ No newline at end of file diff --git a/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/nonAsciiCharacters/NotAsciiJavaInAnyWord.java b/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/nonAsciiCharacters/NotAsciiJavaInAnyWord.java new file mode 100644 index 000000000000..e36f5781c74f --- /dev/null +++ b/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/nonAsciiCharacters/NotAsciiJavaInAnyWord.java @@ -0,0 +1,12 @@ +class X { + int Ж = 0; + class InnerП {} + // commentжп 234 + String s = "12л3орыва0"; + void жжж() { + жжж(); + String s = "Сore"; + System.out.println("Сore"); + String u = "\5\0\51\2\3\0\136\2\21\0\33\2\65\0\20\2\u0200\0\u19b6\2"; // ignore slash u + } +} \ No newline at end of file diff --git a/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/nonAsciiCharacters/NotAsciiJavaInVariousContexts.java b/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/nonAsciiCharacters/NotAsciiJavaInVariousContexts.java new file mode 100644 index 000000000000..e36f5781c74f --- /dev/null +++ b/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/nonAsciiCharacters/NotAsciiJavaInVariousContexts.java @@ -0,0 +1,12 @@ +class X { + int Ж = 0; + class InnerП {} + // commentжп 234 + String s = "12л3орыва0"; + void жжж() { + жжж(); + String s = "Сore"; + System.out.println("Сore"); + String u = "\5\0\51\2\3\0\136\2\21\0\33\2\65\0\20\2\u0200\0\u19b6\2"; // ignore slash u + } +} \ No newline at end of file diff --git a/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/nonAsciiCharacters/Simple.java b/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/nonAsciiCharacters/Simple.java deleted file mode 100644 index 816a2eb2c236..000000000000 --- a/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/nonAsciiCharacters/Simple.java +++ /dev/null @@ -1,12 +0,0 @@ -class X { - int Ж = 0; - class InnerП {} - // commentжп 234 - String s = "12л3орыва0"; - void жжж() { - жжж(); - String s = "Сore"; - System.out.println("Сore"); - String u = "\5\0\51\2\3\0\136\2\21\0\33\2\65\0\20\2\u0200\0\u19b6\2"; // ignore slash u - } -} \ No newline at end of file diff --git a/java/java-tests/testSrc/com/intellij/java/codeInsight/daemon/NonAsciiCharactersTest.java b/java/java-tests/testSrc/com/intellij/java/codeInsight/daemon/NonAsciiCharactersTest.java index 10d68e137b40..6ca6757626a7 100644 --- a/java/java-tests/testSrc/com/intellij/java/codeInsight/daemon/NonAsciiCharactersTest.java +++ b/java/java-tests/testSrc/com/intellij/java/codeInsight/daemon/NonAsciiCharactersTest.java @@ -9,17 +9,17 @@ import org.jetbrains.annotations.NotNull; public class NonAsciiCharactersTest extends DaemonAnalyzerTestCase { private static final String BASE_PATH = "/codeInsight/daemonCodeAnalyzer/nonAsciiCharacters"; + private NonAsciiCharactersInspection myInspection = new NonAsciiCharactersInspection(); @Override protected LocalInspectionTool @NotNull [] configureLocalInspectionTools() { - NonAsciiCharactersInspection inspection = new NonAsciiCharactersInspection(); - inspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_IDENTIFIER_NAME = true; - inspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_STRING = true; - inspection.CHECK_FOR_NOT_ASCII_IDENTIFIER_NAME = true; - inspection.CHECK_FOR_NOT_ASCII_COMMENT = true; - inspection.CHECK_FOR_NOT_ASCII_STRING_LITERAL = true; - inspection.CHECK_FOR_FILES_CONTAINING_BOM = true; - return new LocalInspectionTool[]{inspection}; + return new LocalInspectionTool[]{myInspection}; + } + + @Override + protected void tearDown() throws Exception { + myInspection = null; + super.tearDown(); } private void doTest(String extension) throws Exception { @@ -27,10 +27,92 @@ public class NonAsciiCharactersTest extends DaemonAnalyzerTestCase { UIUtil.dispatchAllInvocationEvents(); } - public void testSimple() throws Exception { + public void testNotAsciiJavaInVariousContexts() throws Exception { + myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_ANY_OTHER_WORD = false; + myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_IDENTIFIER_NAME = false; + myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_COMMENTS = false; + myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_STRING = false; + myInspection.CHECK_FOR_NOT_ASCII_IN_ANY_OTHER_WORD = false; + myInspection.CHECK_FOR_NOT_ASCII_IDENTIFIER_NAME = true; + myInspection.CHECK_FOR_NOT_ASCII_COMMENT = true; + myInspection.CHECK_FOR_NOT_ASCII_STRING_LITERAL = true; + myInspection.CHECK_FOR_FILES_CONTAINING_BOM = false; doTest(".java"); } + public void testNotAsciiJavaInAnyWord() throws Exception { + myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_ANY_OTHER_WORD = false; + myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_IDENTIFIER_NAME = false; + myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_COMMENTS = false; + myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_STRING = false; + myInspection.CHECK_FOR_NOT_ASCII_IN_ANY_OTHER_WORD = true; + myInspection.CHECK_FOR_NOT_ASCII_IDENTIFIER_NAME = true; + myInspection.CHECK_FOR_NOT_ASCII_COMMENT = true; + myInspection.CHECK_FOR_NOT_ASCII_STRING_LITERAL = true; + myInspection.CHECK_FOR_FILES_CONTAINING_BOM = false; + doTest(".java"); + } + + public void testMixedLanguagesJavaInAnyWord() throws Exception { + myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_ANY_OTHER_WORD = true; + myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_IDENTIFIER_NAME = true; + myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_COMMENTS = true; + myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_STRING = true; + myInspection.CHECK_FOR_NOT_ASCII_IN_ANY_OTHER_WORD = false; + myInspection.CHECK_FOR_NOT_ASCII_IDENTIFIER_NAME = false; + myInspection.CHECK_FOR_NOT_ASCII_COMMENT = false; + myInspection.CHECK_FOR_NOT_ASCII_STRING_LITERAL = false; + myInspection.CHECK_FOR_FILES_CONTAINING_BOM = false; + doTest(".java"); + } + + public void testMixedLanguagesXMLInAnyWord() throws Exception { + myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_ANY_OTHER_WORD = true; + myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_IDENTIFIER_NAME = true; + myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_COMMENTS = true; + myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_STRING = true; + myInspection.CHECK_FOR_NOT_ASCII_IN_ANY_OTHER_WORD = false; + myInspection.CHECK_FOR_NOT_ASCII_IDENTIFIER_NAME = false; + myInspection.CHECK_FOR_NOT_ASCII_COMMENT = false; + myInspection.CHECK_FOR_NOT_ASCII_STRING_LITERAL = false; + myInspection.CHECK_FOR_FILES_CONTAINING_BOM = false; + doTest(".xml"); + } + public void testMixedLanguagesXMLInAnyWordExceptString() throws Exception { + myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_ANY_OTHER_WORD = true; + myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_IDENTIFIER_NAME = true; + myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_COMMENTS = true; + myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_STRING = false; + myInspection.CHECK_FOR_NOT_ASCII_IN_ANY_OTHER_WORD = false; + myInspection.CHECK_FOR_NOT_ASCII_IDENTIFIER_NAME = false; + myInspection.CHECK_FOR_NOT_ASCII_COMMENT = false; + myInspection.CHECK_FOR_NOT_ASCII_STRING_LITERAL = false; + myInspection.CHECK_FOR_FILES_CONTAINING_BOM = false; + doTest(".xml"); + } + + public void testMixedLanguagesJavaInVariousContexts() throws Exception { + myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_ANY_OTHER_WORD = false; + myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_IDENTIFIER_NAME = true; + myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_COMMENTS = true; + myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_STRING = true; + myInspection.CHECK_FOR_NOT_ASCII_IN_ANY_OTHER_WORD = false; + myInspection.CHECK_FOR_NOT_ASCII_IDENTIFIER_NAME = false; + myInspection.CHECK_FOR_NOT_ASCII_COMMENT = false; + myInspection.CHECK_FOR_NOT_ASCII_STRING_LITERAL = false; + myInspection.CHECK_FOR_FILES_CONTAINING_BOM = false; + doTest(".java"); + } + public void testGroovy() throws Exception { + myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_ANY_OTHER_WORD = true; + myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_IDENTIFIER_NAME = true; + myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_COMMENTS = true; + myInspection.CHECK_FOR_DIFFERENT_LANGUAGES_IN_STRING = true; + myInspection.CHECK_FOR_NOT_ASCII_IN_ANY_OTHER_WORD = true; + myInspection.CHECK_FOR_NOT_ASCII_IDENTIFIER_NAME = true; + myInspection.CHECK_FOR_NOT_ASCII_COMMENT = true; + myInspection.CHECK_FOR_NOT_ASCII_STRING_LITERAL = true; + myInspection.CHECK_FOR_FILES_CONTAINING_BOM = false; doTest(".groovy"); } } diff --git a/platform/lang-api/resources/messages/CodeInsightBundle.properties b/platform/lang-api/resources/messages/CodeInsightBundle.properties index 2367c5a0cb7e..a5e8bab79091 100644 --- a/platform/lang-api/resources/messages/CodeInsightBundle.properties +++ b/platform/lang-api/resources/messages/CodeInsightBundle.properties @@ -512,19 +512,31 @@ actions.on.save.page.popup.title.arrangement.settings=Open Arrangement Settings actions.on.save.page.checkbox.run.code.cleanup=Run code cleanup actions.on.save.page.code.cleanup.comment=Applies fixes from the code cleanup inspections actions.on.save.page.link.configure.inspections=Configure inspections... -non.ascii.warn.of=Warn of: -label.non.ascii.chars.example.files.containing.bom=E.g. files starting with FEFF -label.non.ascii.chars.example.characters.in.comments=E.g.: // hello \u10D2\u10D4\u10DC\u10D0\u10EA\u10D5\u10D0\u10DA\u10D8 -label.non.ascii.chars.example.characters.in.identifiers=E.g.: int \u041F\u043E\u043B\u0435 = 0; -label.non.ascii.chars.example.characters.in.strings=E.g.: println("\u10D2\u10D4\u10DC\u10D0\u10EA\u10D5\u10D0\u10DA\u10D8"); -label.non.ascii.chars.example.different.languages.in.identifiers=E.g.: void print\u0415\u0440\u0443\u043D\u0434\u0430() -label.non.ascii.chars.example.different.languages.in.string=E.g.: foo("\u0421ore"); //cyrillic \u0421 -checkbox.non.ascii.option.different.languages.in.identifiers=Different languages in identifiers -checkbox.non.ascii.option.different.languages.in.string=Different languages in strings -checkbox.non.ascii.option.files.containing.bom=Files containing BOM -checkbox.non.ascii.option.characters.in.identifiers=Non-ASCII characters in identifiers -checkbox.non.ascii.option.characters.in.comments=Non-ASCII characters in comments -checkbox.non.ascii.option.characters.in.strings=Non-ASCII characters in strings + +non.ascii.chars.inspection.non.ascii.top.label=Warn of non-ASCII characters in: +non.ascii.chars.inspection.option.files.containing.bom.checkbox=Warn of files containing BOM (e.g., files starting with 'FEFF' bytes) +non.ascii.chars.inspection.option.characters.in.any.other.word.checkbox=any other word +non.ascii.chars.inspection.example.characters.in.any.other.word.label=(e.g.: \\<\u0422\u042D\u0413-\u0428\u041C\u042D\u0413\\>0\\</\u0422\u042D\u0413-\u0428\u041C\u042D\u0413\\>) +non.ascii.chars.inspection.option.characters.in.identifiers.checkbox=identifiers +non.ascii.chars.inspection.example.characters.in.identifiers.label=(e.g.: int \u041F\u043E\u043B\u0435 = 0;) +non.ascii.chars.inspection.option.characters.in.comments.checkbox=comments +non.ascii.chars.inspection.example.characters.in.comments.label=(e.g.: // \u10D2\u10D4\u10DC\u10D0\u10EA\u10D5\u10D0\u10DA\u10D8) +non.ascii.chars.inspection.option.characters.in.strings.checkbox=strings +non.ascii.chars.inspection.example.characters.in.strings.label=(e.g.: println("\u10D2\u10D4\u10DC\u10D0\u10EA\u10D5\u10D0\u10DA\u10D8")) +non.ascii.chars.inspection.mixed.chars.top.label=Warn of mixed language characters in: +non.ascii.chars.inspection.option.mixed.languages.in.any.other.word.checkbox=any other word +non.ascii.chars.inspection.example.mixed.languages.in.any.other.word.label=(e.g.: \\<t\u0430g\\>0\\</t\u0430g\\> (cyrillic '\u0430')) +non.ascii.chars.inspection.option.mixed.languages.in.identifiers.checkbox=identifiers +non.ascii.chars.inspection.example.mixed.languages.in.identifiers.label=(e.g.: void print\u0415\u0440\u0443\u043D\u0434\u0430()) +non.ascii.chars.inspection.option.mixed.languages.in.comments.checkbox=comments +non.ascii.chars.inspection.example.mixed.languages.in.comments.label=(e.g.: //c\u043Eunts bugs (cyrillic 'o')) +non.ascii.chars.inspection.option.mixed.languages.in.strings.checkbox=strings +non.ascii.chars.inspection.example.mixed.languages.in.string.label=(e.g.: foo("\u0421ore"); //cyrillic '\u0421' ) +non.ascii.chars.inspection.message.file.contains.bom=File contains BOM: ''{0}''{1} +non.ascii.chars.inspection.message.charset.signature=\ (charset ''{0}'' signature) +non.ascii.chars.inspection.message.non.ascii.characters=Non-ASCII characters +non.ascii.chars.inspection.message.symbols.from.different.languages.found=Mixed languages: {0} symbols found in {1} word + label.file.included.into=Included into: checkbox.reformat.project.file.masks=&File mask(s) checkbox.reformat.project.include.subdirectories=&Include subdirectories diff --git a/platform/lang-api/resources/messages/LangBundle.properties b/platform/lang-api/resources/messages/LangBundle.properties index 27079d4bf6ca..f139b7a90fd7 100644 --- a/platform/lang-api/resources/messages/LangBundle.properties +++ b/platform/lang-api/resources/messages/LangBundle.properties @@ -100,10 +100,6 @@ dialog.message.live.template.with.empty.abbreviation=A live template with an emp dialog.message.live.template.with.empty.text=A live template ''{0}'' with an empty text has been found in ''{1}'' group, such live templates make no sense dialog.message.duplicate.live.templates.in.group=Duplicate `{0}` live templates in {1} group popup.advertisement.press.or.to.replace=Press {0} or {1} to replace -inspection.message.file.contains.bom=File contains BOM: ''{0}''{1} -inspection.message.charset.signature=\ (charset ''{0}'' signature) -inspection.message.non.ascii.characters.in=Non-ASCII characters in {0, choice, 0#an identifier|1#a comment|2#a string literal} -inspection.message.symbols.from.different.languages.found=Symbols from different languages found: {0} hint.text.unfortunately.currently.available.for.batch.mode=Unfortunately ''{0}'' is currently not available for batch mode\n User interaction is required for each problem found progress.title.inspect.code=Running Code Cleanup with profile ''{0}''\u2026 hint.text.can.t.modify.decompiled.code=Can't modify decompiled code diff --git a/platform/lang-impl/src/com/intellij/codeInspection/NonAsciiCharactersInspection.java b/platform/lang-impl/src/com/intellij/codeInspection/NonAsciiCharactersInspection.java index 4ea0f78a784c..8790be06bf77 100644 --- a/platform/lang-impl/src/com/intellij/codeInspection/NonAsciiCharactersInspection.java +++ b/platform/lang-impl/src/com/intellij/codeInspection/NonAsciiCharactersInspection.java @@ -2,26 +2,31 @@ package com.intellij.codeInspection; -import com.intellij.lang.LangBundle; +import com.intellij.codeInsight.CodeInsightBundle; +import com.intellij.lang.Commenter; +import com.intellij.lang.Language; +import com.intellij.lang.LanguageCommenters; import com.intellij.lang.injection.InjectedLanguageManager; import com.intellij.lang.properties.charset.Native2AsciiCharset; +import com.intellij.openapi.editor.colors.TextAttributesKey; import com.intellij.openapi.fileEditor.impl.LoadTextUtil; +import com.intellij.openapi.fileTypes.SyntaxHighlighter; +import com.intellij.openapi.fileTypes.SyntaxHighlighterFactory; import com.intellij.openapi.util.TextRange; import com.intellij.openapi.util.text.StringUtil; import com.intellij.openapi.vfs.CharsetToolkit; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.psi.*; -import com.intellij.util.io.IOUtil; +import com.intellij.psi.impl.source.tree.ForeignLeafPsiElement; +import com.intellij.psi.impl.source.tree.LeafPsiElement; +import com.intellij.usages.ChunkExtractor; import org.jetbrains.annotations.Nls; import org.jetbrains.annotations.NonNls; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import javax.swing.*; import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Set; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -29,9 +34,12 @@ public class NonAsciiCharactersInspection extends LocalInspectionTool { public boolean CHECK_FOR_NOT_ASCII_IDENTIFIER_NAME = true; public boolean CHECK_FOR_NOT_ASCII_STRING_LITERAL; public boolean CHECK_FOR_NOT_ASCII_COMMENT; - + public boolean CHECK_FOR_NOT_ASCII_IN_ANY_OTHER_WORD; + public boolean CHECK_FOR_DIFFERENT_LANGUAGES_IN_IDENTIFIER_NAME = true; public boolean CHECK_FOR_DIFFERENT_LANGUAGES_IN_STRING; + public boolean CHECK_FOR_DIFFERENT_LANGUAGES_IN_COMMENTS; + public boolean CHECK_FOR_DIFFERENT_LANGUAGES_IN_ANY_OTHER_WORD = true; public boolean CHECK_FOR_FILES_CONTAINING_BOM; @Override @@ -51,40 +59,63 @@ public class NonAsciiCharactersInspection extends LocalInspectionTool { @NotNull @Override public PsiElementVisitor buildVisitor(@NotNull ProblemsHolder holder, boolean isOnTheFly, @NotNull LocalInspectionToolSession session) { - if (!isFileWorthIt(session.getFile())) return PsiElementVisitor.EMPTY_VISITOR; + PsiFile file = session.getFile(); + if (!isFileWorthIt(file)) return PsiElementVisitor.EMPTY_VISITOR; + SyntaxHighlighter syntaxHighlighter = SyntaxHighlighterFactory.getSyntaxHighlighter(file.getFileType(), file.getProject(), file.getVirtualFile()); return new PsiElementVisitor() { @Override public void visitElement(@NotNull PsiElement element) { - if (CHECK_FOR_NOT_ASCII_IDENTIFIER_NAME || CHECK_FOR_DIFFERENT_LANGUAGES_IN_IDENTIFIER_NAME) { - PsiElement parent = element.getParent(); - PsiElement identifier; - if (parent instanceof PsiNameIdentifierOwner && - (identifier = ((PsiNameIdentifierOwner)parent).getNameIdentifier()) != null) { - // Groovy has this twisted PSI where method.geNameIdentifier() is some random light element - String text = element.getText(); - if (identifier == element || text.equals(identifier.getText())) { - if (CHECK_FOR_NOT_ASCII_IDENTIFIER_NAME) { - checkAscii(element, text, holder); + if (!(element instanceof LeafPsiElement) + // optimization: ignore very frequent white space element + || element instanceof PsiWhiteSpace) { + return; + } + + PsiElementKind kind = getKind(element, syntaxHighlighter); + TextRange valueRange = null; // the range inside element with the actual contents with quotes/comment prefixes stripped out + switch (kind) { + case STRING: + if (CHECK_FOR_NOT_ASCII_STRING_LITERAL || CHECK_FOR_DIFFERENT_LANGUAGES_IN_STRING) { + String text = element.getText(); + valueRange = StringUtil.isQuotedString(text) ? new TextRange(1, text.length() - 1) : null; + if (CHECK_FOR_DIFFERENT_LANGUAGES_IN_STRING) { + reportMixedLanguages(element, text, holder, valueRange); } - if (CHECK_FOR_DIFFERENT_LANGUAGES_IN_IDENTIFIER_NAME) { - checkSameLanguage(element, text, holder); + if (CHECK_FOR_NOT_ASCII_STRING_LITERAL) { + reportNonAsciiRange(element, text, holder, valueRange); } } - } - } - if (CHECK_FOR_NOT_ASCII_COMMENT) { - if (element instanceof PsiComment) { - checkAsciiRange(element, element.getText(), holder, Kind.COMMENT); - } - } - if (element instanceof PsiLiteralValue) { - String text = element.getText(); - if (CHECK_FOR_NOT_ASCII_STRING_LITERAL) { - checkAsciiRange(element, text, holder, Kind.LITERAL); - } - if (CHECK_FOR_DIFFERENT_LANGUAGES_IN_STRING) { - checkSameLanguage(element, text, holder); - } + break; + case IDENTIFIER: + if (CHECK_FOR_NOT_ASCII_IDENTIFIER_NAME) { + reportNonAsciiRange(element, element.getText(), holder, null); + } + if (CHECK_FOR_DIFFERENT_LANGUAGES_IN_IDENTIFIER_NAME) { + reportMixedLanguages(element, element.getText(), holder, null); + } + break; + case COMMENT: + if (CHECK_FOR_NOT_ASCII_COMMENT || CHECK_FOR_DIFFERENT_LANGUAGES_IN_COMMENTS) { + String text = element.getText(); + valueRange = getCommentRange(element, text); + if (CHECK_FOR_NOT_ASCII_COMMENT) { + reportNonAsciiRange(element, text, holder, valueRange); + } + if (CHECK_FOR_DIFFERENT_LANGUAGES_IN_COMMENTS) { + reportMixedLanguages(element, text, holder, valueRange); + } + } + break; + case OTHER: + if (CHECK_FOR_NOT_ASCII_IN_ANY_OTHER_WORD) { + String text = element.getText(); + reportNonAsciiRange(element, text, holder, valueRange); + } + if (CHECK_FOR_DIFFERENT_LANGUAGES_IN_ANY_OTHER_WORD) { + String text = element.getText(); + reportMixedLanguages(element, text, holder, valueRange); + } + break; } } @@ -92,28 +123,72 @@ public class NonAsciiCharactersInspection extends LocalInspectionTool { public void visitFile(@NotNull PsiFile file) { super.visitFile(file); if (CHECK_FOR_FILES_CONTAINING_BOM) { - if (file.getViewProvider().getBaseLanguage() != file.getLanguage()) { - // don't warn multiple times on files which have multiple views like PHP and JSP - return; - } - VirtualFile virtualFile = file.getVirtualFile(); - byte[] bom = virtualFile == null ? null : virtualFile.getBOM(); - if (bom != null) { - String hex = IntStream.range(0, bom.length) - .map(i -> bom[i]) - .mapToObj(b -> StringUtil.toUpperCase(Integer.toString(b & 0x00ff, 16))) - .collect(Collectors.joining()); - Charset charsetFromBOM = CharsetToolkit.guessFromBOM(bom); - final String signature = charsetFromBOM == null - ? "" - : LangBundle.message("inspection.message.charset.signature", charsetFromBOM.displayName()); - holder.registerProblem(file, LangBundle.message("inspection.message.file.contains.bom", hex, signature)); - } + checkBOM(file, holder); } } }; } + // null means natural range + private static TextRange getCommentRange(@NotNull PsiElement comment, @NotNull String text) { + Language language = comment.getLanguage(); + Commenter commenter = LanguageCommenters.INSTANCE.forLanguage(language); + if (commenter == null) { + return null; + } + for (String prefix : commenter.getLineCommentPrefixes()) { + if (StringUtil.startsWith(text, prefix)) { + return new TextRange(prefix.length(), text.length()); + } + } + String blockCommentPrefix = commenter.getBlockCommentPrefix(); + if (blockCommentPrefix != null && StringUtil.startsWith(text, blockCommentPrefix)) { + String suffix = commenter.getBlockCommentSuffix(); + int endOffset = text.length() - (suffix != null && StringUtil.endsWith(text, blockCommentPrefix.length(), text.length(), suffix) ? suffix.length() : 0); + return new TextRange(blockCommentPrefix.length(), endOffset); + } + return null; + } + + private static void checkBOM(@NotNull PsiFile file, @NotNull ProblemsHolder holder) { + if (file.getViewProvider().getBaseLanguage() != file.getLanguage()) { + // don't warn multiple times on files which have multiple views like PHP and JSP + return; + } + VirtualFile virtualFile = file.getVirtualFile(); + byte[] bom = virtualFile == null ? null : virtualFile.getBOM(); + if (bom != null) { + String hex = IntStream.range(0, bom.length) + .map(i -> bom[i]) + .mapToObj(b -> StringUtil.toUpperCase(Integer.toString(b & 0x00ff, 16))) + .collect(Collectors.joining()); + Charset charsetFromBOM = CharsetToolkit.guessFromBOM(bom); + final String signature = charsetFromBOM == null + ? "" + : CodeInsightBundle.message("non.ascii.chars.inspection.message.charset.signature", charsetFromBOM.displayName()); + holder.registerProblem(file, CodeInsightBundle.message("non.ascii.chars.inspection.message.file.contains.bom", hex, signature)); + } + } + + // if element is an identifier, return its text (its non-trivial in case of Groovy) + private static boolean isIdentifier(@NotNull PsiElement element) { + if (element instanceof ForeignLeafPsiElement) return false; + PsiElement parent = element.getParent(); + PsiElement identifier; + if (parent instanceof PsiNameIdentifierOwner && + (identifier = ((PsiNameIdentifierOwner)parent).getNameIdentifier()) != null) { + // Groovy has this twisted PSI where method.getNameIdentifier() is some random light element + String text = element.getText(); + return identifier == element || text.equals(identifier.getText()); + } + // or it maybe the reference name + if (parent instanceof PsiReference) { + PsiElement refElement = ((PsiReference)parent).getElement(); + return refElement == parent || refElement == element; + } + return false; + } + private static boolean isFileWorthIt(@NotNull PsiFile file) { if (InjectedLanguageManager.getInstance(file.getProject()).isInjectedFragment(file)) return false; VirtualFile virtualFile = file.getVirtualFile(); @@ -126,38 +201,91 @@ public class NonAsciiCharactersInspection extends LocalInspectionTool { return !(charset instanceof Native2AsciiCharset); } - - private static void checkSameLanguage(@NotNull PsiElement element, @NotNull String text, @NotNull ProblemsHolder holder) { - Set scripts = text.codePoints() - .mapToObj(Character.UnicodeScript::of) - .filter(script -> !script.equals(Character.UnicodeScript.COMMON)) - .collect(Collectors.toSet()); - if (scripts.size() > 1) { - List list = new ArrayList<>(scripts); - Collections.sort(list); // a little bit of stability - holder.registerProblem(element, LangBundle.message("inspection.message.symbols.from.different.languages.found", list), - ProblemHighlightType.GENERIC_ERROR_OR_WARNING); + private static void reportMixedLanguages(@NotNull PsiElement element, + @NotNull String text, + @NotNull ProblemsHolder holder, + @Nullable("null means natural range") TextRange elementRange) { + Character.UnicodeScript first = null; + Character.UnicodeScript second = null; + int i; + int codePoint = -1; + int endOffset = elementRange == null ? text.length() : elementRange.getEndOffset(); + int startOffset = elementRange == null ? 0 : elementRange.getStartOffset(); + for (i = startOffset; i < endOffset; i++) { + codePoint = text.codePointAt(i); + Character.UnicodeScript currentScript = Character.UnicodeScript.of(codePoint); + if (ignoreScript(currentScript)) { + if (i == startOffset) startOffset++; + continue; // ignore '123.(&$'... + } + second = currentScript; + if (first == null) { + first = second; + } + else if (first != second) { + break; + } + // advance to the next codepoint + if (Character.isLowSurrogate((char)codePoint)) { + i++; + } } + if (first == null || first == second) { + return; + } + // found two scripts + // now [startOffset..i) are of 'first' script + int j; + for (j = Character.isLowSurrogate((char)codePoint) ? i + 1 : i; j < endOffset; j++) { + codePoint = text.codePointAt(j); + Character.UnicodeScript currentScript = Character.UnicodeScript.of(codePoint); + if (ignoreScript(currentScript)) continue; + if (currentScript != second) { + break; + } + // advance to the next codepoint + if (Character.isLowSurrogate((char)codePoint)) { + j++; + } + } + // ignore trailing COMMON script characters + for (; j > i; j--) { + codePoint = text.codePointAt(j-1); + if (!ignoreScript(Character.UnicodeScript.of(codePoint))) break; + } + // now [i..j) are of 'second' script + // try to report the range which is the least latin + TextRange toReport; + if (first == Character.UnicodeScript.LATIN) { + toReport = new TextRange(i, j); + } + else { + toReport = new TextRange(startOffset, i); + Character.UnicodeScript t = second; + second = first; + first = t; + } + holder.registerProblem(element, toReport, CodeInsightBundle.message("non.ascii.chars.inspection.message.symbols.from.different.languages.found", second, first)); } - private static void checkAscii(@NotNull PsiElement element, @NotNull String text, @NotNull ProblemsHolder holder) { - if (!IOUtil.isAscii(text)) { - holder.registerProblem(element, LangBundle.message("inspection.message.non.ascii.characters.in", Kind.IDENTIFIER.ordinal())); - } + private static boolean ignoreScript(@NotNull Character.UnicodeScript script) { + return script == Character.UnicodeScript.COMMON || script == Character.UnicodeScript.INHERITED; } - private static void checkAsciiRange(@NotNull PsiElement element, - @NotNull String text, - @NotNull ProblemsHolder holder, - @NotNull Kind kind) { + private static void reportNonAsciiRange(@NotNull PsiElement element, + @NotNull String text, + @NotNull ProblemsHolder holder, + @Nullable("null means natural range") TextRange elementRange) { int errorCount = 0; int start = -1; - for (int i = 0; i <= text.length(); i++) { - char c = i >= text.length() ? 0 : text.charAt(i); - if (i == text.length() || c<128) { + int startOffset = elementRange == null ? 0 : elementRange.getStartOffset(); + int endOffset = elementRange == null ? text.length() : elementRange.getEndOffset(); + for (int i = startOffset; i <= endOffset; i++) { + char c = i >= endOffset ? 0 : text.charAt(i); + if (i == endOffset || c < 128) { if (start != -1) { TextRange range = new TextRange(start, i); - holder.registerProblem(element, range, LangBundle.message("inspection.message.non.ascii.characters.in", kind.ordinal())); + holder.registerProblem(element, range, CodeInsightBundle.message("non.ascii.chars.inspection.message.non.ascii.characters")); start = -1; //do not report too many errors if (errorCount++ > 200) break; @@ -175,7 +303,19 @@ public class NonAsciiCharactersInspection extends LocalInspectionTool { return new NonAsciiCharactersInspectionFormUi(this).getPanel(); } - private enum Kind { - IDENTIFIER, COMMENT, LITERAL + enum PsiElementKind { IDENTIFIER, STRING, COMMENT, OTHER} + @NotNull + private static PsiElementKind getKind(@NotNull PsiElement element, SyntaxHighlighter syntaxHighlighter) { + TextAttributesKey[] keys; + if (element.getParent() instanceof PsiLiteralValue || ChunkExtractor.isHighlightedAsString(keys = syntaxHighlighter.getTokenHighlights(((LeafPsiElement)element).getElementType()))) { + return PsiElementKind.STRING; + } + if (isIdentifier(element)) { + return PsiElementKind.IDENTIFIER; + } + if (element instanceof PsiComment || ChunkExtractor.isHighlightedAsComment(keys)) { + return PsiElementKind.COMMENT; + } + return PsiElementKind.OTHER; } } diff --git a/platform/lang-impl/src/com/intellij/codeInspection/NonAsciiCharactersInspectionFormUi.kt b/platform/lang-impl/src/com/intellij/codeInspection/NonAsciiCharactersInspectionFormUi.kt index 226e558fd27f..643639ee95ae 100644 --- a/platform/lang-impl/src/com/intellij/codeInspection/NonAsciiCharactersInspectionFormUi.kt +++ b/platform/lang-impl/src/com/intellij/codeInspection/NonAsciiCharactersInspectionFormUi.kt @@ -7,67 +7,101 @@ import com.intellij.util.ui.CheckBox class NonAsciiCharactersInspectionFormUi(entry: InspectionProfileEntry) { - val panel = panel { - buttonsGroup(CodeInsightBundle.message("non.ascii.warn.of")) { + row { + cell( + CheckBox( + CodeInsightBundle.message("non.ascii.chars.inspection.option.files.containing.bom.checkbox"), + entry, + "CHECK_FOR_FILES_CONTAINING_BOM" + ) + ) + } + buttonsGroup(CodeInsightBundle.message("non.ascii.chars.inspection.non.ascii.top.label")) { row { cell( CheckBox( - CodeInsightBundle.message("checkbox.non.ascii.option.characters.in.identifiers"), + CodeInsightBundle.message("non.ascii.chars.inspection.option.characters.in.identifiers.checkbox"), entry, "CHECK_FOR_NOT_ASCII_IDENTIFIER_NAME" ) - ).comment(CodeInsightBundle.message("label.non.ascii.chars.example.characters.in.identifiers")) + ) + comment(CodeInsightBundle.message("non.ascii.chars.inspection.example.characters.in.identifiers.label")) } row { cell( CheckBox( - CodeInsightBundle.message("checkbox.non.ascii.option.characters.in.comments"), + CodeInsightBundle.message("non.ascii.chars.inspection.option.characters.in.strings.checkbox"), entry, "CHECK_FOR_NOT_ASCII_STRING_LITERAL" ) - ).comment(CodeInsightBundle.message("label.non.ascii.chars.example.characters.in.comments")) + ) + comment(CodeInsightBundle.message("non.ascii.chars.inspection.example.characters.in.strings.label")) } row { cell( CheckBox( - CodeInsightBundle.message("checkbox.non.ascii.option.characters.in.strings"), + CodeInsightBundle.message("non.ascii.chars.inspection.option.characters.in.comments.checkbox"), entry, "CHECK_FOR_NOT_ASCII_COMMENT" ) - ).comment(CodeInsightBundle.message("label.non.ascii.chars.example.characters.in.strings")) + ) + comment(CodeInsightBundle.message("non.ascii.chars.inspection.example.characters.in.comments.label")) } row { cell( CheckBox( - CodeInsightBundle.message("checkbox.non.ascii.option.different.languages.in.identifiers"), + CodeInsightBundle.message("non.ascii.chars.inspection.option.characters.in.any.other.word.checkbox"), + entry, + "CHECK_FOR_NOT_ASCII_IN_ANY_OTHER_WORD" + ) + ) + comment(CodeInsightBundle.message("non.ascii.chars.inspection.example.characters.in.any.other.word.label")) + } + } + buttonsGroup(CodeInsightBundle.message("non.ascii.chars.inspection.mixed.chars.top.label")) { + row { + cell( + CheckBox( + CodeInsightBundle.message("non.ascii.chars.inspection.option.mixed.languages.in.identifiers.checkbox"), entry, "CHECK_FOR_DIFFERENT_LANGUAGES_IN_IDENTIFIER_NAME" ) - ).comment(CodeInsightBundle.message("label.non.ascii.chars.example.different.languages.in.identifiers")) + ) + comment(CodeInsightBundle.message("non.ascii.chars.inspection.example.mixed.languages.in.identifiers.label")) } - row { cell( CheckBox( - CodeInsightBundle.message("checkbox.non.ascii.option.different.languages.in.string"), + CodeInsightBundle.message("non.ascii.chars.inspection.option.mixed.languages.in.strings.checkbox"), entry, "CHECK_FOR_DIFFERENT_LANGUAGES_IN_STRING" ) - ).comment(CodeInsightBundle.message("label.non.ascii.chars.example.different.languages.in.string")) + ) + comment(CodeInsightBundle.message("non.ascii.chars.inspection.example.mixed.languages.in.string.label")) } - row { cell( CheckBox( - CodeInsightBundle.message("checkbox.non.ascii.option.files.containing.bom"), - entry, - "CHECK_FOR_FILES_CONTAINING_BOM" - ) - ).comment(CodeInsightBundle.message("label.non.ascii.chars.example.files.containing.bom")) + CodeInsightBundle.message("non.ascii.chars.inspection.option.mixed.languages.in.comments.checkbox"), + entry, + "CHECK_FOR_DIFFERENT_LANGUAGES_IN_COMMENTS" + ) + ) + comment(CodeInsightBundle.message("non.ascii.chars.inspection.example.mixed.languages.in.comments.label")) + } + row { + cell( + CheckBox( + CodeInsightBundle.message("non.ascii.chars.inspection.option.mixed.languages.in.any.other.word.checkbox"), + entry, + "CHECK_FOR_DIFFERENT_LANGUAGES_IN_ANY_OTHER_WORD" + ) + ) + comment(CodeInsightBundle.message("non.ascii.chars.inspection.example.mixed.languages.in.any.other.word.label")) } } }