MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-18 09:34:34 +07:00
Code Issues Packages Projects Releases Wiki Activity

IDEA-69394: implemented heuristics for wrong encoding

Browse Source
This commit is contained in:
Alexey Kudravtsev
2011-05-18 20:17:34 +04:00
parent 83bbfd27b9
commit 8ef9e5ce08
5 changed files with 221 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
java/java-tests/testSrc/com/intellij/codeInsight/daemon/LossyEncodingTest.java
+25 -2
View File
@@ -24,9 +24,13 @@
*/
package com.intellij.codeInsight.daemon;
import com.intellij.codeInsight.daemon.impl.DaemonCodeAnalyzerImpl;
import com.intellij.codeInsight.daemon.impl.HighlightInfo;
import com.intellij.codeInspection.LocalInspectionTool;
import com.intellij.codeInspection.LossyEncodingInspection;
import com.intellij.openapi.fileEditor.FileDocumentManager;
import com.intellij.openapi.vfs.CharsetToolkit;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.openapi.vfs.encoding.EncodingManager;
import com.intellij.openapi.vfs.encoding.EncodingProjectManager;
import com.intellij.util.ui.UIUtil;
@@ -34,6 +38,7 @@ import org.jetbrains.annotations.NonNls;
import java.nio.charset.Charset;
import java.util.Collection;
import java.util.List;
public class LossyEncodingTest extends LightDaemonAnalyzerTestCase {
@NonNls private static final String BASE_PATH = "/codeInsight/daemonCodeAnalyzer/lossyEncoding";
@@ -97,6 +102,10 @@ public class LossyEncodingTest extends LightDaemonAnalyzerTestCase {
doTestConfiguredFile(true, false);
}
private void doTest(@NonNls String filePath) throws Exception {
doTest(BASE_PATH + "/" + filePath, true, false);
}
public void testNativeEncoding() throws Exception {
EncodingManager.getInstance().setNative2AsciiForPropertiesFiles(null, true);
configureByFile(BASE_PATH + "/" + "NativeEncoding.properties");
@@ -104,7 +113,21 @@ public class LossyEncodingTest extends LightDaemonAnalyzerTestCase {
doTestConfiguredFile(true, false);
}
private void doTest(@NonNls String filePath) throws Exception {
doTest(BASE_PATH + "/" + filePath, true, false);
public static final String THREE_NOTORIOUS_RUSSIAN_LETTERS = "\u0416\u041e\u041f";
public void testDetectWrongEncoding() throws Exception {
configureFromFileText("Win1251.txt", THREE_NOTORIOUS_RUSSIAN_LETTERS);
VirtualFile virtualFile = getFile().getVirtualFile();
assertEquals(CharsetToolkit.UTF8_CHARSET, virtualFile.getCharset());
Charset WINDOWS_1251 = Charset.forName("windows-1251");
virtualFile.setCharset(WINDOWS_1251);
FileDocumentManager.getInstance().saveAllDocuments();
assertEquals(WINDOWS_1251, virtualFile.getCharset());
assertEquals(THREE_NOTORIOUS_RUSSIAN_LETTERS, new String(virtualFile.contentsToByteArray(), WINDOWS_1251));
virtualFile.setCharset(CharsetToolkit.UTF8_CHARSET);
doHighlighting();
List<HighlightInfo> infos = DaemonCodeAnalyzerImpl.getFileLevelHighlights(getProject(), getFile());
HighlightInfo info = assertOneElement(infos);
assertEquals("File was loaded in a wrong encoding: 'UTF-8'", info.description);
}
}
platform/lang-impl/src/com/intellij/codeInspection/LossyEncodingInspection.java
+121 -62
View File
@@ -23,13 +23,16 @@
package com.intellij.codeInspection;
import com.intellij.ide.DataManager;
import com.intellij.lang.injection.InjectedLanguageManager;
import com.intellij.lang.properties.charset.Native2AsciiCharset;
import com.intellij.openapi.actionSystem.DataContext;
import com.intellij.openapi.actionSystem.DefaultActionGroup;
import com.intellij.openapi.fileEditor.FileDocumentManager;
import com.intellij.openapi.fileEditor.impl.LoadTextUtil;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.ui.popup.JBPopupFactory;
import com.intellij.openapi.util.TextRange;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.openapi.vfs.encoding.ChooseFileEncodingAction;
import com.intellij.openapi.vfs.encoding.EncodingManager;
@@ -41,13 +44,130 @@ import org.jetbrains.annotations.NonNls;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.List;
public class LossyEncodingInspection extends LocalInspectionTool {
private static final LocalQuickFix CHANGE_ENCODING_FIX = new LocalQuickFix() {
private static final LocalQuickFix CHANGE_ENCODING_FIX = new ChangeEncodingFix();
private static final LocalQuickFix RELOAD_ENCODING_FIX = new ReloadInAnotherEncodingFix();
@Nls
@NotNull
public String getGroupDisplayName() {
return InspectionsBundle.message("group.names.internationalization.issues");
}
@Nls
@NotNull
public String getDisplayName() {
return InspectionsBundle.message("lossy.encoding");
}
@NonNls
@NotNull
public String getShortName() {
return "LossyEncoding";
}
@Nullable
public ProblemDescriptor[] checkFile(@NotNull PsiFile file, @NotNull InspectionManager manager, boolean isOnTheFly) {
if (InjectedLanguageManager.getInstance(file.getProject()).isInjectedFragment(file)) return null;
if (ArrayUtil.find(file.getPsiRoots(), file) != 0) return null;
VirtualFile virtualFile = file.getVirtualFile();
if (virtualFile == null) return null;
String text = file.getText();
Charset charset = LoadTextUtil.extractCharsetFromFileContent(file.getProject(), virtualFile, text);
// no sense in checking transparently decoded file: all characters there are already safely encoded
if (charset instanceof Native2AsciiCharset) return null;
List<ProblemDescriptor> descriptors = new SmartList<ProblemDescriptor>();
checkIfCharactersWillBeLostAfterSave(file, manager, isOnTheFly, text, charset, descriptors);
checkForFileLoadedInWrongEncoding(file, manager, isOnTheFly, virtualFile, charset, descriptors);
return descriptors.toArray(new ProblemDescriptor[descriptors.size()]);
}
private static void checkForFileLoadedInWrongEncoding(PsiFile file,
InspectionManager manager,
boolean isOnTheFly,
VirtualFile virtualFile,
Charset charset, List<ProblemDescriptor> descriptors) {
if (!FileDocumentManager.getInstance().isFileModified(virtualFile) // when file is modified, it's too late to reload it
&& ChooseFileEncodingAction.isEnabled(virtualFile) // can't reload in another encoding, no point trying
) {
// check if file was loaded in correct encoding
byte[] bytes;
try {
bytes = virtualFile.contentsToByteArray();
}
catch (IOException e) {
return;
}
String separator = FileDocumentManager.getInstance().getLineSeparator(virtualFile, file.getProject());
String toSave = StringUtil.convertLineSeparators(file.getText(), separator);
byte[] bytesToSave = toSave.getBytes(charset);
if (!Arrays.equals(bytesToSave, bytes)) {
descriptors.add(manager.createProblemDescriptor(file, "File was loaded in a wrong encoding: '"+charset+"'",
RELOAD_ENCODING_FIX, ProblemHighlightType.GENERIC_ERROR, isOnTheFly));
}
}
}
private static void checkIfCharactersWillBeLostAfterSave(PsiFile file,
InspectionManager manager,
boolean isOnTheFly,
String text,
Charset charset, List<ProblemDescriptor> descriptors) {
int errorCount = 0;
int start = -1;
for (int i = 0; i <= text.length(); i++) {
char c = i == text.length() ? 0 : text.charAt(i);
if (i == text.length() || isRepresentable(c, charset)) {
if (start != -1) {
TextRange range = new TextRange(start, i);
String message = InspectionsBundle.message("unsupported.character.for.the.charset", charset);
ProblemDescriptor descriptor =
manager.createProblemDescriptor(file, range, message, ProblemHighlightType.GENERIC_ERROR_OR_WARNING, isOnTheFly, CHANGE_ENCODING_FIX);
descriptors.add(descriptor);
start = -1;
//do not report too many errors
if (errorCount++ > 200) break;
}
}
else if (start == -1) {
start = i;
}
}
}
private static boolean isRepresentable(final char c, final Charset charset) {
String str = Character.toString(c);
ByteBuffer out = charset.encode(str);
CharBuffer buffer = charset.decode(out);
return str.equals(buffer.toString());
}
private static class ReloadInAnotherEncodingFix extends ChangeEncodingFix {
@NotNull
@Override
public String getName() {
return "Reload in another encoding";
}
@Override
public void applyFix(@NotNull Project project, @NotNull ProblemDescriptor descriptor) {
if (FileDocumentManager.getInstance().isFileModified(descriptor.getPsiElement().getContainingFile().getVirtualFile())) return;
super.applyFix(project, descriptor);
}
}
private static class ChangeEncodingFix implements LocalQuickFix {
@NotNull
@Override
public String getName() {
@@ -76,66 +196,5 @@ public class LossyEncodingInspection extends LocalInspectionTool {
DataContext dataContext = DataManager.getInstance().getDataContext();
JBPopupFactory.getInstance().createActionGroupPopup(null, group, dataContext, false, false, false, null, 30, null).showInBestPositionFor(dataContext);
}
};
@Nls
@NotNull
public String getGroupDisplayName() {
return InspectionsBundle.message("group.names.internationalization.issues");
}
@Nls
@NotNull
public String getDisplayName() {
return InspectionsBundle.message("lossy.encoding");
}
@NonNls
@NotNull
public String getShortName() {
return "LossyEncoding";
}
@Nullable
public ProblemDescriptor[] checkFile(@NotNull PsiFile file, @NotNull InspectionManager manager, boolean isOnTheFly) {
if (ArrayUtil.find(file.getPsiRoots(), file) != 0) return null;
VirtualFile virtualFile = file.getVirtualFile();
if (virtualFile == null) return null;
String text = file.getText();
Charset charset = LoadTextUtil.extractCharsetFromFileContent(file.getProject(), virtualFile, text);
// no sense in checking transparently decoded file: all characters there are already safely encoded
if (charset instanceof Native2AsciiCharset) return null;
int errorCount = 0;
int start = -1;
List<ProblemDescriptor> descriptors = new SmartList<ProblemDescriptor>();
for (int i = 0; i <= text.length(); i++) {
char c = i == text.length() ? 0 : text.charAt(i);
if (i == text.length() || isRepresentable(c, charset)) {
if (start != -1) {
TextRange range = new TextRange(start, i);
String message = InspectionsBundle.message("unsupported.character.for.the.charset", charset);
ProblemDescriptor descriptor = manager.createProblemDescriptor(file, range, message, ProblemHighlightType.GENERIC_ERROR_OR_WARNING, isOnTheFly,
CHANGE_ENCODING_FIX);
descriptors.add(descriptor);
start = -1;
//do not report too many errors
if (errorCount++ > 200) break;
}
}
else if (start == -1) {
start = i;
}
}
return descriptors.toArray(new ProblemDescriptor[descriptors.size()]);
}
private static boolean isRepresentable(final char c, final Charset charset) {
String str = Character.toString(c);
ByteBuffer out = charset.encode(str);
CharBuffer buffer = charset.decode(out);
return str.equals(buffer.toString());
}
}
platform/platform-impl/src/com/intellij/openapi/vfs/encoding/ChooseFileEncodingAction.java
+30 -23
View File
@@ -31,6 +31,7 @@ import com.intellij.openapi.fileEditor.FileDocumentManager;
import com.intellij.openapi.fileEditor.impl.LoadTextUtil;
import com.intellij.openapi.fileTypes.FileType;
import com.intellij.openapi.fileTypes.FileTypeManager;
import com.intellij.openapi.fileTypes.FileTypes;
import com.intellij.openapi.fileTypes.StdFileTypes;
import com.intellij.openapi.util.Pair;
import com.intellij.openapi.vfs.CharsetToolkit;
@@ -70,27 +71,28 @@ public abstract class ChooseFileEncodingAction extends ComboBoxAction {
}
public static boolean isEnabled(@Nullable VirtualFile virtualFile) {
if (virtualFile == null) {
return false;
}
boolean enabled = true;
if (virtualFile != null) {
Charset charset = cachedCharsetFromContent(virtualFile);
if (charset != null) {
enabled = false;
}
else if (!virtualFile.isDirectory()) {
FileType fileType = FileTypeManager.getInstance().getFileTypeByFile(virtualFile);
if (fileType.isBinary()
|| fileType == StdFileTypes.GUI_DESIGNER_FORM
|| fileType == StdFileTypes.IDEA_MODULE
|| fileType == StdFileTypes.IDEA_PROJECT
|| fileType == StdFileTypes.IDEA_WORKSPACE
|| fileType == StdFileTypes.PATCH
|| fileType == StdFileTypes.PROPERTIES
Charset charset = cachedCharsetFromContent(virtualFile);
if (charset != null) {
enabled = false;
}
else if (!virtualFile.isDirectory()) {
FileType fileType = FileTypeManager.getInstance().getFileTypeByFile(virtualFile);
if (fileType.isBinary()
|| fileType == StdFileTypes.GUI_DESIGNER_FORM
|| fileType == StdFileTypes.IDEA_MODULE
|| fileType == StdFileTypes.IDEA_PROJECT
|| fileType == StdFileTypes.IDEA_WORKSPACE
|| fileType == StdFileTypes.PATCH
|| fileType == StdFileTypes.PROPERTIES
|| fileType == StdFileTypes.XML
|| fileType == StdFileTypes.JSPX
) {
enabled = false;
}
|| fileType == StdFileTypes.XML
|| fileType == StdFileTypes.JSPX && fileType != FileTypes.PLAIN_TEXT // in community tests JSPX==TEXT
) {
enabled = false;
}
}
return enabled;
@@ -204,11 +206,16 @@ public abstract class ChooseFileEncodingAction extends ComboBoxAction {
if (showClear) {
group.add(new ClearThisFileEncodingAction(myVirtualFile));
}
fillCharsetActions(group, myVirtualFile, favorites);
if (favorites.isEmpty() && !showClear) {
fillCharsetActions(group, myVirtualFile, Arrays.asList(CharsetToolkit.getAvailableCharsets()));
}
else {
fillCharsetActions(group, myVirtualFile, favorites);
DefaultActionGroup more = new DefaultActionGroup("more", true);
group.add(more);
fillCharsetActions(more, myVirtualFile, Arrays.asList(CharsetToolkit.getAvailableCharsets()));
DefaultActionGroup more = new DefaultActionGroup("more", true);
group.add(more);
fillCharsetActions(more, myVirtualFile, Arrays.asList(CharsetToolkit.getAvailableCharsets()));
}
return group;
}
}
platform/platform-impl/src/com/intellij/openapi/wm/impl/status/EncodingPanel.java
+35
View File
@@ -18,6 +18,12 @@ package com.intellij.openapi.wm.impl.status;
import com.intellij.ide.DataManager;
import com.intellij.openapi.actionSystem.*;
import com.intellij.openapi.actionSystem.impl.SimpleDataContext;
import com.intellij.openapi.application.ApplicationManager;
import com.intellij.openapi.editor.Document;
import com.intellij.openapi.editor.Editor;
import com.intellij.openapi.editor.EditorFactory;
import com.intellij.openapi.editor.event.DocumentAdapter;
import com.intellij.openapi.editor.event.DocumentEvent;
import com.intellij.openapi.fileEditor.FileEditorManager;
import com.intellij.openapi.fileEditor.FileEditorManagerEvent;
import com.intellij.openapi.project.Project;
@@ -27,13 +33,18 @@ import com.intellij.openapi.util.Disposer;
import com.intellij.openapi.util.IconLoader;
import com.intellij.openapi.util.Pair;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.openapi.vfs.VirtualFileAdapter;
import com.intellij.openapi.vfs.VirtualFileManager;
import com.intellij.openapi.vfs.VirtualFilePropertyEvent;
import com.intellij.openapi.vfs.encoding.ChooseFileEncodingAction;
import com.intellij.openapi.vfs.encoding.EncodingManager;
import com.intellij.openapi.vfs.encoding.EncodingManagerImpl;
import com.intellij.openapi.vfs.impl.BulkVirtualFileListenerAdapter;
import com.intellij.openapi.wm.CustomStatusBarWidget;
import com.intellij.openapi.wm.StatusBar;
import com.intellij.openapi.wm.StatusBarWidget;
import com.intellij.ui.awt.RelativePoint;
import com.intellij.util.Alarm;
import com.intellij.util.ui.UIUtil;
import org.jetbrains.annotations.NotNull;
@@ -118,6 +129,30 @@ public class EncodingPanel extends EditorBasedWidget implements StatusBarWidget.
}
}
}, this);
ApplicationManager.getApplication().getMessageBus().connect(this).subscribe(VirtualFileManager.VFS_CHANGES, new BulkVirtualFileListenerAdapter(new VirtualFileAdapter() {
@Override
public void propertyChanged(VirtualFilePropertyEvent event) {
if (VirtualFile.PROP_ENCODING.equals(event.getPropertyName())) {
update();
}
}
}));
final Alarm update = new Alarm();
EditorFactory.getInstance().getEventMulticaster().addDocumentListener(new DocumentAdapter() {
@Override
public void documentChanged(DocumentEvent e) {
Document document = e.getDocument();
Editor selectedEditor = getEditor();
if (selectedEditor == null || selectedEditor.getDocument() != document) return;
update.cancelAllRequests();
update.addRequest(new Runnable() {
@Override
public void run() {
if (!isDisposed()) update();
}
}, 200);
}
}, this);
}
private void showPopup(MouseEvent e) {
platform/platform-resources-en/src/inspectionDescriptions/LossyEncoding.html
+10 -5
View File
@@ -1,12 +1,17 @@
<html>
<body>
<font face="verdana" size="-1">
This inspection warns you of characters that current encoding is incapable to represent.<br>
E.g. when you are trying to type international characters in an <b>US-ASCII</b>-encoded file.<br>
Typically, you would fix this by changing the file encoding,
This inspection warns you of characters that the current document encoding is incapable to represent.<br>
For example, when you are<br>
<ul>
<li>typing international characters in a document configured to <b>US-ASCII</b> charset. Some characters will be lost on save.</li>
<li>or loading <b>UTF-8</b>-encoded file using <b>ISO-8859-1</b> one-byte charset. Some characters will be displayed incorrectly.</li>
</ul>
You fix this by changing the file encoding,
either by specifying the encoding directly in the file, e.g. by editing <b>encoding=</b> attribute in the XML prolog of XML file,
or configuring the <b>Settings|General|File Encoding|Default encoding</b> setting,
or by setting up the file/directory encoding in the <b>Settings|File/Directory Options|File Encodings</b>.
or configuring the <b>Settings|Project Settings|File Encodings</b>.
</font>
</body>