MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-13 09:19:13 +07:00
Code Issues Packages Projects Releases Wiki Activity

IDEA-110268 Lossy encoding: false positive for separated 0A/0D as UTF-8

Browse Source
This commit is contained in:
Alexey Kudravtsev
2014-02-14 13:32:18 +04:00
parent 9f6ada3f3f
commit 4170df8d4c
7 changed files with 81 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
java/java-tests/testData/codeInsight/daemonCodeAnalyzer/lossyEncoding/InconsistentLineSeparators.txt
+7
View File
@@ -0,0 +1,7 @@
package temp.04
class Book {
static constraints = {
}
}
java/java-tests/testData/codeInsight/daemonCodeAnalyzer/lossyEncoding/Win1251.txt
+1
View File
@@ -0,0 +1 @@
���
java/java-tests/testSrc/com/intellij/codeInsight/daemon/LossyEncodingTest.java
+58 -9
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2013 JetBrains s.r.o.
* Copyright 2000-2014 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -28,7 +28,11 @@ import com.intellij.codeInsight.daemon.impl.DaemonCodeAnalyzerEx;
import com.intellij.codeInsight.daemon.impl.HighlightInfo;
import com.intellij.codeInspection.LocalInspectionTool;
import com.intellij.codeInspection.LossyEncodingInspection;
import com.intellij.openapi.command.WriteCommandAction;
import com.intellij.openapi.editor.Document;
import com.intellij.openapi.fileEditor.FileDocumentManager;
import com.intellij.openapi.fileTypes.FileTypes;
import com.intellij.openapi.fileTypes.StdFileTypes;
import com.intellij.openapi.vfs.CharsetToolkit;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.openapi.vfs.encoding.EncodingManager;
@@ -41,7 +45,7 @@ import java.nio.charset.Charset;
import java.util.Collection;
import java.util.List;
public class LossyEncodingTest extends LightDaemonAnalyzerTestCase {
public class LossyEncodingTest extends DaemonAnalyzerTestCase {
@NonNls private static final String BASE_PATH = "/codeInsight/daemonCodeAnalyzer/lossyEncoding";
@NotNull
@@ -53,6 +57,7 @@ public class LossyEncodingTest extends LightDaemonAnalyzerTestCase {
public void testText() throws Exception {
doTest("Text.txt");
Charset ascii = CharsetToolkit.forName("US-ASCII");
VirtualFile myVFile = myFile.getVirtualFile();
EncodingManager.getInstance().setEncoding(myVFile, ascii);
assertEquals(ascii, myVFile.getCharset());
int start = myEditor.getCaretModel().getOffset();
@@ -68,11 +73,12 @@ public class LossyEncodingTest extends LightDaemonAnalyzerTestCase {
backspace();
backspace();
doTestConfiguredFile(true, false, null);
doDoTest(true, false);
}
public void testNativeConversion() throws Exception {
configureFromFileText("x.properties","a=<caret>v");
configureByText(StdFileTypes.PROPERTIES, "a=<caret>v");
EncodingProjectManager.getInstance(getProject()).setNative2AsciiForPropertiesFiles(null, true);
UIUtil.dispatchAllInvocationEvents(); //reload files
@@ -104,7 +110,7 @@ public class LossyEncodingTest extends LightDaemonAnalyzerTestCase {
configureByFile(BASE_PATH + "/" + "MultipleRanges.xml");
type("US-ASCII");
doTestConfiguredFile(true, false, null);
doDoTest(true, false);
}
private void doTest(@NonNls String filePath) throws Exception {
@@ -115,17 +121,28 @@ public class LossyEncodingTest extends LightDaemonAnalyzerTestCase {
EncodingManager.getInstance().setNative2AsciiForPropertiesFiles(null, true);
configureByFile(BASE_PATH + "/" + "NativeEncoding.properties");
doTestConfiguredFile(true, false, null);
doDoTest(true, false);
}
public void testDetectWrongEncoding() throws Exception {
public void testDetectWrongEncoding0() throws Exception {
String threeNotoriousRussianLetters = "\u0416\u041e\u041f";
configureFromFileText("Win1251.txt", threeNotoriousRussianLetters);
configureByText(FileTypes.PLAIN_TEXT, threeNotoriousRussianLetters);
VirtualFile virtualFile = getFile().getVirtualFile();
final Document document = FileDocumentManager.getInstance().getDocument(virtualFile);
WriteCommandAction.runWriteCommandAction(getProject(), new Runnable() {
@Override
public void run() {
document.insertString(0, " ");
document.deleteString(0, 1);
}
});
assertTrue(FileDocumentManager.getInstance().isDocumentUnsaved(document));
assertEquals(CharsetToolkit.UTF8_CHARSET, virtualFile.getCharset());
Charset WINDOWS_1251 = Charset.forName("windows-1251");
virtualFile.setCharset(WINDOWS_1251);
FileDocumentManager.getInstance().saveAllDocuments();
FileDocumentManager.getInstance().saveAllDocuments(); // save in wrong encoding
assertEquals(WINDOWS_1251, virtualFile.getCharset());
assertEquals(threeNotoriousRussianLetters, new String(virtualFile.contentsToByteArray(), WINDOWS_1251));
virtualFile.setCharset(CharsetToolkit.UTF8_CHARSET);
@@ -135,4 +152,36 @@ public class LossyEncodingTest extends LightDaemonAnalyzerTestCase {
HighlightInfo info = assertOneElement(infos);
assertEquals("File was loaded in the wrong encoding: 'UTF-8'", info.getDescription());
}
public void testDetectWrongEncoding() throws Exception {
String threeNotoriousRussianLetters = "\u0416\u041e\u041f";
VirtualFile virtualFile = getVirtualFile(BASE_PATH + "/" + "Win1251" + ".txt");
virtualFile.setCharset(CharsetToolkit.UTF8_CHARSET);
configureByExistingFile(virtualFile);
final Document document = FileDocumentManager.getInstance().getDocument(virtualFile);
assertFalse(FileDocumentManager.getInstance().isDocumentUnsaved(document));
assertEquals(CharsetToolkit.UTF8_CHARSET, virtualFile.getCharset());
//Charset WINDOWS_1251 = Charset.forName("windows-1251");
//virtualFile.setCharset(WINDOWS_1251);
//FileDocumentManager.getInstance().saveAllDocuments(); // save in wrong encoding
//assertEquals(WINDOWS_1251, virtualFile.getCharset());
//assertEquals(threeNotoriousRussianLetters, new String(virtualFile.contentsToByteArray(), WINDOWS_1251));
doHighlighting();
List<HighlightInfo> infos = DaemonCodeAnalyzerEx.getInstanceEx(getProject()).getFileLevelHighlights(getProject(), getFile());
HighlightInfo info = assertOneElement(infos);
assertEquals("File was loaded in the wrong encoding: 'UTF-8'", info.getDescription());
}
public void testInconsistentLineSeparators() throws Exception {
VirtualFile virtualFile = getVirtualFile(BASE_PATH + "/" + getTestName(false) + ".txt");
configureByExistingFile(virtualFile);
FileDocumentManager.getInstance().saveAllDocuments();
final Document document = FileDocumentManager.getInstance().getDocument(virtualFile);
assertFalse(FileDocumentManager.getInstance().isDocumentUnsaved(document));
doHighlighting();
List<HighlightInfo> infos = DaemonCodeAnalyzerEx.getInstanceEx(getProject()).getFileLevelHighlights(getProject(), getFile());
assertEmpty(infos);
}
}
java/testFramework/src/com/intellij/codeInsight/CodeInsightTestCase.java
+2 -4
View File
@@ -42,10 +42,7 @@ import com.intellij.openapi.roots.ModuleRootManager;
import com.intellij.openapi.util.Segment;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.openapi.vfs.LocalFileSystem;
import com.intellij.openapi.vfs.VfsUtil;
import com.intellij.openapi.vfs.VfsUtilCore;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.openapi.vfs.*;
import com.intellij.psi.PsiClass;
import com.intellij.psi.PsiDocumentManager;
import com.intellij.psi.PsiFile;
@@ -155,6 +152,7 @@ public abstract class CodeInsightTestCase extends PsiTestCase {
}
final VirtualFile vFile = LocalFileSystem.getInstance().refreshAndFindFileByIoFile(tempFile);
assert vFile != null;
vFile.setCharset(CharsetToolkit.UTF8_CHARSET);
VfsUtil.saveText(vFile, text);
final VirtualFile vdir = LocalFileSystem.getInstance().refreshAndFindFileByIoFile(dir);
platform/lang-impl/src/com/intellij/codeInspection/LossyEncodingInspection.java
+11 -23
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2013 JetBrains s.r.o.
* Copyright 2000-2014 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -33,13 +33,11 @@ import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.editor.Document;
import com.intellij.openapi.editor.Editor;
import com.intellij.openapi.fileEditor.FileDocumentManager;
import com.intellij.openapi.fileEditor.impl.FileDocumentManagerImpl;
import com.intellij.openapi.fileEditor.impl.LoadTextUtil;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.ui.popup.ListPopup;
import com.intellij.openapi.util.TextRange;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.openapi.vfs.encoding.ChangeFileEncodingAction;
import com.intellij.openapi.vfs.encoding.EncodingUtil;
@@ -104,7 +102,7 @@ public class LossyEncodingInspection extends LocalInspectionTool {
if (charset instanceof Native2AsciiCharset) return null;
List<ProblemDescriptor> descriptors = new SmartList<ProblemDescriptor>();
boolean ok = checkFileLoadedInWrongEncoding(file, manager, isOnTheFly, text, virtualFile, charset, descriptors);
boolean ok = checkFileLoadedInWrongEncoding(file, manager, isOnTheFly, virtualFile, charset, descriptors);
if (ok) {
checkIfCharactersWillBeLostAfterSave(file, manager, isOnTheFly, text, charset, descriptors);
}
@@ -115,7 +113,6 @@ public class LossyEncodingInspection extends LocalInspectionTool {
private static boolean checkFileLoadedInWrongEncoding(@NotNull PsiFile file,
@NotNull InspectionManager manager,
boolean isOnTheFly,
@NotNull String text,
@NotNull VirtualFile virtualFile,
@NotNull Charset charset,
@NotNull List<ProblemDescriptor> descriptors) {
@@ -124,7 +121,7 @@ public class LossyEncodingInspection extends LocalInspectionTool {
) {
return true;
}
if (!isGoodCharset(virtualFile, text, charset, file.getProject())) {
if (!isGoodCharset(virtualFile, charset)) {
descriptors.add(manager.createProblemDescriptor(file, "File was loaded in the wrong encoding: '"+charset+"'",
RELOAD_ENCODING_FIX, ProblemHighlightType.GENERIC_ERROR, isOnTheFly));
return false;
@@ -134,38 +131,29 @@ public class LossyEncodingInspection extends LocalInspectionTool {
// check if file was loaded in correct encoding
// returns true if text converted with charset is equals to the bytes currently on disk
private static boolean isGoodCharset(@NotNull VirtualFile virtualFile,
@NotNull String text,
@NotNull Charset charset,
@NotNull Project project) {
private static boolean isGoodCharset(@NotNull VirtualFile virtualFile, @NotNull Charset charset) {
FileDocumentManager documentManager = FileDocumentManager.getInstance();
Document document = documentManager.getDocument(virtualFile);
if (document == null) return true;
byte[] bytes;
byte[] loadedBytes;
byte[] bytesToSave;
try {
bytes = virtualFile.contentsToByteArray();
loadedBytes = virtualFile.contentsToByteArray();
bytesToSave = new String(loadedBytes, charset).getBytes(charset);
}
catch (IOException e) {
catch (Exception e) {
return true;
}
String separator = LoadTextUtil.detectLineSeparator(virtualFile, false);
if (separator == null) {
separator = documentManager.isDocumentUnsaved(document) ?
FileDocumentManagerImpl.getLineSeparator(document, virtualFile) :
FileDocumentManager.getInstance().getLineSeparator(null, project);
}
String toSave = StringUtil.convertLineSeparators(text, separator);
byte[] bom = virtualFile.getBOM();
byte[] bytesToSave = toSave.getBytes(charset);
if (bom != null && !ArrayUtil.startsWith(bytesToSave, bom)) {
bytesToSave = ArrayUtil.mergeArrays(bom, bytesToSave); // for 2-byte encodings String.getBytes(Charset) adds BOM automatically
}
boolean equals = Arrays.equals(bytesToSave, bytes);
boolean equals = Arrays.equals(bytesToSave, loadedBytes);
if (!equals && LOG.isDebugEnabled()) {
try {
FileUtil.writeToFile(new File("C:\\temp\\bytesToSave"), bytesToSave);
FileUtil.writeToFile(new File("C:\\temp\\bytes"), bytes);
FileUtil.writeToFile(new File("C:\\temp\\loadedBytes"), loadedBytes);
}
catch (IOException e) {
throw new RuntimeException(e);
platform/testFramework/src/com/intellij/testFramework/LightPlatformCodeInsightTestCase.java
-6
View File
@@ -121,7 +121,6 @@ public abstract class LightPlatformCodeInsightTestCase extends LightPlatformTest
* has &lt;caret&gt; marker where caret should be placed when file is loaded in editor and &lt;selection&gt;&lt;/selection&gt;
* denoting selection bounds.
* @param filePath - relative path from %IDEA_INSTALLATION_HOME%/testData/
* @throws Exception
*/
protected void configureByFile(@TestDataFile @NonNls @NotNull String filePath) {
try {
@@ -300,7 +299,6 @@ public abstract class LightPlatformCodeInsightTestCase extends LightPlatformTest
* Validates that content of the editor as well as caret and selection matches one specified in data file that
* should be formed with the same format as one used in configureByFile
* @param filePath - relative path from %IDEA_INSTALLATION_HOME%/testData/
* @throws Exception
*/
protected void checkResultByFile(@TestDataFile @NonNls @NotNull String filePath) {
checkResultByFile(null, filePath, false);
@@ -312,7 +310,6 @@ public abstract class LightPlatformCodeInsightTestCase extends LightPlatformTest
* @param message - this check specific message. Added to text, caret position, selection checking. May be null
* @param filePath - relative path from %IDEA_INSTALLATION_HOME%/testData/
* @param ignoreTrailingSpaces - whether trailing spaces in editor in data file should be stripped prior to comparing.
* @throws Exception
*/
protected void checkResultByFile(@Nullable String message, @TestDataFile @NotNull String filePath, final boolean ignoreTrailingSpaces) {
bringRealEditorBack();
@@ -342,7 +339,6 @@ public abstract class LightPlatformCodeInsightTestCase extends LightPlatformTest
/**
* Same as checkResultByFile but text is provided directly.
* @param fileText
*/
protected void checkResultByText(@NonNls @NotNull String fileText) {
checkResultByText(null, fileText, false, null);
@@ -351,7 +347,6 @@ public abstract class LightPlatformCodeInsightTestCase extends LightPlatformTest
/**
* Same as checkResultByFile but text is provided directly.
* @param message - this check specific message. Added to text, caret position, selection checking. May be null
* @param fileText
* @param ignoreTrailingSpaces - whether trailing spaces in editor in data file should be stripped prior to comparing.
*/
protected void checkResultByText(final String message, @NotNull String fileText, final boolean ignoreTrailingSpaces) {
@@ -361,7 +356,6 @@ public abstract class LightPlatformCodeInsightTestCase extends LightPlatformTest
/**
* Same as checkResultByFile but text is provided directly.
* @param message - this check specific message. Added to text, caret position, selection checking. May be null
* @param fileText
* @param ignoreTrailingSpaces - whether trailing spaces in editor in data file should be stripped prior to comparing.
*/
protected void checkResultByText(final String message, @NotNull final String fileText, final boolean ignoreTrailingSpaces, final String filePath) {
platform/util/src/com/intellij/util/LineSeparator.java
+2 -2
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2012 JetBrains s.r.o.
* Copyright 2000-2014 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -37,7 +37,7 @@ public enum LineSeparator {
private static final Logger LOG = Logger.getInstance(LineSeparator.class);
private final String mySeparatorString;
LineSeparator(String separatorString) {
LineSeparator(@NotNull String separatorString) {
mySeparatorString = separatorString;
}