MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

IDEA-177424 Corrupting file encoding upon change

Browse Source 
For files (pre)detected as text, perform charset detection using the entire file content (instead of 8K buffer) to avoid misdetecting files with UTF-8-specific byte sequences far into the text as US-ASCII. Balk out early if detected binary from 8K buffer.
This commit is contained in:
Alexey Kudravtsev
2017-08-21 13:42:10 +03:00
parent 1d2ac5662d
commit 6f21733a53
5 changed files with 3343 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files
java/java-tests/testSrc/com/intellij/java/codeInsight/daemon/LossyEncodingTest.java
+5 -32
View File
@@ -21,15 +21,14 @@ import com.intellij.codeInsight.daemon.impl.DaemonCodeAnalyzerEx;
import com.intellij.codeInsight.daemon.impl.HighlightInfo;
import com.intellij.codeInspection.LocalInspectionTool;
import com.intellij.codeInspection.LossyEncodingInspection;
import com.intellij.openapi.command.WriteCommandAction;
import com.intellij.openapi.editor.Document;
import com.intellij.openapi.fileEditor.FileDocumentManager;
import com.intellij.openapi.fileTypes.FileTypes;
import com.intellij.openapi.fileTypes.StdFileTypes;
import com.intellij.openapi.vfs.CharsetToolkit;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.openapi.vfs.encoding.EncodingManager;
import com.intellij.openapi.vfs.encoding.EncodingProjectManager;
import com.intellij.util.ObjectUtils;
import com.intellij.util.ui.UIUtil;
import org.jetbrains.annotations.NonNls;
import org.jetbrains.annotations.NotNull;
@@ -127,37 +126,11 @@ public class LossyEncodingTest extends DaemonAnalyzerTestCase {
doDoTest(true, false);
}
public void testDetectWrongEncoding0() throws Exception {
String threeNotoriousRussianLetters = "\u0416\u041e\u041f";
configureByText(FileTypes.PLAIN_TEXT, threeNotoriousRussianLetters);
VirtualFile virtualFile = getFile().getVirtualFile();
final Document document = FileDocumentManager.getInstance().getDocument(virtualFile);
WriteCommandAction.runWriteCommandAction(getProject(), () -> {
document.insertString(0, " ");
document.deleteString(0, 1);
});
assertTrue(FileDocumentManager.getInstance().isDocumentUnsaved(document));
assertEquals(CharsetToolkit.UTF8_CHARSET, virtualFile.getCharset());
Charset WINDOWS_1251 = Charset.forName("windows-1251");
virtualFile.setCharset(WINDOWS_1251);
FileDocumentManager.getInstance().saveAllDocuments(); // save in wrong encoding
assertEquals(WINDOWS_1251, virtualFile.getCharset());
assertEquals(threeNotoriousRussianLetters, new String(virtualFile.contentsToByteArray(), WINDOWS_1251));
virtualFile.setCharset(CharsetToolkit.UTF8_CHARSET);
doHighlighting();
List<HighlightInfo> infos = DaemonCodeAnalyzerEx.getInstanceEx(getProject()).getFileLevelHighlights(getProject(), getFile());
HighlightInfo info = assertOneElement(infos);
assertEquals("File was loaded in the wrong encoding: 'UTF-8'", info.getDescription());
}
public void testDetectWrongEncoding() {
VirtualFile virtualFile = getVirtualFile(BASE_PATH + "/" + "Win1251.txt");
VirtualFile virtualFile = getVirtualFile(BASE_PATH + "/Win1251.txt");
virtualFile.setCharset(CharsetToolkit.UTF8_CHARSET);
configureByExistingFile(virtualFile);
final Document document = FileDocumentManager.getInstance().getDocument(virtualFile);
Document document = ObjectUtils.notNull(FileDocumentManager.getInstance().getDocument(virtualFile));
assertFalse(FileDocumentManager.getInstance().isDocumentUnsaved(document));
assertEquals(CharsetToolkit.UTF8_CHARSET, virtualFile.getCharset());
@@ -172,7 +145,7 @@ public class LossyEncodingTest extends DaemonAnalyzerTestCase {
VirtualFile virtualFile = getVirtualFile(BASE_PATH + "/" + "surrogate.txt");
virtualFile.setCharset(CharsetToolkit.UTF8_CHARSET);
configureByExistingFile(virtualFile);
final Document document = FileDocumentManager.getInstance().getDocument(virtualFile);
final Document document = ObjectUtils.notNull(FileDocumentManager.getInstance().getDocument(virtualFile));
assertFalse(FileDocumentManager.getInstance().isDocumentUnsaved(document));
assertEquals(CharsetToolkit.UTF8_CHARSET, virtualFile.getCharset());
@@ -184,7 +157,7 @@ public class LossyEncodingTest extends DaemonAnalyzerTestCase {
VirtualFile virtualFile = getVirtualFile(BASE_PATH + "/" + getTestName(false) + ".txt");
configureByExistingFile(virtualFile);
FileDocumentManager.getInstance().saveAllDocuments();
final Document document = FileDocumentManager.getInstance().getDocument(virtualFile);
final Document document = ObjectUtils.notNull(FileDocumentManager.getInstance().getDocument(virtualFile));
assertFalse(FileDocumentManager.getInstance().isDocumentUnsaved(document));
doHighlighting();
List<HighlightInfo> infos = DaemonCodeAnalyzerEx.getInstanceEx(getProject()).getFileLevelHighlights(getProject(), getFile());
platform/core-impl/src/com/intellij/openapi/fileEditor/impl/LoadTextUtil.java
+33 -49
View File
@@ -47,7 +47,6 @@ import java.nio.CharBuffer;
import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CharsetEncoder;
import java.util.function.Function;
public final class LoadTextUtil {
private static final Logger LOG = Logger.getInstance("#com.intellij.openapi.fileEditor.impl.LoadTextUtil");
@@ -194,42 +193,32 @@ public final class LoadTextUtil {
}
}
// guess from file type or content
@NotNull
private static Trinity<Charset,CharsetToolkit.GuessedEncoding,byte[]> detectCharset(@NotNull VirtualFile virtualFile,
@NotNull byte[] content,
int startOffset, int endOffset,
@NotNull FileType fileType,
@NotNull Function<VirtualFile, Charset> computeCharsetIfNotDetected) {
Charset charset = null;
private static Trinity<Charset,CharsetToolkit.GuessedEncoding,byte[]> detectHardCharset(@NotNull VirtualFile virtualFile,
@NotNull byte[] content,
int startOffset, int endOffset,
@NotNull FileType fileType) {
Charset hardCodedCharset;
String charsetName = fileType.getCharset(virtualFile, content);
Trinity<Charset, CharsetToolkit.GuessedEncoding, byte[]> guessed = guessFromContent(virtualFile, content, startOffset, endOffset);
if (charsetName == null) {
Charset hardCodedCharset = guessed == null ? null : guessed.first;
if (hardCodedCharset != null) {
charset = hardCodedCharset;
}
hardCodedCharset = guessed == null ? null : guessed.first;
}
else {
charset = CharsetToolkit.forName(charsetName);
hardCodedCharset = CharsetToolkit.forName(charsetName);
}
if (charset == null) {
charset = computeCharsetIfNotDetected.apply(virtualFile);
if (hardCodedCharset == null && guessed != null && guessed.second != null && guessed.second == CharsetToolkit.GuessedEncoding.VALID_UTF8) {
return Trinity.create(CharsetToolkit.UTF8_CHARSET, guessed.getSecond(), guessed.getThird());
}
if (charset == null && guessed != null && guessed.second != null) {
if (guessed.second == CharsetToolkit.GuessedEncoding.VALID_UTF8) return Trinity.create(CharsetToolkit.UTF8_CHARSET, guessed.getSecond(),guessed.getThird());
if (guessed.second == CharsetToolkit.GuessedEncoding.SEVEN_BIT) return Trinity.create(CharsetToolkit.US_ASCII_CHARSET, guessed.getSecond(),guessed.getThird());
}
return Trinity.create(charset, guessed == null ? null : guessed.getSecond(), guessed == null ? null : guessed.getThird());
return Trinity.create(hardCodedCharset, guessed == null ? null : guessed.getSecond(), guessed == null ? null : guessed.getThird());
}
@NotNull
public static Charset detectCharsetAndSetBOM(@NotNull VirtualFile virtualFile, @NotNull byte[] content, @NotNull FileType fileType) {
Charset internalCharset = doDetectCharsetAndSetBOM(virtualFile, content, 0,content.length, true, fileType,
virtualFile.isCharsetSet() ? virtualFile.getCharset() : null,
LoadTextUtil::getDefaultCharsetFromEncodingManager).getFirst();
Charset internalCharset = detectInternalCharsetAndSetBOM(virtualFile, content, 0, content.length, true, fileType).getFirst();
return internalCharset instanceof SevenBitCharset ? ((SevenBitCharset)internalCharset).myBaseCharset : internalCharset;
}
@@ -247,32 +236,31 @@ public final class LoadTextUtil {
}
@NotNull
private static Trinity<Charset, CharsetToolkit.GuessedEncoding, byte[]> doDetectCharsetAndSetBOM(@NotNull VirtualFile virtualFile,
@NotNull byte[] content,
int startOffset, int endOffset,
boolean saveBOM,
@NotNull FileType fileType,
@Nullable Charset initialCharset,
@NotNull Function<VirtualFile, Charset> computeCharsetIfNotDetected) {
Trinity<Charset, CharsetToolkit.GuessedEncoding, byte[]>
info = detectCharset(virtualFile, content, startOffset, endOffset, fileType, computeCharsetIfNotDetected);
Charset detectedCharset = info.getFirst();
private static Trinity<Charset, CharsetToolkit.GuessedEncoding, byte[]>
detectInternalCharsetAndSetBOM(@NotNull VirtualFile file,
@NotNull byte[] content,
int startOffset, int endOffset,
boolean saveBOM,
@NotNull FileType fileType) {
Trinity<Charset, CharsetToolkit.GuessedEncoding, byte[]> info = detectHardCharset(file, content, startOffset, endOffset, fileType);
Charset detectedHardCharset = info.getFirst();
CharsetToolkit.GuessedEncoding guessed = info.getSecond();
byte[] bom = info.getThird();
Charset charset = initialCharset != null ? initialCharset : detectedCharset;
// can be overridden by BOM
Charset fromBOM = bom == null ? null : detectedCharset;
// but should not override native_to_ascii wrapped utf-XXX
if (fromBOM != null && (!charset.name().startsWith("NATIVE_TO_ASCII_") || !charset.name().endsWith(fromBOM.name()))) {
charset = fromBOM;
Charset charset;
if (detectedHardCharset == null) {
charset = file.isCharsetSet() ? file.getCharset() : getDefaultCharsetFromEncodingManager(file);
}
else {
charset = detectedHardCharset;
}
if (saveBOM && bom != null && bom.length != 0) {
virtualFile.setBOM(bom);
setCharsetWasDetectedFromBytes(virtualFile, AUTO_DETECTED_FROM_BOM);
file.setBOM(bom);
setCharsetWasDetectedFromBytes(file, AUTO_DETECTED_FROM_BOM);
}
virtualFile.setCharset(charset);
file.setCharset(charset);
Charset result = charset;
// optimisation
@@ -306,7 +294,7 @@ public final class LoadTextUtil {
info = null;
}
else {
Charset defaultCharset = ObjectUtils.notNull(EncodingManager.getInstance().getEncoding(virtualFile, true), CharsetToolkit.getDefaultSystemCharset());
Charset defaultCharset = getDefaultCharsetFromEncodingManager(virtualFile);
info = guessFromBytes(content, startOffset, endOffset, defaultCharset);
byte[] bom = info.getThird();
CharsetToolkit.GuessedEncoding guessed = info.getSecond();
@@ -580,9 +568,7 @@ public final class LoadTextUtil {
boolean saveDetectedSeparators,
boolean saveBOM) {
Trinity<Charset, CharsetToolkit.GuessedEncoding, byte[]>
info = doDetectCharsetAndSetBOM(virtualFile, bytes, 0, bytes.length, saveBOM, virtualFile.getFileType(),
virtualFile.isCharsetSet() ? virtualFile.getCharset() : null,
LoadTextUtil::getDefaultCharsetFromEncodingManager);
info = detectInternalCharsetAndSetBOM(virtualFile, bytes, 0, bytes.length, saveBOM, virtualFile.getFileType());
Charset internalCharset = info.getFirst();
byte[] bom = info.getThird();
Pair<CharSequence, String> result = convertBytes(bytes, Math.min(bom == null ? 0 : bom.length, bytes.length), bytes.length, internalCharset);
@@ -600,9 +586,7 @@ public final class LoadTextUtil {
boolean saveBOM,
@NotNull FileType fileType,
@NotNull NullableConsumer<CharSequence> fileTextProcessor) {
Charset initialCharset = EncodingManager.getInstance().getEncoding(virtualFile, true);
Trinity<Charset, CharsetToolkit.GuessedEncoding, byte[]>
info = doDetectCharsetAndSetBOM(virtualFile, bytes, startOffset, endOffset, saveBOM, fileType, initialCharset, __->null);
Trinity<Charset, CharsetToolkit.GuessedEncoding, byte[]> info = detectInternalCharsetAndSetBOM(virtualFile, bytes, startOffset, endOffset, saveBOM, fileType);
Charset internalCharset = info.getFirst();
CharsetToolkit.GuessedEncoding guessed = info.getSecond();
if (internalCharset == null || guessed == CharsetToolkit.GuessedEncoding.BINARY || guessed == CharsetToolkit.GuessedEncoding.INVALID_UTF8) {
platform/platform-impl/src/com/intellij/openapi/fileTypes/impl/FileTypeManagerImpl.java
+110 -67
View File
@@ -60,6 +60,7 @@ import com.intellij.util.concurrency.AppExecutorUtil;
import com.intellij.util.concurrency.BoundedTaskExecutor;
import com.intellij.util.containers.ConcurrentPackedBitsArray;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.containers.HashSetQueue;
import com.intellij.util.io.URLUtil;
import com.intellij.util.messages.MessageBus;
import com.intellij.util.messages.MessageBusConnection;
@@ -76,8 +77,6 @@ import java.io.*;
import java.net.URL;
import java.nio.channels.FileChannel;
import java.util.*;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
@@ -253,8 +252,10 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements Persistent
log("F: after() queued to redetect: " + files);
}
if (filesToRedetect.addAll(files)) {
awakeReDetectExecutor();
synchronized (filesToRedetect) {
if (filesToRedetect.addAll(files)) {
awakeReDetectExecutor();
}
}
}
}
@@ -351,20 +352,23 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements Persistent
}
private final BoundedTaskExecutor reDetectExecutor = new BoundedTaskExecutor("FileTypeManager redetect pool", PooledThreadExecutor.INSTANCE, 1, this);
private final BlockingQueue<VirtualFile> filesToRedetect = new LinkedBlockingDeque<>();
private final HashSetQueue<VirtualFile> filesToRedetect = new HashSetQueue<>();
private static final int CHUNK_SIZE = 10;
private void awakeReDetectExecutor() {
reDetectExecutor.submit(new Runnable() {
private static final int CHUNK = 10;
@Override
public void run() {
List<VirtualFile> files = new ArrayList<>();
int drained = filesToRedetect.drainTo(files, CHUNK);
reDetect(files);
if (drained == CHUNK) {
awakeReDetectExecutor();
reDetectExecutor.submit(() -> {
List<VirtualFile> files = new ArrayList<>(CHUNK_SIZE);
synchronized (filesToRedetect) {
for (int i = 0; i < CHUNK_SIZE; i++) {
VirtualFile file = filesToRedetect.poll();
if (file == null) break;
files.add(file);
}
}
if (files.size() == CHUNK_SIZE) {
awakeReDetectExecutor();
}
reDetect(files);
});
}
@@ -381,7 +385,9 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements Persistent
@TestOnly
@NotNull
Collection<VirtualFile> dumpReDetectQueue() {
return new ArrayList<>(filesToRedetect);
synchronized (filesToRedetect) {
return new ArrayList<>(filesToRedetect);
}
}
@TestOnly
@@ -501,6 +507,7 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements Persistent
return getFileTypeByFileName((CharSequence)fileName);
}
@Override
@NotNull
public FileType getFileTypeByFileName(@NotNull CharSequence fileName) {
FileType type = myPatternsTable.findAssociatedFileType(fileName);
@@ -765,80 +772,77 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements Persistent
return file.getFileSystem() instanceof FileSystemInterface;
}
private boolean processFirstBytes(@NotNull final InputStream stream, final int length, @NotNull Processor<ByteSequence> processor) throws IOException {
/**
* Read first {@code firstChunkLength} bytes from the {@code stream} to pass them to the {@code processor}.
* {@code processor} may later request all the bytes from the stream by calling {@code Getter<ByteSequence>.get()}
*/
private void processFirstBytes(@NotNull InputStream stream, int fileLength, int firstChunkLength, @NotNull PairConsumer<ByteSequence, Getter<ByteSequence>> processor) throws IOException {
final byte[] bytes = FileUtilRt.getThreadLocalBuffer();
assert bytes.length >= length : "Cannot process more than " + bytes.length + " in one call, requested:" + length;
assert bytes.length >= firstChunkLength : "Cannot process more than " + bytes.length + " in one call, requested:" + firstChunkLength;
int n = stream.read(bytes, 0, length);
int n = readSafely(stream, bytes, 0, firstChunkLength);
if (n<=0) return;
ByteSequence firstChunk = new ByteSequence(bytes, 0, n);
processor.consume(firstChunk, ()->{
if (fileLength <= n) {
// the file is small, fit into the first chunk
return firstChunk;
}
byte[] buffer = bytes.length >= fileLength ? bytes : ArrayUtil.realloc(bytes, fileLength);
int read;
try {
read = readSafely(stream, buffer, n, fileLength - n);
}
catch (IOException e) {
return null;
}
if (read <= 0) return null;
return new ByteSequence(buffer, 0, n+read);
});
}
private int readSafely(InputStream stream, byte[] buffer, int offset, int length) throws IOException {
int n = stream.read(buffer, offset, length);
if (n <= 0) {
// maybe locked because someone else is writing to it
// repeat inside read action to guarantee all writes are finished
if (toLog()) {
log("F: processFirstBytes(): inputStream.read() returned "+n+"; retrying with read action. stream="+ streamInfo(stream));
}
n = ApplicationManager.getApplication().runReadAction((ThrowableComputable<Integer, IOException>)() -> stream.read(bytes, 0, length));
n = ApplicationManager.getApplication().runReadAction((ThrowableComputable<Integer, IOException>)() -> stream.read(buffer, offset, length));
if (toLog()) {
log("F: processFirstBytes(): under read action inputStream.read() returned "+n+"; stream="+ streamInfo(stream));
}
if (n <= 0) {
return false;
}
}
return processor.process(new ByteSequence(bytes, 0, n));
return n;
}
@NotNull
private FileType detectFromContentAndCache(@NotNull final VirtualFile file) throws IOException {
long start = System.currentTimeMillis();
Ref<FileType> result = new Ref<>(UnknownFileType.INSTANCE);
boolean r = false;
InputStream inputStream = ((FileSystemInterface)file.getFileSystem()).getInputStream(file);
try {
if (toLog()) {
log("F: detectFromContentAndCache(" + file.getName() + "):" + " inputStream=" + streamInfo(inputStream));
}
r = processFirstBytes(inputStream, DETECT_BUFFER_SIZE, byteSequence -> {
// use PlainTextFileType because it doesn't supply its own charset detector
// help set charset in the process to avoid double charset detection from content
LoadTextUtil.processTextFromBinaryPresentationOrNull(byteSequence.getBytes(),
byteSequence.getOffset(), byteSequence.getOffset()+byteSequence.getLength(),
file, true, true,
PlainTextFileType.INSTANCE, (@Nullable CharSequence text) -> {
FileTypeDetector[] detectors = Extensions.getExtensions(FileTypeDetector.EP_NAME);
if (toLog()) {
log("F: detectFromContentAndCache.processFirstBytes(" + file.getName() + "): byteSequence.length=" + byteSequence.getLength() +
"; isText=" + (text != null) + "; text='" + (text == null ? null : StringUtil.first(text, 100, true)) + "'" +
", detectors=" + Arrays.toString(detectors));
}
FileType detected = null;
for (FileTypeDetector detector : detectors) {
try {
detected = detector.detect(file, byteSequence, text);
}
catch (Exception e) {
LOG.error("Detector " + detector + " (" + detector.getClass() + ") exception occurred:", e);
}
if (detected != null) {
if (toLog()) {
log("F: detectFromContentAndCache.processFirstBytes(" + file.getName() + "): detector " + detector + " type as " + detected.getName());
}
break;
}
}
if (detected == null) {
detected = text == null ? UnknownFileType.INSTANCE : PlainTextFileType.INSTANCE;
if (toLog()) {
log("F: detectFromContentAndCache.processFirstBytes(" + file.getName() + "): " +
"no detector was able to detect. assigned " + detected.getName());
}
}
result.set(detected);
});
return true;
processFirstBytes(inputStream, (int)file.getLength(), DETECT_BUFFER_SIZE, (firstChunk,entireFileBytesGetter) -> {
detect(file, result, firstChunk);
if (result.get() == UnknownFileType.INSTANCE) {
// detected as binary
return;
}
int firstChunkLength = firstChunk.getLength();
// It seems the file is text but the problem is we might have detected its charset wrong
// The first DETECT_BUFFER_SIZE bytes might have been not enough to e.g. encounter some specific UTF-8 byte sequences
// Need to scan all the file text
ByteSequence entireFile = entireFileBytesGetter.get();
if (entireFile != null && entireFile.getLength() != firstChunkLength) {
detect(file, result, entireFile);
}
});
}
finally {
@@ -848,7 +852,6 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements Persistent
byte[] buffer = new byte[50];
int n = newStream.read(buffer, 0, buffer.length);
log("F: detectFromContentAndCache(" + file.getName() + "): result: " + result.get().getName() +
"; processor ret: " + r +
"; stream: " + streamInfo(inputStream) +
"; newStream: " + streamInfo(newStream) +
"; read: " + n +
@@ -874,6 +877,46 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements Persistent
return fileType;
}
private void detect(@NotNull VirtualFile file, Ref<FileType> result, ByteSequence byteSequence) {
// use PlainTextFileType because it doesn't supply its own charset detector
// help set charset in the process to avoid double charset detection from content
LoadTextUtil.processTextFromBinaryPresentationOrNull(byteSequence.getBytes(),
byteSequence.getOffset(), byteSequence.getOffset()+byteSequence.getLength(),
file, true, true,
PlainTextFileType.INSTANCE, (@Nullable CharSequence text) -> {
FileTypeDetector[] detectors = Extensions.getExtensions(FileTypeDetector.EP_NAME);
if (toLog()) {
log("F: detectFromContentAndCache.processFirstBytes(" + file.getName() + "): byteSequence.length=" + byteSequence.getLength() +
"; isText=" + (text != null) + "; text='" + (text == null ? null : StringUtil.first(text, 100, true)) + "'" +
", detectors=" + Arrays.toString(detectors));
}
FileType detected = null;
for (FileTypeDetector detector : detectors) {
try {
detected = detector.detect(file, byteSequence, text);
}
catch (Exception e) {
LOG.error("Detector " + detector + " (" + detector.getClass() + ") exception occurred:", e);
}
if (detected != null) {
if (toLog()) {
log("F: detectFromContentAndCache.processFirstBytes(" + file.getName() + "): detector " + detector + " type as " + detected.getName());
}
break;
}
}
if (detected == null) {
detected = text == null ? UnknownFileType.INSTANCE : PlainTextFileType.INSTANCE;
if (toLog()) {
log("F: detectFromContentAndCache.processFirstBytes(" + file.getName() + "): " +
"no detector was able to detect. assigned " + detected.getName());
}
}
result.set(detected);
});
}
// for diagnostics
@SuppressWarnings("ConstantConditions")
private static Object streamInfo(InputStream stream) throws IOException {
@@ -1297,7 +1340,7 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements Persistent
}
StringTokenizer tokenizer = new StringTokenizer(semicolonDelimited, FileTypeConsumer.EXTENSION_DELIMITER, false);
ArrayList<FileNameMatcher> list = new ArrayList<>();
ArrayList<FileNameMatcher> list = new ArrayList<>(semicolonDelimited.length() / "py;".length());
while (tokenizer.hasMoreTokens()) {
list.add(new ExtensionFileNameMatcher(tokenizer.nextToken().trim()));
}
platform/platform-tests/testData/vfs/encoding/BIGCHANGES
+3186
View File
File diff suppressed because it is too large Load Diff
platform/platform-tests/testSrc/com/intellij/openapi/vfs/encoding/FileEncodingTest.java
+9
View File
@@ -931,4 +931,13 @@ public class FileEncodingTest extends PlatformTestCase implements TestDialog {
manager.setDefaultCharsetName(oldProject);
}
}
public void testBigFileAutoDetectedAsTextMustDetermineItsEncodingFromTheWholeTextToMinimizePossibilityOfUmlautInTheEndMisdetectionError() {
VirtualFile vTestRoot = getTestRoot();
VirtualFile file = vTestRoot.findChild("BIGCHANGES");
assertNotNull(file);
assertNull(file.getBOM());
assertEquals(CharsetToolkit.UTF8_CHARSET, file.getCharset());
}
}