diff --git a/platform/diff-impl/src/com/intellij/diff/contents/FileAwareDocumentContent.java b/platform/diff-impl/src/com/intellij/diff/contents/FileAwareDocumentContent.java index 3fd88499d197..a750e72f136e 100644 --- a/platform/diff-impl/src/com/intellij/diff/contents/FileAwareDocumentContent.java +++ b/platform/diff-impl/src/com/intellij/diff/contents/FileAwareDocumentContent.java @@ -7,7 +7,6 @@ import com.intellij.openapi.fileTypes.FileType; import com.intellij.openapi.fileTypes.FileTypes; import com.intellij.openapi.fileTypes.PlainTextFileType; import com.intellij.openapi.project.Project; -import com.intellij.openapi.util.Pair; import com.intellij.openapi.util.text.StringUtil; import com.intellij.openapi.vcs.FilePath; import com.intellij.openapi.vfs.CharsetToolkit; @@ -100,10 +99,9 @@ public class FileAwareDocumentContent extends DocumentContentImpl { @NotNull private Builder create(@NotNull byte[] content) { assert myCharset != null; - // TODO: detect charset like in LoadTextUtil (Native2Ascii, etc) ? - Pair pair = CharsetToolkit.bytesToStringWithCharset(content, myCharset); - myCharset = pair.second; - return create(pair.first); + + myCharset = guessCharset(content, myCharset, myFileType, myHighlightFile); + return create(CharsetToolkit.decodeString(content, myCharset)); } @NotNull @@ -112,4 +110,27 @@ public class FileAwareDocumentContent extends DocumentContentImpl { return new FileAwareDocumentContent(myProject, myDocument, myFileType, myHighlightFile, mySeparator, myCharset); } } + + @NotNull + private static Charset guessCharset(@NotNull byte[] content, + @NotNull Charset currentCharset, + @Nullable FileType fileType, + @Nullable VirtualFile highlightFile) { + + CharsetToolkit toolkit = new CharsetToolkit(content, currentCharset); + toolkit.setEnforce8Bit(true); + + Charset charset = toolkit.guessFromBOM(); + + if (charset == null) { + CharsetToolkit.GuessedEncoding guessed = toolkit.guessFromContent(content.length); + if (guessed == CharsetToolkit.GuessedEncoding.VALID_UTF8) charset = CharsetToolkit.UTF8_CHARSET; + } + + if (charset == null) { + charset = currentCharset; + } + + return charset; + } } diff --git a/platform/util/src/com/intellij/openapi/vfs/CharsetToolkit.java b/platform/util/src/com/intellij/openapi/vfs/CharsetToolkit.java index 1b31e2b8bbb1..25a44601e7e6 100644 --- a/platform/util/src/com/intellij/openapi/vfs/CharsetToolkit.java +++ b/platform/util/src/com/intellij/openapi/vfs/CharsetToolkit.java @@ -15,7 +15,6 @@ */ package com.intellij.openapi.vfs; -import com.intellij.openapi.util.Pair; import com.intellij.util.ArrayUtil; import gnu.trove.THashMap; import org.jetbrains.annotations.NonNls; @@ -297,16 +296,16 @@ public class CharsetToolkit { @NotNull public static String bytesToString(@NotNull byte[] bytes, @NotNull final Charset defaultCharset) { - return bytesToStringWithCharset(bytes, defaultCharset).getFirst(); + Charset charset = new CharsetToolkit(bytes, defaultCharset).guessEncoding(bytes.length); + if (charset == null) charset = defaultCharset; // binary content. This is silly but method contract says to return something anyway + return decodeString(bytes, charset); } @NotNull - public static Pair bytesToStringWithCharset(@NotNull byte[] bytes, @NotNull final Charset defaultCharset) { - Charset charset = new CharsetToolkit(bytes, defaultCharset).guessEncoding(bytes.length); - if (charset == null) charset = defaultCharset; // binary content. This is silly but method contract says to return something anyway + public static String decodeString(@NotNull byte[] bytes, @NotNull final Charset charset) { int bomLength = getBOMLength(bytes, charset); final CharBuffer charBuffer = charset.decode(ByteBuffer.wrap(bytes, bomLength, bytes.length - bomLength)); - return Pair.create(charBuffer.toString(), charset); + return charBuffer.toString(); } public enum GuessedEncoding {