MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

diff: more careful detect charset

Browse Source 
* avoid detecting ASCII for files without special characters
This commit is contained in:
Aleksey Pivovarov
2015-04-17 20:09:37 +03:00
parent 9dc12b33c9
commit a97771cccf
2 changed files with 31 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/diff-impl/src/com/intellij/diff/contents/FileAwareDocumentContent.java
+26 -5
View File
@@ -7,7 +7,6 @@ import com.intellij.openapi.fileTypes.FileType;
import com.intellij.openapi.fileTypes.FileTypes;
import com.intellij.openapi.fileTypes.PlainTextFileType;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.util.Pair;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.openapi.vcs.FilePath;
import com.intellij.openapi.vfs.CharsetToolkit;
@@ -100,10 +99,9 @@ public class FileAwareDocumentContent extends DocumentContentImpl {
@NotNull
private Builder create(@NotNull byte[] content) {
assert myCharset != null;
// TODO: detect charset like in LoadTextUtil (Native2Ascii, etc) ?
Pair<String, Charset> pair = CharsetToolkit.bytesToStringWithCharset(content, myCharset);
myCharset = pair.second;
return create(pair.first);
myCharset = guessCharset(content, myCharset, myFileType, myHighlightFile);
return create(CharsetToolkit.decodeString(content, myCharset));
}
@NotNull
@@ -112,4 +110,27 @@ public class FileAwareDocumentContent extends DocumentContentImpl {
return new FileAwareDocumentContent(myProject, myDocument, myFileType, myHighlightFile, mySeparator, myCharset);
}
}
@NotNull
private static Charset guessCharset(@NotNull byte[] content,
@NotNull Charset currentCharset,
@Nullable FileType fileType,
@Nullable VirtualFile highlightFile) {
CharsetToolkit toolkit = new CharsetToolkit(content, currentCharset);
toolkit.setEnforce8Bit(true);
Charset charset = toolkit.guessFromBOM();
if (charset == null) {
CharsetToolkit.GuessedEncoding guessed = toolkit.guessFromContent(content.length);
if (guessed == CharsetToolkit.GuessedEncoding.VALID_UTF8) charset = CharsetToolkit.UTF8_CHARSET;
}
if (charset == null) {
charset = currentCharset;
}
return charset;
}
}
platform/util/src/com/intellij/openapi/vfs/CharsetToolkit.java
+5 -6
View File
@@ -15,7 +15,6 @@
*/
package com.intellij.openapi.vfs;
import com.intellij.openapi.util.Pair;
import com.intellij.util.ArrayUtil;
import gnu.trove.THashMap;
import org.jetbrains.annotations.NonNls;
@@ -297,16 +296,16 @@ public class CharsetToolkit {
@NotNull
public static String bytesToString(@NotNull byte[] bytes, @NotNull final Charset defaultCharset) {
return bytesToStringWithCharset(bytes, defaultCharset).getFirst();
Charset charset = new CharsetToolkit(bytes, defaultCharset).guessEncoding(bytes.length);
if (charset == null) charset = defaultCharset; // binary content. This is silly but method contract says to return something anyway
return decodeString(bytes, charset);
}
@NotNull
public static Pair<String, Charset> bytesToStringWithCharset(@NotNull byte[] bytes, @NotNull final Charset defaultCharset) {
Charset charset = new CharsetToolkit(bytes, defaultCharset).guessEncoding(bytes.length);
if (charset == null) charset = defaultCharset; // binary content. This is silly but method contract says to return something anyway
public static String decodeString(@NotNull byte[] bytes, @NotNull final Charset charset) {
int bomLength = getBOMLength(bytes, charset);
final CharBuffer charBuffer = charset.decode(ByteBuffer.wrap(bytes, bomLength, bytes.length - bomLength));
return Pair.create(charBuffer.toString(), charset);
return charBuffer.toString();
}
public enum GuessedEncoding {