diff --git a/platform/platform-api/src/com/intellij/openapi/diff/BinaryContent.java b/platform/platform-api/src/com/intellij/openapi/diff/BinaryContent.java index a1c9d7ff13f8..e1bfa17532bc 100644 --- a/platform/platform-api/src/com/intellij/openapi/diff/BinaryContent.java +++ b/platform/platform-api/src/com/intellij/openapi/diff/BinaryContent.java @@ -19,12 +19,12 @@ import com.intellij.openapi.editor.Document; import com.intellij.openapi.editor.EditorFactory; import com.intellij.openapi.fileEditor.OpenFileDescriptor; import com.intellij.openapi.fileTypes.FileType; +import com.intellij.openapi.vfs.CharsetToolkit; import com.intellij.openapi.vfs.VirtualFile; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.charset.Charset; import java.nio.charset.IllegalCharsetNameException; @@ -64,16 +64,14 @@ public class BinaryContent extends DiffContent { String text = null; try { if (myCharset == null) { - text = new String(myBytes); + text = CharsetToolkit.bytesToString(myBytes); } else { - text = new String(myBytes, myCharset.name()); + text = CharsetToolkit.bytesToString(myBytes, myCharset); } } catch (IllegalCharsetNameException e) { } - catch (UnsupportedEncodingException e) { - } // Still NULL? only if not supported or an exception was thrown. // Decode a string using the truly default encoding. diff --git a/platform/platform-api/src/com/intellij/openapi/vfs/CharsetToolkit.java b/platform/platform-api/src/com/intellij/openapi/vfs/CharsetToolkit.java index ae3de1aa1ae4..0206d8cec175 100644 --- a/platform/platform-api/src/com/intellij/openapi/vfs/CharsetToolkit.java +++ b/platform/platform-api/src/com/intellij/openapi/vfs/CharsetToolkit.java @@ -198,7 +198,12 @@ public class CharsetToolkit { @NotNull public static String bytesToString(@NotNull byte[] bytes) { - Charset charset = new CharsetToolkit(bytes, EncodingManager.getInstance().getDefaultCharset()).guessEncoding(bytes.length); + return bytesToString(bytes, EncodingManager.getInstance().getDefaultCharset()); + } + + @NotNull + public static String bytesToString(@NotNull byte[] bytes, @NotNull final Charset defaultCharset) { + Charset charset = new CharsetToolkit(bytes, defaultCharset).guessEncoding(bytes.length); int bomLength = getBOMLength(bytes, charset); final CharBuffer charBuffer = charset.decode(ByteBuffer.wrap(bytes, bomLength, bytes.length - bomLength)); return charBuffer.toString(); diff --git a/platform/platform-impl/src/com/intellij/openapi/fileEditor/impl/LoadTextUtil.java b/platform/platform-impl/src/com/intellij/openapi/fileEditor/impl/LoadTextUtil.java index 64b5df8ec1bb..8946b7244d77 100644 --- a/platform/platform-impl/src/com/intellij/openapi/fileEditor/impl/LoadTextUtil.java +++ b/platform/platform-impl/src/com/intellij/openapi/fileEditor/impl/LoadTextUtil.java @@ -361,6 +361,7 @@ public final class LoadTextUtil { return convertBytes(bytes, charset, offset).getFirst(); } + // do not need to think about BOM here. it is processed outside @NotNull private static Pair convertBytes(@NotNull byte[] bytes, Charset charset, final int startOffset) { ByteBuffer byteBuffer = ByteBuffer.wrap(bytes, startOffset, bytes.length - startOffset); diff --git a/platform/vcs-api/src/com/intellij/openapi/vcs/vfs/VcsVirtualFile.java b/platform/vcs-api/src/com/intellij/openapi/vcs/vfs/VcsVirtualFile.java index bccbf2439017..a0ae2a47cb52 100644 --- a/platform/vcs-api/src/com/intellij/openapi/vcs/vfs/VcsVirtualFile.java +++ b/platform/vcs-api/src/com/intellij/openapi/vcs/vfs/VcsVirtualFile.java @@ -22,11 +22,13 @@ import com.intellij.openapi.ui.Messages; import com.intellij.openapi.vcs.VcsBundle; import com.intellij.openapi.vcs.VcsException; import com.intellij.openapi.vcs.history.VcsFileRevision; +import com.intellij.openapi.vfs.CharsetToolkit; import com.intellij.openapi.vfs.VirtualFileSystem; import com.intellij.util.ArrayUtil; import org.jetbrains.annotations.NotNull; import java.io.IOException; +import java.nio.charset.Charset; /** * author: lesya @@ -37,6 +39,7 @@ public class VcsVirtualFile extends AbstractVcsVirtualFile { private byte[] myContent; private final VcsFileRevision myFileRevision; private boolean myContentLoadFailed = false; + private Charset myCharset; public VcsVirtualFile(String path, VcsFileRevision revision, VirtualFileSystem fileSystem) { @@ -75,6 +78,7 @@ public class VcsVirtualFile extends AbstractVcsVirtualFile { myModificationStamp++; setRevision(myFileRevision.getRevisionNumber().asString()); myContent = myFileRevision.getContent(); + myCharset = new CharsetToolkit(myContent).guessEncoding(myContent.length); ApplicationManager.getApplication().runWriteAction(new Runnable() { public void run() { vcsFileSystem.fireContentsChanged(this, VcsVirtualFile.this, 0); @@ -110,6 +114,11 @@ public class VcsVirtualFile extends AbstractVcsVirtualFile { } + @Override + public Charset getCharset() { + if (myCharset != null) return myCharset; + return super.getCharset(); + } public boolean isDirectory() { return false; diff --git a/platform/vcs-impl/src/com/intellij/openapi/vcs/history/FileHistoryPanelImpl.java b/platform/vcs-impl/src/com/intellij/openapi/vcs/history/FileHistoryPanelImpl.java index fd30de097b8f..0bb332887532 100644 --- a/platform/vcs-impl/src/com/intellij/openapi/vcs/history/FileHistoryPanelImpl.java +++ b/platform/vcs-impl/src/com/intellij/openapi/vcs/history/FileHistoryPanelImpl.java @@ -24,7 +24,6 @@ import com.intellij.openapi.command.CommandProcessor; import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.editor.Document; import com.intellij.openapi.fileEditor.OpenFileDescriptor; -import com.intellij.openapi.fileEditor.impl.LoadTextUtil; import com.intellij.openapi.fileTypes.FileType; import com.intellij.openapi.progress.ProcessCanceledException; import com.intellij.openapi.progress.ProgressIndicator; @@ -728,6 +727,10 @@ public class FileHistoryPanelImpl 0) && (selectionSize < 3)); @@ -933,7 +936,9 @@ public class FileHistoryPanelImpl getProcessedFiles() { diff --git a/plugins/cvs/cvs-plugin/src/com/intellij/cvsSupport2/changeBrowser/CvsContentRevision.java b/plugins/cvs/cvs-plugin/src/com/intellij/cvsSupport2/changeBrowser/CvsContentRevision.java index 2b1c6a962d6a..c683be57fbfe 100644 --- a/plugins/cvs/cvs-plugin/src/com/intellij/cvsSupport2/changeBrowser/CvsContentRevision.java +++ b/plugins/cvs/cvs-plugin/src/com/intellij/cvsSupport2/changeBrowser/CvsContentRevision.java @@ -43,6 +43,7 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import java.io.File; +import java.nio.charset.Charset; public class CvsContentRevision implements ContentRevision { protected final RevisionOrDate myRevision; @@ -70,7 +71,8 @@ public class CvsContentRevision implements ContentRevision { if (myContent == null) { byte[] content = loadContent(); if (content != null) { - myContent = CharsetToolkit.bytesToString(content); + final Charset charset = myLocalFile.getCharset(); + myContent = charset == null ? CharsetToolkit.bytesToString(content) : CharsetToolkit.bytesToString(content, charset); } } return myContent; @@ -114,4 +116,4 @@ public class CvsContentRevision implements ContentRevision { public String toString() { return "CvsContentRevision:" + myFile + "@" + myRevision; } -} \ No newline at end of file +} diff --git a/plugins/cvs/cvs-plugin/src/com/intellij/cvsSupport2/cvsstatuses/CvsChangeProvider.java b/plugins/cvs/cvs-plugin/src/com/intellij/cvsSupport2/cvsstatuses/CvsChangeProvider.java index 76413b1a4b57..ea9d65a894d2 100644 --- a/plugins/cvs/cvs-plugin/src/com/intellij/cvsSupport2/cvsstatuses/CvsChangeProvider.java +++ b/plugins/cvs/cvs-plugin/src/com/intellij/cvsSupport2/cvsstatuses/CvsChangeProvider.java @@ -49,6 +49,7 @@ import com.intellij.openapi.vcs.VcsException; import com.intellij.openapi.vcs.actions.VcsContextFactory; import com.intellij.openapi.vcs.changes.*; import com.intellij.openapi.vcs.history.VcsRevisionNumber; +import com.intellij.openapi.vfs.CharsetToolkit; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.util.containers.HashMap; import com.intellij.vcsUtil.VcsUtil; @@ -58,7 +59,6 @@ import org.jetbrains.annotations.Nullable; import org.netbeans.lib.cvsclient.admin.Entry; import java.io.File; -import java.io.UnsupportedEncodingException; import java.text.ParseException; import java.util.*; @@ -553,14 +553,11 @@ public class CvsChangeProvider implements ChangeProvider { if (myContent == null) { try { byte[] fileBytes = getUpToDateBinaryContent(); - myContent = fileBytes == null ? null : new String(fileBytes, myPath.getCharset().name()); + myContent = fileBytes == null ? null : CharsetToolkit.bytesToString(fileBytes, myPath.getCharset()); } catch (CannotFindCvsRootException e) { myContent = null; } - catch (UnsupportedEncodingException e) { - myContent = null; - } } return myContent; } diff --git a/plugins/cvs/cvs-plugin/src/com/intellij/cvsSupport2/javacvsImpl/io/ReceiveTextFilePreprocessor.java b/plugins/cvs/cvs-plugin/src/com/intellij/cvsSupport2/javacvsImpl/io/ReceiveTextFilePreprocessor.java index dcdda8339bca..c29d89a22dfb 100644 --- a/plugins/cvs/cvs-plugin/src/com/intellij/cvsSupport2/javacvsImpl/io/ReceiveTextFilePreprocessor.java +++ b/plugins/cvs/cvs-plugin/src/com/intellij/cvsSupport2/javacvsImpl/io/ReceiveTextFilePreprocessor.java @@ -17,8 +17,8 @@ package com.intellij.cvsSupport2.javacvsImpl.io; import com.intellij.cvsSupport2.cvsoperations.common.ReceivedFileProcessor; import com.intellij.cvsSupport2.util.CvsVfsUtil; -import com.intellij.openapi.vfs.VirtualFile; import com.intellij.openapi.vfs.CharsetToolkit; +import com.intellij.openapi.vfs.VirtualFile; import com.intellij.psi.codeStyle.CodeStyleSettingsManager; import com.intellij.util.containers.HashMap; import com.intellij.util.text.LineReader; @@ -26,7 +26,6 @@ import org.netbeans.lib.cvsclient.file.IReaderFactory; import org.netbeans.lib.cvsclient.file.IReceiveTextFilePreprocessor; import java.io.*; -import java.nio.ByteBuffer; import java.nio.charset.Charset; import java.util.Collection; import java.util.Iterator; @@ -62,7 +61,7 @@ public class ReceiveTextFilePreprocessor implements IReceiveTextFilePreprocessor target.write(bytes); } else { - target.write(charSet.encode(CharsetToolkit.UTF8_CHARSET.decode(ByteBuffer.wrap(bytes))).array()); + target.write(charSet.encode(CharsetToolkit.bytesToString(bytes, CharsetToolkit.UTF8_CHARSET)).array()); } if (each.hasNext()) { if (charSet == null) @@ -94,4 +93,4 @@ public class ReceiveTextFilePreprocessor implements IReceiveTextFilePreprocessor public void saveLineSeparatorForFile(VirtualFile virtualFile, String lineSeparatorFor) { myFileToSeparator.put(CvsVfsUtil.getFileFor(virtualFile), lineSeparatorFor); } -} \ No newline at end of file +} diff --git a/plugins/git4idea/src/git4idea/GitContentRevision.java b/plugins/git4idea/src/git4idea/GitContentRevision.java index c2da85989453..98c870b8adb7 100644 --- a/plugins/git4idea/src/git4idea/GitContentRevision.java +++ b/plugins/git4idea/src/git4idea/GitContentRevision.java @@ -23,6 +23,7 @@ import com.intellij.openapi.vcs.VcsException; import com.intellij.openapi.vcs.changes.ContentRevision; import com.intellij.openapi.vcs.changes.CurrentContentRevision; import com.intellij.openapi.vcs.history.VcsRevisionNumber; +import com.intellij.openapi.vfs.CharsetToolkit; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.vcsUtil.VcsUtil; import git4idea.commands.GitFileUtils; @@ -73,7 +74,7 @@ public class GitContentRevision implements ContentRevision { if (myCharset == null) { myCharset = myFile.getCharset(myProject); } - return result == null ? null : new String(result, myCharset); + return result == null ? null : CharsetToolkit.bytesToString(result, myCharset); } @NotNull diff --git a/plugins/svn4idea/src/org/jetbrains/idea/svn/SvnContentRevision.java b/plugins/svn4idea/src/org/jetbrains/idea/svn/SvnContentRevision.java index 726a755be414..c9f53befcb2c 100644 --- a/plugins/svn4idea/src/org/jetbrains/idea/svn/SvnContentRevision.java +++ b/plugins/svn4idea/src/org/jetbrains/idea/svn/SvnContentRevision.java @@ -20,6 +20,7 @@ import com.intellij.openapi.vcs.FilePath; import com.intellij.openapi.vcs.VcsException; import com.intellij.openapi.vcs.changes.ContentRevision; import com.intellij.openapi.vcs.history.VcsRevisionNumber; +import com.intellij.openapi.vfs.CharsetToolkit; import org.jetbrains.annotations.NonNls; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -71,7 +72,7 @@ class SvnContentRevision implements ContentRevision { try { final byte[] byteContent = getUpToDateBinaryContent(); if (byteContent != null) { - content = new String(byteContent, myFile.getCharset().name()); + content = CharsetToolkit.bytesToString(byteContent, myFile.getCharset()); myContent = new SoftReference(content); } } diff --git a/plugins/svn4idea/src/org/jetbrains/idea/svn/actions/CompareWithBranchAction.java b/plugins/svn4idea/src/org/jetbrains/idea/svn/actions/CompareWithBranchAction.java index 8d6c0599d059..f487f65b7f97 100644 --- a/plugins/svn4idea/src/org/jetbrains/idea/svn/actions/CompareWithBranchAction.java +++ b/plugins/svn4idea/src/org/jetbrains/idea/svn/actions/CompareWithBranchAction.java @@ -35,6 +35,7 @@ import com.intellij.openapi.vcs.AbstractVcsHelper; import com.intellij.openapi.vcs.FileStatus; import com.intellij.openapi.vcs.FileStatusManager; import com.intellij.openapi.vcs.changes.Change; +import com.intellij.openapi.vfs.CharsetToolkit; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.util.WaitForProgressToShow; import org.jetbrains.annotations.Nullable; @@ -57,7 +58,6 @@ import org.tmatesoft.svn.util.SVNLogType; import java.io.ByteArrayOutputStream; import java.io.File; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -213,9 +213,8 @@ public class CompareWithBranchAction extends AnAction implements DumbAware { if (success.isNull()) { return; } - ByteBuffer contents = ByteBuffer.wrap(baos.toByteArray()); SimpleDiffRequest req = new SimpleDiffRequest(myProject, SvnBundle.message("compare.with.branch.diff.title")); - req.setContents(new SimpleContent(myVirtualFile.getCharset().decode(contents).toString()), + req.setContents(new SimpleContent(CharsetToolkit.bytesToString(baos.toByteArray(), myVirtualFile.getCharset())), new FileContent(myProject, myVirtualFile)); req.setContentTitles(remoteTitleBuilder.toString(), myVirtualFile.getPresentableUrl()); DiffManager.getInstance().getDiffTool().show(req); diff --git a/plugins/svn4idea/src/org/jetbrains/idea/svn/annotate/SvnAnnotationProvider.java b/plugins/svn4idea/src/org/jetbrains/idea/svn/annotate/SvnAnnotationProvider.java index 7ac5a2a4ce41..b126b3459be3 100644 --- a/plugins/svn4idea/src/org/jetbrains/idea/svn/annotate/SvnAnnotationProvider.java +++ b/plugins/svn4idea/src/org/jetbrains/idea/svn/annotate/SvnAnnotationProvider.java @@ -23,6 +23,7 @@ import com.intellij.openapi.vcs.VcsException; import com.intellij.openapi.vcs.annotate.AnnotationProvider; import com.intellij.openapi.vcs.annotate.FileAnnotation; import com.intellij.openapi.vcs.history.VcsFileRevision; +import com.intellij.openapi.vcs.history.VcsHistoryUtil; import com.intellij.openapi.vfs.VirtualFile; import org.jetbrains.idea.svn.*; import org.jetbrains.idea.svn.history.SvnFileRevision; @@ -70,8 +71,8 @@ public class SvnAnnotationProvider implements AnnotationProvider { myVcs.createWCClient().doGetFileContents(ioFile, SVNRevision.UNDEFINED, SVNRevision.BASE, true, buffer); contents = LoadTextUtil.getTextByBinaryPresentation(buffer.toByteArray(), file, false).toString(); } else { - revision.loadContent(); - contents = LoadTextUtil.getTextByBinaryPresentation(revision.getContent(), file, false).toString(); + final byte[] bytes = VcsHistoryUtil.loadRevisionContent(revision); + contents = LoadTextUtil.getTextByBinaryPresentation(bytes, file, false).toString(); } final SvnFileAnnotation result = new SvnFileAnnotation(myVcs, file, contents); diff --git a/plugins/svn4idea/src/org/jetbrains/idea/svn/history/SvnRepositoryContentRevision.java b/plugins/svn4idea/src/org/jetbrains/idea/svn/history/SvnRepositoryContentRevision.java index dbe3c94c211a..62aa2037f42c 100644 --- a/plugins/svn4idea/src/org/jetbrains/idea/svn/history/SvnRepositoryContentRevision.java +++ b/plugins/svn4idea/src/org/jetbrains/idea/svn/history/SvnRepositoryContentRevision.java @@ -71,7 +71,7 @@ public class SvnRepositoryContentRevision implements ContentRevision { public String getContent() throws VcsException { if (myContent == null) { final ByteArrayOutputStream buffer = loadContent(); - myContent = CharsetToolkit.bytesToString(buffer.toByteArray()); + myContent = CharsetToolkit.bytesToString(buffer.toByteArray(), myFilePath.getCharset()); } return myContent; } @@ -170,4 +170,4 @@ public class SvnRepositoryContentRevision implements ContentRevision { public String getPath() { return myPath; } -} \ No newline at end of file +} diff --git a/plugins/svn4idea/src/org/jetbrains/idea/svn/status/DiffContentRevision.java b/plugins/svn4idea/src/org/jetbrains/idea/svn/status/DiffContentRevision.java index 3cb9c35ae034..2d280b21d3ec 100644 --- a/plugins/svn4idea/src/org/jetbrains/idea/svn/status/DiffContentRevision.java +++ b/plugins/svn4idea/src/org/jetbrains/idea/svn/status/DiffContentRevision.java @@ -20,12 +20,15 @@ import com.intellij.openapi.vcs.FilePath; import com.intellij.openapi.vcs.VcsException; import com.intellij.openapi.vcs.changes.ContentRevision; import com.intellij.openapi.vcs.history.VcsRevisionNumber; +import com.intellij.openapi.vfs.CharsetToolkit; import com.intellij.vcsUtil.VcsUtil; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.tmatesoft.svn.core.SVNException; import org.tmatesoft.svn.core.io.SVNRepository; +import java.nio.charset.Charset; + public class DiffContentRevision implements ContentRevision { private String myPath; private SVNRepository myRepository; @@ -54,7 +57,9 @@ public class DiffContentRevision implements ContentRevision { } catch (SVNException e) { throw new VcsException(e); } - myContents = new String(bos.getInternalBuffer(), 0, bos.size()); + final byte[] bytes = bos.toByteArray(); + final Charset charset = myFilePath.getCharset(); + myContents = charset == null ? CharsetToolkit.bytesToString(bytes) : CharsetToolkit.bytesToString(bytes, charset); } return myContents; }