VCS: encoding - always try to take into account file encoding, not default one; cut BOM at the beginning

platform/platform-api/src/com/intellij/openapi/diff/BinaryContent.java

@@ -19,12 +19,12 @@ import com.intellij.openapi.editor.Document;
 import com.intellij.openapi.editor.EditorFactory;
 import com.intellij.openapi.fileEditor.OpenFileDescriptor;
 import com.intellij.openapi.fileTypes.FileType;
+import com.intellij.openapi.vfs.CharsetToolkit;
 import com.intellij.openapi.vfs.VirtualFile;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
 import java.io.IOException;
-import java.io.UnsupportedEncodingException;
 import java.nio.charset.Charset;
 import java.nio.charset.IllegalCharsetNameException;
 
@@ -64,16 +64,14 @@ public class BinaryContent extends DiffContent {
       String text = null;
       try {
         if (myCharset == null) {
-          text = new String(myBytes);
+          text = CharsetToolkit.bytesToString(myBytes);
         }
         else {
-          text = new String(myBytes, myCharset.name());
+          text = CharsetToolkit.bytesToString(myBytes, myCharset);
         }
       }
       catch (IllegalCharsetNameException e) {
       }
-      catch (UnsupportedEncodingException e) {
-      }
 
       //  Still NULL? only if not supported or an exception was thrown.
       //  Decode a string using the truly default encoding.

platform/platform-api/src/com/intellij/openapi/vfs/CharsetToolkit.java

@@ -198,7 +198,12 @@ public class CharsetToolkit {
 
   @NotNull
   public static String bytesToString(@NotNull byte[] bytes) {
-    Charset charset = new CharsetToolkit(bytes, EncodingManager.getInstance().getDefaultCharset()).guessEncoding(bytes.length);
+    return bytesToString(bytes, EncodingManager.getInstance().getDefaultCharset());
+  }
+
+  @NotNull
+  public static String bytesToString(@NotNull byte[] bytes, @NotNull final Charset defaultCharset) {
+    Charset charset = new CharsetToolkit(bytes, defaultCharset).guessEncoding(bytes.length);
     int bomLength = getBOMLength(bytes, charset);
     final CharBuffer charBuffer = charset.decode(ByteBuffer.wrap(bytes, bomLength, bytes.length - bomLength));
     return charBuffer.toString();

platform/platform-impl/src/com/intellij/openapi/fileEditor/impl/LoadTextUtil.java

@@ -361,6 +361,7 @@ public final class LoadTextUtil {
     return convertBytes(bytes, charset, offset).getFirst();
   }
 
+  // do not need to think about BOM here. it is processed outside
   @NotNull
   private static Pair<CharSequence, String> convertBytes(@NotNull byte[] bytes, Charset charset, final int startOffset) {
     ByteBuffer byteBuffer = ByteBuffer.wrap(bytes, startOffset, bytes.length - startOffset);

platform/vcs-api/src/com/intellij/openapi/vcs/vfs/VcsVirtualFile.java

@@ -22,11 +22,13 @@ import com.intellij.openapi.ui.Messages;
 import com.intellij.openapi.vcs.VcsBundle;
 import com.intellij.openapi.vcs.VcsException;
 import com.intellij.openapi.vcs.history.VcsFileRevision;
+import com.intellij.openapi.vfs.CharsetToolkit;
 import com.intellij.openapi.vfs.VirtualFileSystem;
 import com.intellij.util.ArrayUtil;
 import org.jetbrains.annotations.NotNull;
 
 import java.io.IOException;
+import java.nio.charset.Charset;
 
 /**
  * author: lesya
@@ -37,6 +39,7 @@ public class VcsVirtualFile extends AbstractVcsVirtualFile {
   private byte[] myContent;
   private final VcsFileRevision myFileRevision;
   private boolean myContentLoadFailed = false;
+  private Charset myCharset;
 
   public VcsVirtualFile(String path,
                         VcsFileRevision revision, VirtualFileSystem fileSystem) {
@@ -75,6 +78,7 @@ public class VcsVirtualFile extends AbstractVcsVirtualFile {
       myModificationStamp++;
       setRevision(myFileRevision.getRevisionNumber().asString());
       myContent = myFileRevision.getContent();
+      myCharset = new CharsetToolkit(myContent).guessEncoding(myContent.length);
       ApplicationManager.getApplication().runWriteAction(new Runnable() {
         public void run() {
           vcsFileSystem.fireContentsChanged(this, VcsVirtualFile.this, 0);
@@ -110,6 +114,11 @@ public class VcsVirtualFile extends AbstractVcsVirtualFile {
 
   }
 
+  @Override
+  public Charset getCharset() {
+    if (myCharset != null) return myCharset;
+    return super.getCharset();
+  }
 
   public boolean isDirectory() {
     return false;

platform/vcs-impl/src/com/intellij/openapi/vcs/history/FileHistoryPanelImpl.java

@@ -24,7 +24,6 @@ import com.intellij.openapi.command.CommandProcessor;
 import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.openapi.editor.Document;
 import com.intellij.openapi.fileEditor.OpenFileDescriptor;
-import com.intellij.openapi.fileEditor.impl.LoadTextUtil;
 import com.intellij.openapi.fileTypes.FileType;
 import com.intellij.openapi.progress.ProcessCanceledException;
 import com.intellij.openapi.progress.ProgressIndicator;
@@ -728,6 +727,10 @@ public class FileHistoryPanelImpl<S extends CommittedChangeList, U extends Chang
     result.add(new CreatePatchFromChangesAction() {
       public void update(final AnActionEvent e) {
         e.getPresentation().setVisible(true);
+        if (myFilePath.isNonLocal()) {
+          e.getPresentation().setEnabled(false);
+          return;
+        }
         // in order to do not load changes only for action update
         final int selectionSize = getSelection().size();
         e.getPresentation().setEnabled((selectionSize > 0) && (selectionSize < 3));
@@ -933,7 +936,9 @@ public class FileHistoryPanelImpl<S extends CommittedChangeList, U extends Chang
         revisionContent = VcsHistoryUtil.loadRevisionContent(revision);
       }
       catch (IOException e) {
-        LOG.error(e);
+        LOG.info(e);
+        Messages.showMessageDialog(VcsBundle.message("message.text.cannot.load.revision", e.getLocalizedMessage()),
+                                   VcsBundle.message("message.title.get.revision.content"), Messages.getInformationIcon());
         return;
       }
       catch (VcsException e) {
@@ -1167,9 +1172,9 @@ public class FileHistoryPanelImpl<S extends CommittedChangeList, U extends Chang
         }
       }
 
-      final ContentRevision startRevision = new LoadedContentRevision(myFilePath, revisions[0]);
+      final ContentRevision startRevision = new LoadedContentRevision(myFilePath, revisions[0], myVcs.getProject());
       final ContentRevision endRevision = (revisions.length == 1) ? new CurrentContentRevision(myFilePath) :
-                                          new LoadedContentRevision(myFilePath, revisions[revisions.length - 1]);
+                                          new LoadedContentRevision(myFilePath, revisions[revisions.length - 1], myVcs.getProject());
 
       return new Change[]{new Change(startRevision, endRevision)};
     }
@@ -1179,21 +1184,21 @@ public class FileHistoryPanelImpl<S extends CommittedChangeList, U extends Chang
   private static class LoadedContentRevision implements ContentRevision {
     private final FilePath myFile;
     private final VcsFileRevision myRevision;
+    private final Project myProject;
 
-    private LoadedContentRevision(final FilePath file, final VcsFileRevision revision) {
+    private LoadedContentRevision(final FilePath file, final VcsFileRevision revision, final Project project) {
       myFile = file;
       myRevision = revision;
+      myProject = project;
     }
 
     public String getContent() throws VcsException {
-      final byte[] bytes;
       try {
-        bytes = VcsHistoryUtil.loadRevisionContent(myRevision);
+        return VcsHistoryUtil.loadRevisionContentGuessEncoding(myRevision, myFile.getVirtualFile(), myProject);
       }
       catch (IOException e) {
         throw new VcsException(VcsBundle.message("message.text.cannot.load.revision", e.getLocalizedMessage()));
       }
-      return LoadTextUtil.getTextByBinaryPresentation(bytes, myFile.getVirtualFile(), false).toString();
     }
 
     @NotNull

platform/vcs-impl/src/com/intellij/openapi/vcs/history/VcsHistoryUtil.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2009 JetBrains s.r.o.
+ * Copyright 2000-2011 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,6 +21,11 @@ import com.intellij.openapi.fileTypes.FileType;
 import com.intellij.openapi.project.Project;
 import com.intellij.openapi.vcs.FilePath;
 import com.intellij.openapi.vcs.VcsException;
+import com.intellij.openapi.vfs.CharsetToolkit;
+import com.intellij.openapi.vfs.VirtualFile;
+import com.intellij.openapi.vfs.encoding.EncodingManager;
+import com.intellij.openapi.vfs.encoding.EncodingProjectManager;
+import org.jetbrains.annotations.Nullable;
 
 import java.io.IOException;
 import java.nio.charset.Charset;
@@ -88,6 +93,31 @@ public class VcsHistoryUtil {
     return content;
   }
 
+  public static String loadRevisionContentGuessEncoding(final VcsFileRevision revision, @Nullable final VirtualFile file,
+                                                        @Nullable final Project project) throws VcsException, IOException {
+    final byte[] bytes = loadRevisionContent(revision);
+    if (file != null) {
+      return new String(bytes, file.getCharset());
+    }
+    EncodingManager e = project != null ? EncodingProjectManager.getInstance(project) : null;
+    if (e == null) {
+      e = EncodingManager.getInstance();
+    }
+
+    return CharsetToolkit.bytesToString(bytes, e.getDefaultCharset());
+  }
+
+  public static String loadRevisionContentGuessEncoding(final VcsFileRevision revision, @Nullable final Project project) throws VcsException, IOException {
+    final byte[] bytes = loadRevisionContent(revision);
+
+    EncodingManager e = project != null ? EncodingProjectManager.getInstance(project) : null;
+    if (e == null) {
+      e = EncodingManager.getInstance();
+    }
+
+    return CharsetToolkit.bytesToString(bytes, e.getDefaultCharset());
+  }
+
   private static DiffContent createContent(Project project, byte[] content1, VcsFileRevision revision, Document doc, Charset charset, FileType fileType) {
     if (isCurrent(revision) && (doc != null)) { return new DocumentContent(project, doc); }
     return new BinaryContent(content1, charset, fileType);

platform/vcs-impl/src/com/intellij/openapi/vcs/impl/AbstractVcsHelperImpl.java

@@ -648,13 +648,14 @@ public class AbstractVcsHelperImpl extends AbstractVcsHelper {
     return showMergeDialog(files, provider);
   }
 
-  private static DiffContent getContentForVersion(final VcsFileRevision version, final File file) throws IOException {
+  private static DiffContent getContentForVersion(final VcsFileRevision version, final File file) throws IOException, VcsException {
     VirtualFile vFile = LocalFileSystem.getInstance().findFileByIoFile(file);
     if (vFile != null && (version instanceof CurrentRevision) && !vFile.getFileType().isBinary()) {
       return new DocumentContent(FileDocumentManager.getInstance().getDocument(vFile), vFile.getFileType());
     }
     else {
-      return new SimpleContent(new String(version.getContent()), FileTypeManager.getInstance().getFileTypeByFileName(file.getName()));
+      return new SimpleContent(VcsHistoryUtil.loadRevisionContentGuessEncoding(version, vFile, null),
+                               FileTypeManager.getInstance().getFileTypeByFileName(file.getName()));
     }
   }
 

platform/vcs-impl/src/com/intellij/openapi/vcs/merge/MultipleFileMergeDialog.java

@@ -34,6 +34,7 @@ import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.openapi.vcs.VcsBundle;
 import com.intellij.openapi.vcs.VcsException;
 import com.intellij.openapi.vcs.changes.VcsDirtyScopeManager;
+import com.intellij.openapi.vfs.CharsetToolkit;
 import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.projectImport.ProjectOpenProcessor;
 import com.intellij.ui.ColoredTableCellRenderer;
@@ -51,7 +52,6 @@ import javax.swing.event.ListSelectionListener;
 import javax.swing.table.TableCellRenderer;
 import java.awt.event.ActionEvent;
 import java.awt.event.ActionListener;
-import java.nio.ByteBuffer;
 import java.util.*;
 
 /**
@@ -330,7 +330,7 @@ public class MultipleFileMergeDialog extends DialogWrapper {
   }
 
   private static String decodeContent(final VirtualFile file, final byte[] content) {
-    return StringUtil.convertLineSeparators(file.getCharset().decode(ByteBuffer.wrap(content)).toString());
+    return StringUtil.convertLineSeparators(CharsetToolkit.bytesToString(content, file.getCharset()));
   }
 
   public List<VirtualFile> getProcessedFiles() {

plugins/cvs/cvs-plugin/src/com/intellij/cvsSupport2/changeBrowser/CvsContentRevision.java

@@ -43,6 +43,7 @@ import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
 import java.io.File;
+import java.nio.charset.Charset;
 
 public class CvsContentRevision implements ContentRevision {
   protected final RevisionOrDate myRevision;
@@ -70,7 +71,8 @@ public class CvsContentRevision implements ContentRevision {
     if (myContent == null) {
       byte[] content = loadContent();
       if (content != null) {
-        myContent = CharsetToolkit.bytesToString(content);
+        final Charset charset = myLocalFile.getCharset();
+        myContent = charset == null ? CharsetToolkit.bytesToString(content) : CharsetToolkit.bytesToString(content, charset);
       }
     }
     return myContent;
@@ -114,4 +116,4 @@ public class CvsContentRevision implements ContentRevision {
   public String toString() {
     return "CvsContentRevision:" + myFile + "@" + myRevision;
   }
-}
+}

plugins/cvs/cvs-plugin/src/com/intellij/cvsSupport2/cvsstatuses/CvsChangeProvider.java

@@ -49,6 +49,7 @@ import com.intellij.openapi.vcs.VcsException;
 import com.intellij.openapi.vcs.actions.VcsContextFactory;
 import com.intellij.openapi.vcs.changes.*;
 import com.intellij.openapi.vcs.history.VcsRevisionNumber;
+import com.intellij.openapi.vfs.CharsetToolkit;
 import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.util.containers.HashMap;
 import com.intellij.vcsUtil.VcsUtil;
@@ -58,7 +59,6 @@ import org.jetbrains.annotations.Nullable;
 import org.netbeans.lib.cvsclient.admin.Entry;
 
 import java.io.File;
-import java.io.UnsupportedEncodingException;
 import java.text.ParseException;
 import java.util.*;
 
@@ -553,14 +553,11 @@ public class CvsChangeProvider implements ChangeProvider {
       if (myContent == null) {
         try {
           byte[] fileBytes = getUpToDateBinaryContent();
-          myContent = fileBytes == null ? null : new String(fileBytes, myPath.getCharset().name());
+          myContent = fileBytes == null ? null : CharsetToolkit.bytesToString(fileBytes, myPath.getCharset());
         }
         catch (CannotFindCvsRootException e) {
           myContent = null;
         }
-        catch (UnsupportedEncodingException e) {
-          myContent = null;
-        }
       }
       return myContent;
     }

plugins/cvs/cvs-plugin/src/com/intellij/cvsSupport2/javacvsImpl/io/ReceiveTextFilePreprocessor.java

@@ -17,8 +17,8 @@ package com.intellij.cvsSupport2.javacvsImpl.io;
 
 import com.intellij.cvsSupport2.cvsoperations.common.ReceivedFileProcessor;
 import com.intellij.cvsSupport2.util.CvsVfsUtil;
-import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.openapi.vfs.CharsetToolkit;
+import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.psi.codeStyle.CodeStyleSettingsManager;
 import com.intellij.util.containers.HashMap;
 import com.intellij.util.text.LineReader;
@@ -26,7 +26,6 @@ import org.netbeans.lib.cvsclient.file.IReaderFactory;
 import org.netbeans.lib.cvsclient.file.IReceiveTextFilePreprocessor;
 
 import java.io.*;
-import java.nio.ByteBuffer;
 import java.nio.charset.Charset;
 import java.util.Collection;
 import java.util.Iterator;
@@ -62,7 +61,7 @@ public class ReceiveTextFilePreprocessor implements IReceiveTextFilePreprocessor
               target.write(bytes);
             }
             else {
-              target.write(charSet.encode(CharsetToolkit.UTF8_CHARSET.decode(ByteBuffer.wrap(bytes))).array());
+              target.write(charSet.encode(CharsetToolkit.bytesToString(bytes, CharsetToolkit.UTF8_CHARSET)).array());
             }
             if (each.hasNext()) {
               if (charSet == null)
@@ -94,4 +93,4 @@ public class ReceiveTextFilePreprocessor implements IReceiveTextFilePreprocessor
   public void saveLineSeparatorForFile(VirtualFile virtualFile, String lineSeparatorFor) {
     myFileToSeparator.put(CvsVfsUtil.getFileFor(virtualFile), lineSeparatorFor);
   }
-}
+}

plugins/git4idea/src/git4idea/GitContentRevision.java

@@ -23,6 +23,7 @@ import com.intellij.openapi.vcs.VcsException;
 import com.intellij.openapi.vcs.changes.ContentRevision;
 import com.intellij.openapi.vcs.changes.CurrentContentRevision;
 import com.intellij.openapi.vcs.history.VcsRevisionNumber;
+import com.intellij.openapi.vfs.CharsetToolkit;
 import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.vcsUtil.VcsUtil;
 import git4idea.commands.GitFileUtils;
@@ -73,7 +74,7 @@ public class GitContentRevision implements ContentRevision {
     if (myCharset == null) {
       myCharset = myFile.getCharset(myProject);
     }
-    return result == null ? null : new String(result, myCharset);
+    return result == null ? null : CharsetToolkit.bytesToString(result, myCharset);
   }
 
   @NotNull

plugins/svn4idea/src/org/jetbrains/idea/svn/SvnContentRevision.java

@@ -20,6 +20,7 @@ import com.intellij.openapi.vcs.FilePath;
 import com.intellij.openapi.vcs.VcsException;
 import com.intellij.openapi.vcs.changes.ContentRevision;
 import com.intellij.openapi.vcs.history.VcsRevisionNumber;
+import com.intellij.openapi.vfs.CharsetToolkit;
 import org.jetbrains.annotations.NonNls;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
@@ -71,7 +72,7 @@ class SvnContentRevision implements ContentRevision {
       try {
         final byte[] byteContent = getUpToDateBinaryContent();
         if (byteContent != null) {
-          content = new String(byteContent, myFile.getCharset().name());
+          content = CharsetToolkit.bytesToString(byteContent, myFile.getCharset());
           myContent = new SoftReference<String>(content);
         }
       }

plugins/svn4idea/src/org/jetbrains/idea/svn/actions/CompareWithBranchAction.java

@@ -35,6 +35,7 @@ import com.intellij.openapi.vcs.AbstractVcsHelper;
 import com.intellij.openapi.vcs.FileStatus;
 import com.intellij.openapi.vcs.FileStatusManager;
 import com.intellij.openapi.vcs.changes.Change;
+import com.intellij.openapi.vfs.CharsetToolkit;
 import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.util.WaitForProgressToShow;
 import org.jetbrains.annotations.Nullable;
@@ -57,7 +58,6 @@ import org.tmatesoft.svn.util.SVNLogType;
 
 import java.io.ByteArrayOutputStream;
 import java.io.File;
-import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -213,9 +213,8 @@ public class CompareWithBranchAction extends AnAction implements DumbAware {
       if (success.isNull()) {
         return;
       }
-      ByteBuffer contents = ByteBuffer.wrap(baos.toByteArray());
       SimpleDiffRequest req = new SimpleDiffRequest(myProject, SvnBundle.message("compare.with.branch.diff.title"));
-      req.setContents(new SimpleContent(myVirtualFile.getCharset().decode(contents).toString()),
+      req.setContents(new SimpleContent(CharsetToolkit.bytesToString(baos.toByteArray(), myVirtualFile.getCharset())),
                       new FileContent(myProject, myVirtualFile));
       req.setContentTitles(remoteTitleBuilder.toString(), myVirtualFile.getPresentableUrl());
       DiffManager.getInstance().getDiffTool().show(req);

plugins/svn4idea/src/org/jetbrains/idea/svn/annotate/SvnAnnotationProvider.java

@@ -23,6 +23,7 @@ import com.intellij.openapi.vcs.VcsException;
 import com.intellij.openapi.vcs.annotate.AnnotationProvider;
 import com.intellij.openapi.vcs.annotate.FileAnnotation;
 import com.intellij.openapi.vcs.history.VcsFileRevision;
+import com.intellij.openapi.vcs.history.VcsHistoryUtil;
 import com.intellij.openapi.vfs.VirtualFile;
 import org.jetbrains.idea.svn.*;
 import org.jetbrains.idea.svn.history.SvnFileRevision;
@@ -70,8 +71,8 @@ public class SvnAnnotationProvider implements AnnotationProvider {
             myVcs.createWCClient().doGetFileContents(ioFile, SVNRevision.UNDEFINED, SVNRevision.BASE, true, buffer);
             contents = LoadTextUtil.getTextByBinaryPresentation(buffer.toByteArray(), file, false).toString();
           } else {
-            revision.loadContent();
-            contents = LoadTextUtil.getTextByBinaryPresentation(revision.getContent(), file, false).toString();
+            final byte[] bytes = VcsHistoryUtil.loadRevisionContent(revision);
+            contents = LoadTextUtil.getTextByBinaryPresentation(bytes, file, false).toString();
           }
 
           final SvnFileAnnotation result = new SvnFileAnnotation(myVcs, file, contents);

plugins/svn4idea/src/org/jetbrains/idea/svn/history/SvnRepositoryContentRevision.java

@@ -71,7 +71,7 @@ public class SvnRepositoryContentRevision implements ContentRevision {
   public String getContent() throws VcsException {
     if (myContent == null) {
       final ByteArrayOutputStream buffer = loadContent();
-      myContent = CharsetToolkit.bytesToString(buffer.toByteArray());
+      myContent = CharsetToolkit.bytesToString(buffer.toByteArray(), myFilePath.getCharset());
     }
     return myContent;
   }
@@ -170,4 +170,4 @@ public class SvnRepositoryContentRevision implements ContentRevision {
   public String getPath() {
     return myPath;
   }
-}
+}

plugins/svn4idea/src/org/jetbrains/idea/svn/status/DiffContentRevision.java

@@ -20,12 +20,15 @@ import com.intellij.openapi.vcs.FilePath;
 import com.intellij.openapi.vcs.VcsException;
 import com.intellij.openapi.vcs.changes.ContentRevision;
 import com.intellij.openapi.vcs.history.VcsRevisionNumber;
+import com.intellij.openapi.vfs.CharsetToolkit;
 import com.intellij.vcsUtil.VcsUtil;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.tmatesoft.svn.core.SVNException;
 import org.tmatesoft.svn.core.io.SVNRepository;
 
+import java.nio.charset.Charset;
+
 public class DiffContentRevision implements ContentRevision {
   private String myPath;
   private SVNRepository myRepository;
@@ -54,7 +57,9 @@ public class DiffContentRevision implements ContentRevision {
       } catch (SVNException e) {
         throw new VcsException(e);
       }
-      myContents = new String(bos.getInternalBuffer(), 0, bos.size());
+      final byte[] bytes = bos.toByteArray();
+      final Charset charset = myFilePath.getCharset();
+      myContents = charset == null ? CharsetToolkit.bytesToString(bytes) : CharsetToolkit.bytesToString(bytes, charset);
     }
     return myContents;
   }