moved CharsetToolkit to util. Illegal dependencies on EncodingRegistry removed

java/idea-ui/src/com/intellij/ide/util/projectWizard/importSources/util/CommonSourceRootDetectionUtil.java

@@ -18,10 +18,10 @@ package com.intellij.ide.util.projectWizard.importSources.util;
 import com.intellij.openapi.util.Pair;
 import com.intellij.openapi.util.SystemInfo;
 import com.intellij.openapi.util.io.FileUtilRt;
+import com.intellij.openapi.vfs.CharsetToolkit;
 import com.intellij.openapi.vfs.VfsUtilCore;
 import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.util.NullableFunction;
-import com.intellij.util.text.CharsetUtil;
 import com.intellij.util.text.StringFactory;
 import org.jetbrains.annotations.Nullable;
 
@@ -109,7 +109,7 @@ public abstract class CommonSourceRootDetectionUtil<F> {
 
   private static char[] loadFileTextSkippingBom(File file) throws IOException {
     //noinspection IOResourceOpenedButNotSafelyClosed
-    InputStream stream = CharsetUtil.inputStreamSkippingBOM(new BufferedInputStream(new FileInputStream(file)));
+    InputStream stream = CharsetToolkit.inputStreamSkippingBOM(new BufferedInputStream(new FileInputStream(file)));
     Reader reader = new InputStreamReader(stream);
     try {
       return FileUtilRt.loadText(reader, (int)file.length());

jps/jps-builders/testSrc/org/jetbrains/jps/incremental/artifacts/ArtifactBuilderOverwriteTest.java

@@ -16,8 +16,8 @@
 package org.jetbrains.jps.incremental.artifacts;
 
 import com.intellij.openapi.util.io.FileUtil;
+import com.intellij.openapi.vfs.CharsetToolkit;
 import com.intellij.util.PathUtil;
-import com.intellij.util.text.CharsetUtil;
 import org.jetbrains.jps.model.artifact.JpsArtifact;
 
 import java.io.File;
@@ -175,7 +175,7 @@ public class ArtifactBuilderOverwriteTest extends ArtifactBuilderTestCase {
       ZipOutputStream output = new ZipOutputStream(new FileOutputStream(file));
       try {
         output.putNextEntry(new ZipEntry(fileNameInArchive));
-        output.write(text.getBytes(CharsetUtil.UTF8));
+        output.write(text.getBytes(CharsetToolkit.UTF8));
         output.closeEntry();
       }
       finally {

platform/lang-impl/src/com/intellij/ui/debugger/extensions/PlaybackDebugger.java

@@ -31,6 +31,7 @@ import com.intellij.openapi.ui.playback.PlaybackContext;
 import com.intellij.openapi.ui.playback.PlaybackRunner;
 import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.openapi.vfs.*;
+import com.intellij.openapi.vfs.encoding.EncodingRegistry;
 import com.intellij.openapi.wm.IdeFrame;
 import com.intellij.openapi.wm.WindowManager;
 import com.intellij.openapi.wm.ex.WindowManagerEx;
@@ -270,7 +271,7 @@ public class PlaybackDebugger implements UiDebuggerExtension, PlaybackRunner.Sta
 
   private void loadFrom(@NotNull VirtualFile file) {
     try {
-      final String text = CharsetToolkit.bytesToString(file.contentsToByteArray());
+      final String text = CharsetToolkit.bytesToString(file.contentsToByteArray(), EncodingRegistry.getInstance().getDefaultCharset());
       fillDocument(text);
       myChanged = false;
     }

platform/lvcs-impl/src/com/intellij/history/integration/IdeaGateway.java

@@ -37,6 +37,7 @@ import com.intellij.openapi.util.Key;
 import com.intellij.openapi.util.Pair;
 import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.openapi.vfs.*;
+import com.intellij.openapi.vfs.encoding.EncodingRegistry;
 import com.intellij.openapi.vfs.newvfs.ManagingFS;
 import com.intellij.openapi.vfs.newvfs.NewVirtualFile;
 import com.intellij.util.NullableFunction;
@@ -338,7 +339,7 @@ public class IdeaGateway {
     try {
       VirtualFile file = findVirtualFile(path);
       if (file == null) {
-        return CharsetToolkit.bytesToString(bytes);
+        return CharsetToolkit.bytesToString(bytes, EncodingRegistry.getInstance().getDefaultCharset());
       }
       return new String(bytes, file.getCharset().name());
     }

platform/platform-api/src/com/intellij/openapi/diff/BinaryContent.java

@@ -23,6 +23,7 @@ import com.intellij.openapi.fileTypes.UIBasedFileType;
 import com.intellij.openapi.vfs.CharsetToolkit;
 import com.intellij.openapi.vfs.LocalFileSystem;
 import com.intellij.openapi.vfs.VirtualFile;
+import com.intellij.openapi.vfs.encoding.EncodingRegistry;
 import com.intellij.testFramework.LightVirtualFile;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
@@ -74,7 +75,7 @@ public class BinaryContent extends DiffContent {
       String text = null;
       try {
         if (myCharset == null) {
-          text = CharsetToolkit.bytesToString(myBytes);
+          text = CharsetToolkit.bytesToString(myBytes, EncodingRegistry.getInstance().getDefaultCharset());
         }
         else {
           text = CharsetToolkit.bytesToString(myBytes, myCharset);

platform/platform-tests/testSrc/com/intellij/openapi/components/impl/ProjectStoreBaseTestCase.java

@@ -20,8 +20,8 @@ import com.intellij.openapi.components.State;
 import com.intellij.openapi.components.Storage;
 import com.intellij.openapi.util.JDOMBuilder;
 import com.intellij.openapi.util.JDOMUtil;
+import com.intellij.openapi.vfs.CharsetToolkit;
 import com.intellij.testFramework.PlatformLangTestCase;
-import com.intellij.util.text.CharsetUtil;
 
 import java.io.UnsupportedEncodingException;
 
@@ -34,7 +34,7 @@ public abstract class ProjectStoreBaseTestCase extends PlatformLangTestCase {
                        JDOMBuilder.tag("option", JDOMBuilder.attr("name", "VALUE"), JDOMBuilder.attr("value", "true")))
       )),
       "\n");
-    return iprContent.getBytes(CharsetUtil.UTF8);
+    return iprContent.getBytes(CharsetToolkit.UTF8);
   }
 
   @State(

platform/util/src/com/intellij/openapi/vfs/CharsetToolkit.java

@@ -15,18 +15,13 @@
  */
 package com.intellij.openapi.vfs;
 
-import com.intellij.openapi.vfs.encoding.EncodingRegistry;
 import com.intellij.util.ArrayUtil;
-import com.intellij.util.text.CharsetUtil;
 import gnu.trove.THashMap;
 import org.jetbrains.annotations.NonNls;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
+import java.io.*;
 import java.nio.ByteBuffer;
 import java.nio.CharBuffer;
 import java.nio.charset.Charset;
@@ -78,19 +73,24 @@ import java.util.Map;
  * @author Guillaume LAFORGE
  */
 public class CharsetToolkit {
-  @NonNls public static final String UTF8 = CharsetUtil.UTF8;
+  @NonNls public static final String UTF8 = "UTF-8";
   public static final Charset UTF8_CHARSET = Charset.forName(UTF8);
   public static final Charset UTF_16LE_CHARSET = Charset.forName("UTF-16LE");
   public static final Charset UTF_16BE_CHARSET = Charset.forName("UTF-16BE");
   public static final Charset UTF_32BE_CHARSET = Charset.forName("UTF-32BE");
   public static final Charset UTF_32LE_CHARSET = Charset.forName("UTF-32LE");
   public static final Charset UTF_16_CHARSET = Charset.forName("UTF-16");
+  private static final byte FF = (byte)0xff;
+  private static final byte FE = (byte)0xfe;
+  private static final byte EF = (byte)0xef;
+  private static final byte BB = (byte)0xbb;
+  private static final byte BF = (byte)0xbf;
 
   private final byte[] buffer;
   private final Charset defaultCharset;
   private boolean enforce8Bit = false;
 
-  public static final byte[] UTF8_BOM = CharsetUtil.UTF8_BOM;
+  public static final byte[] UTF8_BOM = {0xffffffef, 0xffffffbb, 0xffffffbf};
   public static final byte[] UTF16LE_BOM = {-1, -2, };
   public static final byte[] UTF16BE_BOM = {-2, -1, };
   public static final byte[] UTF32BE_BOM = {0, 0, -2, -1, };
@@ -126,6 +126,96 @@ public class CharsetToolkit {
     this.defaultCharset = defaultCharset == null ? getDefaultSystemCharset() : defaultCharset;
   }
 
+  @NotNull
+  public static InputStream inputStreamSkippingBOM(@NotNull InputStream stream) throws IOException {
+    assert stream.markSupported() :stream;
+    stream.mark(4);
+    boolean mustReset = true;
+    try {
+      int ret = stream.read();
+      if (ret == -1) {
+        return stream; // no bom
+      }
+      byte b0 = (byte)ret;
+      if (b0 != EF && b0 != FF && b0 != FE && b0 != 0) return stream; // no bom
+
+      ret = stream.read();
+      if (ret == -1) {
+        return stream; // no bom
+      }
+      byte b1 = (byte)ret;
+      if (b0 == FF && b1 == FE) {
+        stream.mark(2);
+        ret = stream.read();
+        if (ret == -1) {
+          return stream;  // utf-16 LE
+        }
+        byte b2 = (byte)ret;
+        if (b2 != 0) {
+          return stream; // utf-16 LE
+        }
+        ret = stream.read();
+        if (ret == -1) {
+          return stream;
+        }
+        byte b3 = (byte)ret;
+        if (b3 != 0) {
+          return stream; // utf-16 LE
+        }
+
+        // utf-32 LE
+        mustReset = false;
+        return stream;
+      }
+      if (b0 == FE && b1 == FF) {
+        mustReset = false;
+        return stream; // utf-16 BE
+      }
+      if (b0 == EF && b1 == BB) {
+        ret = stream.read();
+        if (ret == -1) {
+          return stream; // no bom
+        }
+        byte b2 = (byte)ret;
+        if (b2 == BF) {
+          mustReset = false;
+          return stream; // utf-8 bom
+        }
+
+        // no bom
+        return stream;
+      }
+
+      if (b0 == 0 && b1 == 0) {
+        ret = stream.read();
+        if (ret == -1) {
+          return stream;  // no bom
+        }
+        byte b2 = (byte)ret;
+        if (b2 != FE) {
+          return stream; // no bom
+        }
+        ret = stream.read();
+        if (ret == -1) {
+          return stream;  // no bom
+        }
+        byte b3 = (byte)ret;
+        if (b3 != FF) {
+          return stream; // no bom
+        }
+
+        mustReset = false;
+        return stream; // UTF-32 BE
+      }
+
+      // no bom
+      return stream;
+    }
+    finally {
+      if (mustReset) stream.reset();
+    }
+  }
+
   /**
    * If US-ASCII is recognized, enforce to return the default encoding, rather than US-ASCII.
    * It might be a file without any special character in the range 128-255, but that may be or become
@@ -197,11 +287,6 @@ public class CharsetToolkit {
     return null;
   }
 
-  @NotNull
-  public static String bytesToString(@NotNull byte[] bytes) {
-    return bytesToString(bytes, EncodingRegistry.getInstance().getDefaultCharset());
-  }
-
   @NotNull
   public static String bytesToString(@NotNull byte[] bytes, @NotNull final Charset defaultCharset) {
     Charset charset = new CharsetToolkit(bytes, defaultCharset).guessEncoding(bytes.length);
@@ -335,10 +420,6 @@ public class CharsetToolkit {
     return guessEncoding(guess_length, defaultCharset);
   }
 
-  public static Charset guessEncoding(@NotNull File f, int bufferLength) throws IOException {
-    return guessEncoding(f, bufferLength, EncodingRegistry.getInstance().getDefaultCharset());
-  }
-
   public static Charset guessEncoding(@NotNull File f, int bufferLength, Charset defaultCharset) throws IOException {
     byte[] buffer = new byte[bufferLength];
     int read;
@@ -437,7 +518,7 @@ public class CharsetToolkit {
    * @return true if the buffer has a BOM for UTF8.
    */
   public static boolean hasUTF8Bom(@NotNull byte[] bom) {
-    return CharsetUtil.hasUTF8Bom(bom);
+    return ArrayUtil.startsWith(bom, UTF8_BOM);
   }
 
   /**
@@ -482,7 +563,12 @@ public class CharsetToolkit {
 
   @NotNull
   public static byte[] getUtf8Bytes(@NotNull String s) {
-    return CharsetUtil.getUtf8Bytes(s);
+    try {
+      return s.getBytes(CharsetToolkit.UTF8);
+    }
+    catch (UnsupportedEncodingException e) {
+      throw new RuntimeException("UTF-8 must be supported", e);
+    }
   }
 
   public static int getBOMLength(@NotNull byte[] content, Charset charset) {
@@ -545,12 +631,4 @@ public class CharsetToolkit {
 
     return charset;
   }
-
-  /**
-   * @deprecated use {@link CharsetUtil#inputStreamSkippingBOM(java.io.InputStream)} instead
-   */
-  @NotNull
-  public static InputStream inputStreamSkippingBOM(@NotNull InputStream stream) throws IOException {
-    return CharsetUtil.inputStreamSkippingBOM(stream);
-  }
 }

platform/util/src/com/intellij/util/text/CharsetUtil.java

@@ -1,140 +0,0 @@
-/*
- * Copyright 2000-2012 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.intellij.util.text;
-
-import com.intellij.util.ArrayUtil;
-import org.jetbrains.annotations.NonNls;
-import org.jetbrains.annotations.NotNull;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.UnsupportedEncodingException;
-
-/**
- * @author nik
- */
-public class CharsetUtil {
-  public static final byte[] UTF8_BOM = {0xffffffef, 0xffffffbb, 0xffffffbf};
-  @NonNls public static final String UTF8 = "UTF-8";
-  private static final byte FF = (byte)0xff;
-  private static final byte FE = (byte)0xfe;
-  private static final byte EF = (byte)0xef;
-  private static final byte BB = (byte)0xbb;
-  private static final byte BF = (byte)0xbf;
-
-  public static boolean hasUTF8Bom(byte[] bom) {
-    return ArrayUtil.startsWith(bom, UTF8_BOM);
-  }
-
-  public static byte[] getUtf8Bytes(String s) {
-    try {
-      return s.getBytes(UTF8);
-    }
-    catch (UnsupportedEncodingException e) {
-      throw new RuntimeException("UTF-8 must be supported", e);
-    }
-  }
-
-  @NotNull
-  public static InputStream inputStreamSkippingBOM(@NotNull InputStream stream) throws IOException {
-    assert stream.markSupported() :stream;
-    stream.mark(4);
-    boolean mustReset = true;
-    try {
-      int ret = stream.read();
-      if (ret == -1) {
-        return stream; // no bom
-      }
-      byte b0 = (byte)ret;
-      if (b0 != EF && b0 != FF && b0 != FE && b0 != 0) return stream; // no bom
-
-      ret = stream.read();
-      if (ret == -1) {
-        return stream; // no bom
-      }
-      byte b1 = (byte)ret;
-      if (b0 == FF && b1 == FE) {
-        stream.mark(2);
-        ret = stream.read();
-        if (ret == -1) {
-          return stream;  // utf-16 LE
-        }
-        byte b2 = (byte)ret;
-        if (b2 != 0) {
-          return stream; // utf-16 LE
-        }
-        ret = stream.read();
-        if (ret == -1) {
-          return stream;
-        }
-        byte b3 = (byte)ret;
-        if (b3 != 0) {
-          return stream; // utf-16 LE
-        }
-
-        // utf-32 LE
-        mustReset = false;
-        return stream;
-      }
-      if (b0 == FE && b1 == FF) {
-        mustReset = false;
-        return stream; // utf-16 BE
-      }
-      if (b0 == EF && b1 == BB) {
-        ret = stream.read();
-        if (ret == -1) {
-          return stream; // no bom
-        }
-        byte b2 = (byte)ret;
-        if (b2 == BF) {
-          mustReset = false;
-          return stream; // utf-8 bom
-        }
-
-        // no bom
-        return stream;
-      }
-
-      if (b0 == 0 && b1 == 0) {
-        ret = stream.read();
-        if (ret == -1) {
-          return stream;  // no bom
-        }
-        byte b2 = (byte)ret;
-        if (b2 != FE) {
-          return stream; // no bom
-        }
-        ret = stream.read();
-        if (ret == -1) {
-          return stream;  // no bom
-        }
-        byte b3 = (byte)ret;
-        if (b3 != FF) {
-          return stream; // no bom
-        }
-
-        mustReset = false;
-        return stream; // UTF-32 BE
-      }
-
-      // no bom
-      return stream;
-    }
-    finally {
-      if (mustReset) stream.reset();
-    }
-  }
-}

platform/util/src/com/intellij/util/text/XmlCharsetDetector.java

@@ -16,6 +16,7 @@
 package com.intellij.util.text;
 
 import com.intellij.openapi.util.text.StringUtil;
+import com.intellij.openapi.vfs.CharsetToolkit;
 import com.intellij.util.ArrayUtil;
 import org.jetbrains.annotations.NonNls;
 import org.jetbrains.annotations.NotNull;
@@ -26,17 +27,17 @@ import org.jetbrains.annotations.Nullable;
  */
 public class XmlCharsetDetector {
   @NonNls private static final String XML_PROLOG_START = "<?xml";
-  @NonNls private static final byte[] XML_PROLOG_START_BYTES = CharsetUtil.getUtf8Bytes(XML_PROLOG_START);
+  @NonNls private static final byte[] XML_PROLOG_START_BYTES = CharsetToolkit.getUtf8Bytes(XML_PROLOG_START);
   @NonNls private static final String ENCODING = "encoding";
-  @NonNls private static final byte[] ENCODING_BYTES = CharsetUtil.getUtf8Bytes(ENCODING);
+  @NonNls private static final byte[] ENCODING_BYTES = CharsetToolkit.getUtf8Bytes(ENCODING);
   @NonNls private static final String XML_PROLOG_END = "?>";
-  @NonNls private static final byte[] XML_PROLOG_END_BYTES = CharsetUtil.getUtf8Bytes(XML_PROLOG_END);
+  @NonNls private static final byte[] XML_PROLOG_END_BYTES = CharsetToolkit.getUtf8Bytes(XML_PROLOG_END);
 
   @Nullable
   public static String extractXmlEncodingFromProlog(final byte[] bytes) {
     int index = 0;
-    if (CharsetUtil.hasUTF8Bom(bytes)) {
-      index = CharsetUtil.UTF8_BOM.length;
+    if (CharsetToolkit.hasUTF8Bom(bytes)) {
+      index = CharsetToolkit.UTF8_BOM.length;
     }
 
     index = skipWhiteSpace(index, bytes);

platform/vcs-api/src/com/intellij/openapi/vcs/impl/ContentRevisionCache.java

@@ -27,6 +27,7 @@ import com.intellij.openapi.vcs.changes.FilePathsHelper;
 import com.intellij.openapi.vcs.changes.VcsDirtyScope;
 import com.intellij.openapi.vcs.history.VcsRevisionNumber;
 import com.intellij.openapi.vfs.CharsetToolkit;
+import com.intellij.openapi.vfs.encoding.EncodingRegistry;
 import com.intellij.reference.SoftReference;
 import com.intellij.util.Consumer;
 import com.intellij.util.containers.HashSet;
@@ -167,7 +168,7 @@ public class ContentRevisionCache {
       return charBuffer.toString();
     }
 
-    return CharsetToolkit.bytesToString(bytes);
+    return CharsetToolkit.bytesToString(bytes, EncodingRegistry.getInstance().getDefaultCharset());
   }
 
   @Nullable

plugins/cvs/cvs-plugin/src/com/intellij/cvsSupport2/changeBrowser/CvsContentRevision.java

@@ -38,6 +38,7 @@ import com.intellij.openapi.vcs.actions.VcsContextFactory;
 import com.intellij.openapi.vcs.changes.ContentRevision;
 import com.intellij.openapi.vcs.history.VcsRevisionNumber;
 import com.intellij.openapi.vfs.CharsetToolkit;
+import com.intellij.openapi.vfs.encoding.EncodingRegistry;
 import org.jetbrains.annotations.NonNls;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
@@ -72,7 +73,7 @@ public class CvsContentRevision implements ContentRevision {
       byte[] content = loadContent();
       if (content != null) {
         final Charset charset = myLocalFile.getCharset();
-        myContent = charset == null ? CharsetToolkit.bytesToString(content) : CharsetToolkit.bytesToString(content, charset);
+        myContent = charset == null ? CharsetToolkit.bytesToString(content, EncodingRegistry.getInstance().getDefaultCharset()) : CharsetToolkit.bytesToString(content, charset);
       }
     }
     return myContent;

plugins/maven/src/main/java/org/jetbrains/idea/maven/utils/MavenJDOMUtil.java

@@ -22,6 +22,7 @@ import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.openapi.vfs.CharsetToolkit;
 import com.intellij.openapi.vfs.VfsUtil;
 import com.intellij.openapi.vfs.VirtualFile;
+import com.intellij.openapi.vfs.encoding.EncodingRegistry;
 import com.intellij.psi.impl.source.parsing.xml.XmlBuilder;
 import com.intellij.psi.impl.source.parsing.xml.XmlBuilderDriver;
 import org.jdom.Element;
@@ -57,7 +58,7 @@ public class MavenJDOMUtil {
 
   @Nullable
   public static Element read(byte[] bytes, @Nullable ErrorHandler handler) {
-    return doRead(CharsetToolkit.bytesToString(bytes), handler);
+    return doRead(CharsetToolkit.bytesToString(bytes, EncodingRegistry.getInstance().getDefaultCharset()), handler);
   }
 
   @Nullable

plugins/svn4idea/src/org/jetbrains/idea/svn/status/DiffContentRevision.java

@@ -21,6 +21,7 @@ import com.intellij.openapi.vcs.VcsException;
 import com.intellij.openapi.vcs.changes.ContentRevision;
 import com.intellij.openapi.vcs.history.VcsRevisionNumber;
 import com.intellij.openapi.vfs.CharsetToolkit;
+import com.intellij.openapi.vfs.encoding.EncodingRegistry;
 import com.intellij.vcsUtil.VcsUtil;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
@@ -59,7 +60,7 @@ public class DiffContentRevision implements ContentRevision {
       }
       final byte[] bytes = bos.toByteArray();
       final Charset charset = myFilePath.getCharset();
-      myContents = charset == null ? CharsetToolkit.bytesToString(bytes) : CharsetToolkit.bytesToString(bytes, charset);
+      myContents = charset == null ? CharsetToolkit.bytesToString(bytes, EncodingRegistry.getInstance().getDefaultCharset()) : CharsetToolkit.bytesToString(bytes, charset);
     }
     return myContents;
   }