IDEA-101502 diff: allow to ignore separators in directory diff

platform/diff-api/src/com/intellij/ide/diff/DirDiffSettings.java

@@ -69,6 +69,7 @@ public class DirDiffSettings {
 
   public enum CompareMode {
     CONTENT("Content"), // the most honest, the slowest. Compares size, if equal compares contents. Ignores timestamps
+    CONTENT_IGNORE_SEPARATORS("Content Ignoring Line Separators"),
     SIZE("Size"), // Compares size only
     TIMESTAMP("Size and Timestamp"); // Compares size, if equal compares timestamps
 

platform/diff-impl/src/com/intellij/diff/DiffContentFactoryImpl.java

@@ -52,7 +52,6 @@ import org.jetbrains.annotations.Nullable;
 import java.awt.datatransfer.DataFlavor;
 import java.io.File;
 import java.io.IOException;
-import java.nio.charset.CharacterCodingException;
 import java.nio.charset.Charset;
 
 public class DiffContentFactoryImpl extends DiffContentFactoryEx {
@@ -409,11 +408,8 @@ public class DiffContentFactoryImpl extends DiffContentFactoryEx {
     if (isBOM) charset = bomCharset;
 
     boolean malformedContent = false;
-    String text;
-    try {
-      text = CharsetToolkit.tryDecodeString(content, charset);
-    }
-    catch (CharacterCodingException e) {
+    String text = CharsetToolkit.tryDecodeString(content, charset);
+    if (text == null) {
       text = CharsetToolkit.decodeString(content, charset);
       malformedContent = true;
     }

platform/diff-impl/src/com/intellij/openapi/diff/impl/dir/DTree.java

@@ -19,12 +19,16 @@ import com.intellij.ide.diff.DiffElement;
 import com.intellij.ide.diff.DiffErrorElement;
 import com.intellij.ide.diff.DiffType;
 import com.intellij.ide.diff.DirDiffSettings;
+import com.intellij.openapi.util.text.StringUtil;
+import com.intellij.openapi.vfs.CharsetToolkit;
+import com.intellij.util.ThreeState;
 import com.intellij.util.containers.HashMap;
 import com.intellij.util.containers.SortedList;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
 import java.io.IOException;
+import java.nio.charset.Charset;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Comparator;
@@ -158,6 +162,9 @@ public class DTree {
           case CONTENT:
             equals = isEqualContents(src, trg);
             break;
+          case CONTENT_IGNORE_SEPARATORS:
+            equals = isEqualContentsIgnoreSeparators(src, trg);
+            break;
           case SIZE:
             equals = isEqualSizes(src, trg);
             break;
@@ -257,6 +264,53 @@ public class DTree {
     }
   }
 
+  private static boolean isEqualContentsIgnoreSeparators(DiffElement<?> file1, DiffElement<?> file2) {
+    if (file1.isContainer() || file2.isContainer()) return false;
+
+    if (file1.getFileType().isBinary() && file2.getFileType().isBinary()) {
+      return isEqualContents(file1, file2);
+    }
+
+    try {
+      byte[] content1 = file1.getContent();
+      byte[] content2 = file2.getContent();
+
+      Charset charset1 = file1.getCharset();
+      Charset charset2 = file2.getCharset();
+
+      if (Arrays.equals(file1.getContent(), file2.getContent())) return true;
+      if (content1 == null || content2 == null) return false;
+
+      ThreeState isEqual = isEqualContentsIgnoreSeparators(content1, content2, charset1);
+      if (isEqual != ThreeState.UNSURE) return isEqual.toBoolean();
+
+      if (!charset1.equals(charset2)) {
+        isEqual = isEqualContentsIgnoreSeparators(content1, content2, charset2);
+        if (isEqual != ThreeState.UNSURE) return isEqual.toBoolean();
+      }
+
+      return false;
+    }
+    catch (IOException e) {
+      return false;
+    }
+  }
+
+  @NotNull
+  private static ThreeState isEqualContentsIgnoreSeparators(byte[] content1, byte[] content2, @NotNull Charset charset) {
+    String text1 = CharsetToolkit.tryDecodeString(content1, charset);
+    if (text1 == null) return ThreeState.UNSURE;
+
+    String text2 = CharsetToolkit.tryDecodeString(content2, charset);
+    if (text2 == null) return ThreeState.UNSURE;
+
+    String convertedText1 = StringUtil.convertLineSeparators(text1);
+    String convertedText2 = StringUtil.convertLineSeparators(text2);
+
+    boolean isEquals = StringUtil.equals(convertedText1, convertedText2);
+    return ThreeState.fromBoolean(isEquals);
+  }
+
   public DiffType getType() {
     return myType;
   }

platform/platform-impl/src/com/intellij/ide/diff/VirtualFileDiffElement.java

@@ -25,6 +25,7 @@ import com.intellij.openapi.fileChooser.FileChooser;
 import com.intellij.openapi.fileChooser.FileChooserDescriptor;
 import com.intellij.openapi.fileEditor.FileDocumentManager;
 import com.intellij.openapi.fileEditor.OpenFileDescriptor;
+import com.intellij.openapi.fileTypes.FileType;
 import com.intellij.openapi.fileTypes.FileTypeManager;
 import com.intellij.openapi.progress.ProgressManager;
 import com.intellij.openapi.project.Project;
@@ -40,6 +41,7 @@ import org.jetbrains.annotations.Nullable;
 import javax.swing.*;
 import java.io.File;
 import java.io.IOException;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.Callable;
@@ -151,6 +153,17 @@ public class VirtualFileDiffElement extends DiffElement<VirtualFile> {
     return myFile.getFileSystem() instanceof LocalFileSystem;
   }
 
+  @NotNull
+  @Override
+  public Charset getCharset() {
+    return myFile.getCharset();
+  }
+
+  @Override
+  public FileType getFileType() {
+    return myFile.getFileType();
+  }
+
   @Override
   public VirtualFileDiffElement copyTo(DiffElement<VirtualFile> container, String relativePath) {
     try {

platform/util/src/com/intellij/openapi/vfs/CharsetToolkit.java

@@ -312,14 +312,19 @@ public class CharsetToolkit {
     return charBuffer.toString();
   }
 
-  @NotNull
-  public static String tryDecodeString(@NotNull byte[] bytes, @NotNull final Charset charset) throws CharacterCodingException {
-    int bomLength = getBOMLength(bytes, charset);
-    ByteBuffer buffer = ByteBuffer.wrap(bytes, bomLength, bytes.length - bomLength);
-    CharsetDecoder decoder = charset.newDecoder()
-      .onMalformedInput(CodingErrorAction.REPORT)
-      .onUnmappableCharacter(CodingErrorAction.REPORT);
-    return decoder.decode(buffer).toString();
+  @Nullable
+  public static String tryDecodeString(@NotNull byte[] bytes, @NotNull final Charset charset) {
+    try {
+      int bomLength = getBOMLength(bytes, charset);
+      ByteBuffer buffer = ByteBuffer.wrap(bytes, bomLength, bytes.length - bomLength);
+      CharsetDecoder decoder = charset.newDecoder()
+        .onMalformedInput(CodingErrorAction.REPORT)
+        .onUnmappableCharacter(CodingErrorAction.REPORT);
+      return decoder.decode(buffer).toString();
+    }
+    catch (CharacterCodingException e) {
+      return null;
+    }
   }
 
   public enum GuessedEncoding {