moved python smart copy-paste logic to python module

so fixed PY-8982, PY-7709, PY-7470, PY-6994, PY-6992, PY-6987, PY-6888

python/src/META-INF/python-plugin-common.xml

@@ -404,8 +404,6 @@
     <moduleService serviceInterface="com.jetbrains.python.psi.resolve.PythonPathCache"
                    serviceImplementation="com.jetbrains.python.psi.resolve.PythonModulePathCache"/>
 
-    <preserveIndentOnPaste fileType="Python"/>
-
     <problemFileHighlightFilter implementation="com.jetbrains.python.codeInsight.PyProblemFileHighlightFilter"/>
 
     <editorSmartKeysConfigurable instance="com.jetbrains.python.codeInsight.PySmartKeysOptions"/>

python/src/com/jetbrains/python/codeInsight/PySmartKeysOptions.java

@@ -10,6 +10,7 @@ import com.intellij.openapi.options.UnnamedConfigurable;
 public class PySmartKeysOptions extends BeanConfigurable<CodeInsightSettings> implements UnnamedConfigurable {
   public PySmartKeysOptions() {
     super(CodeInsightSettings.getInstance());
+    CodeInsightSettings.getInstance().REFORMAT_ON_PASTE = CodeInsightSettings.NO_REFORMAT;   //TODO: remove combobox from settings
     checkBox("INDENT_TO_CARET_ON_PASTE", "Smart indent pasted lines");
   }
 }

python/src/com/jetbrains/python/editor/PythonCopyPasteProcessor.java

@@ -9,13 +9,15 @@ import com.intellij.openapi.util.text.CharFilter;
 import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.psi.PsiElement;
 import com.intellij.psi.PsiFile;
-import com.intellij.psi.PsiWhiteSpace;
 import com.intellij.psi.codeStyle.CodeStyleSettingsManager;
 import com.intellij.psi.util.PsiTreeUtil;
-import com.intellij.psi.util.PsiUtilCore;
 import com.jetbrains.python.PythonFileType;
-import com.jetbrains.python.psi.PyFile;
-import com.jetbrains.python.psi.PyStringLiteralExpression;
+import com.jetbrains.python.psi.PyClass;
+import com.jetbrains.python.psi.PyFunction;
+import com.jetbrains.python.psi.PyStatementList;
+import com.jetbrains.python.psi.PyUtil;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
 
 import java.util.List;
 
@@ -24,6 +26,7 @@ import java.util.List;
  */
 public class PythonCopyPasteProcessor implements CopyPastePreProcessor {
 
+  @Nullable
   @Override
   public String preprocessOnCopy(PsiFile file, int[] startOffsets, int[] endOffsets, String text) {
     return null;
@@ -45,95 +48,103 @@ public class PythonCopyPasteProcessor implements CopyPastePreProcessor {
         return useTabs? ch != '\t' : ch != ' ';
       }
     };
+
     final CaretModel caretModel = editor.getCaretModel();
+    final SelectionModel selectionModel = editor.getSelectionModel();
     final Document document = editor.getDocument();
-    String newText = text;
+    final int caretOffset = selectionModel.getSelectionStart() != selectionModel.getSelectionEnd() ?
+                            selectionModel.getSelectionStart() : caretModel.getOffset();
+    final int lineNumber = document.getLineNumber(caretOffset);
+    final int lineStartOffset = getLineStartSafeOffset(document, lineNumber);
 
-    if (file instanceof PyFile) {
-      final int caretOffset = caretModel.getOffset();
-      final int lineNumber = document.getLineNumber(caretOffset);
-      final int lineStartOffset = getLineStartSafeOffset(document, lineNumber);
+    final String indentText = getIndentText(file, document, caretOffset, lineNumber);
 
-      final List<String> strings = StringUtil.split(text, "\n");
-      if (StringUtil.countChars(text, '\n') > 0 || StringUtil.startsWithWhitespace(text)) { //2, 3, 4 case from doc
-        final PsiElement element = PsiUtilCore.getElementAtOffset(file, caretOffset - 1);
-        if (PsiTreeUtil.getParentOfType(element, PyStringLiteralExpression.class) != null) return text;
+    int toRemove = calculateIndentToRemove(text, NOT_INDENT_FILTER);
+    if (StringUtil.isEmptyOrSpaces(indentText) && isApplicable(file, text, caretOffset)) {
+      caretModel.moveToOffset(lineStartOffset);
+      editor.getSelectionModel().setSelection(lineStartOffset, selectionModel.getSelectionEnd());
+    }
 
-        final SelectionModel selectionModel = editor.getSelectionModel();
-        if (selectionModel.getSelectionStart() != selectionModel.getSelectionEnd()) {
-          final int line = document.getLineNumber(selectionModel.getSelectionStart());
-          final int lineOffset = getLineStartSafeOffset(document, line);
-          final PsiElement ws = file.findElementAt(lineOffset);
-          int offset = ws instanceof PsiWhiteSpace? ws.getTextRange().getEndOffset() : selectionModel.getSelectionStart();
-          if (text.equals(selectionModel.getSelectedText())) return text;
-          caretModel.moveToOffset(offset);
-          selectionModel.setSelection(offset, selectionModel.getSelectionEnd());
-        }
-        else {
-          if (isNotApplicable(document, caretOffset, lineStartOffset)) {
-            return text;
-          }
-          caretModel.moveToOffset(lineStartOffset);
-        }
-        String spaceString;
-        int indent = 0;
-
-        //calculate indent to normalize text
-        if (strings.size() > 0) {
-          spaceString = strings.get(0);   // insert single line
-          indent = StringUtil.findFirst(spaceString, NOT_INDENT_FILTER);
-          if (indent < 0)
-            indent = StringUtil.isEmptyOrSpaces(spaceString) ? spaceString.length() : 0;
-
-          if (!StringUtil.startsWithWhitespace(spaceString) && strings.size() > 1) {    // insert multi-line
-            spaceString = strings.get(1);
-            indent = StringUtil.findFirst(spaceString, NOT_INDENT_FILTER);
-            if (indent < 0)
-              indent = StringUtil.isEmptyOrSpaces(spaceString) ? spaceString.length() : 0;
-
-            if (indent == CodeStyleSettingsManager.getSettings(project).getIndentSize(PythonFileType.INSTANCE)) {
-              indent = 0;
-            }
-            else {
-              final String trimmed = StringUtil.trimLeading(strings.get(0));    //decrease indent if needed
-              if (trimmed.startsWith("def ") || trimmed.startsWith("if ") || trimmed.startsWith("try:") ||
-                  trimmed.startsWith("class ") || trimmed.startsWith("for ") || trimmed.startsWith("elif ") ||
-                  trimmed.startsWith("else:") || trimmed.startsWith("except") || trimmed.startsWith("while ")) {
-                indent = StringUtil.findFirst(spaceString, NOT_INDENT_FILTER) / 2;
-                if (indent < 0) indent = 0;
-              }
-            }
-          }
-        }
-
-        if (!StringUtil.isEmptyOrSpaces(text))    // do not process empty lines
-          text = StringUtil.trimTrailing(text);
-
-        if (!StringUtil.startsWithWhitespace(text)) {   // add missed whitespaces
-          if (indent > 0) {
-            final String indentSymbol = useTabs? "\t" :" ";
-            newText = StringUtil.repeat(indentSymbol, indent) + text;
-          }
-          else
-            newText = new String(text);
-        }
-        else {
-          newText = new String(text);   // to indent correctly (see PasteHandler)
-        }
-
-        if ((element instanceof PsiWhiteSpace || element.getTextOffset() == 0) &&
-            (StringUtil.countChars(element.getText(), '\n') <= 2 && !StringUtil.isEmptyOrSpaces(text))) {
-          newText += "\n";
-        }
-        else
-          newText = new String(newText);      //user already prepared place to paste to and we just want to indent right
+    final List<String> strings = StringUtil.split(text, "\n", false);
+    String newText = "";
+    if (StringUtil.isEmptyOrSpaces(indentText)) {
+      for (String s : strings) {
+        newText += indentText + StringUtil.trimStart(s, StringUtil.repeat(useTabs? "\t" : " ", toRemove));
       }
     }
+    else {
+      newText = text;
+    }
+
+    String toString = document.getText(TextRange.create(lineStartOffset, document.getLineEndOffset(lineNumber)));
+    if (addLinebreak(text, toString, useTabs) && selectionModel.getSelectionStart() == selectionModel.getSelectionEnd())
+      newText += "\n";
     return newText;
   }
 
-  private boolean isNotApplicable(Document document, int caretOffset, int lineStartOffset) {
-    return !StringUtil.isEmptyOrSpaces(document.getText(TextRange.create(lineStartOffset, caretOffset)));
+  private static String getIndentText(@NotNull final PsiFile file,
+                                      @NotNull final Document document,
+                                      int caretOffset,
+                                      int lineNumber) {
+
+    final PsiElement nonWS = PyUtil.findNextNonWhitespaceAtOffset(file, caretOffset);
+    int lineStartOffset = getLineStartSafeOffset(document, lineNumber);
+    String indentText = document.getText(TextRange.create(lineStartOffset, caretOffset));
+
+    if (nonWS != null && document.getLineNumber(nonWS.getTextOffset()) == lineNumber) {
+      indentText = document.getText(TextRange.create(lineStartOffset, nonWS.getTextOffset()));
+    }
+    else if (caretOffset == lineStartOffset) {
+      final PsiElement ws = file.findElementAt(lineStartOffset);
+      if (ws != null) {
+        final String wsText = ws.getText();
+        final List<String> strings = StringUtil.split(wsText, "\n");
+        if (strings.size() >= 1) {
+          indentText = strings.get(0);
+        }
+      }
+    }
+    return indentText;
+  }
+
+  private static int calculateIndentToRemove(@NotNull String text, @NotNull final CharFilter filter) {
+    final List<String> strings = StringUtil.split(text, "\n", false);
+    int minIndent = StringUtil.findFirst(text, filter);
+    for (String  s : strings) {
+      final int indent = StringUtil.findFirst(s, filter);
+      if (indent < minIndent)
+        minIndent = indent;
+    }
+    return minIndent;
+  }
+
+  private static boolean isApplicable(@NotNull final PsiFile file, @NotNull String text, int caretOffset) {
+    final boolean useTabs =
+      CodeStyleSettingsManager.getSettings(file.getProject()).useTabCharacter(PythonFileType.INSTANCE);
+    final PsiElement nonWS = PyUtil.findNextNonWhitespaceAtOffset(file, caretOffset);
+    if (nonWS == null || text.endsWith("\n"))
+      return true;
+    if (inStatementList(file, caretOffset) && (text.startsWith(useTabs ? "\t" : " ") || StringUtil.split(text, "\n").size() > 1))
+      return true;
+    return false;
+  }
+
+  private static boolean inStatementList(@NotNull final PsiFile file, int caretOffset) {
+    final PsiElement element = file.findElementAt(caretOffset);
+    final PsiElement element1 = file.findElementAt(caretOffset);
+    return PsiTreeUtil.getParentOfType(element, PyStatementList.class) != null ||
+           PsiTreeUtil.getParentOfType(element1, PyStatementList.class) != null ||
+           PsiTreeUtil.getParentOfType(element, PyFunction.class) != null ||
+           PsiTreeUtil.getParentOfType(element1, PyFunction.class) != null ||
+           PsiTreeUtil.getParentOfType(element, PyClass.class) != null ||
+           PsiTreeUtil.getParentOfType(element1, PyClass.class) != null;
+  }
+
+  private static boolean addLinebreak(@NotNull String text, @NotNull String toString, boolean useTabs) {
+    if ((text.startsWith(useTabs ? "\t" : " ") || StringUtil.split(text, "\n").size() > 1)
+        && !text.endsWith("\n") && !StringUtil.isEmptyOrSpaces(toString))
+      return true;
+    return false;
   }
 
   public static int getLineStartSafeOffset(final Document document, int line) {

python/src/com/jetbrains/python/psi/PyUtil.java

@@ -1181,5 +1181,26 @@ public class PyUtil {
     }
     return element instanceof PsiWhiteSpace ? null : element;
   }
+
+  @Nullable
+  public static PsiElement findNextNonWhitespaceAtOffset(@NotNull final PsiFile psiFile, int caretOffset) {
+    PsiElement element = psiFile.findElementAt(caretOffset);
+    if (element == null) {
+      return null;
+    }
+
+    final Document document = PsiDocumentManager.getInstance(psiFile.getProject()).getDocument(psiFile);
+    int lineEndOffset = 0;
+    if (document != null) {
+      int lineNumber = document.getLineNumber(caretOffset);
+      lineEndOffset = document.getLineEndOffset(lineNumber);
+    }
+    while (caretOffset <= lineEndOffset && element instanceof PsiWhiteSpace) {
+      caretOffset++;
+      element = psiFile.findElementAt(caretOffset);
+    }
+    return element instanceof PsiWhiteSpace ? null : element;
+  }
+
 }
 

python/testData/copyPaste/Empty.after.py

@@ -0,0 +1,4 @@
+class A:
+    a = 1
+
+    b = 2

python/testData/copyPaste/Empty.dst.py

@@ -0,0 +1,3 @@
+class A:
+    a = 1<caret>
+    b = 2

python/testData/copyPaste/Empty.src.py

@@ -0,0 +1,2 @@
+<selection>
+</selection>

python/testData/copyPaste/Indent2.after.py

@@ -1,6 +1,4 @@
 def bar():
     x = 1
     y = 2
-
-
 var = "string"

python/testData/copyPaste/Indent6994.after.py

@@ -0,0 +1,7 @@
+def g():
+    if True:
+        pass
+    else:
+        pass
+
+    b = 1

python/testData/copyPaste/Indent6994.dst.py

@@ -0,0 +1,3 @@
+def g():
+    <selection>a = 1</selection><caret>
+    b = 1

python/testData/copyPaste/Indent6994.src.py

@@ -0,0 +1,5 @@
+<selection>if True:
+  pass
+else:
+  pass
+</selection>

python/testData/copyPaste/Indent7709.after.py

@@ -0,0 +1,4 @@
+a = 1
+# some comment
+
+b = 2

python/testData/copyPaste/Indent7709.dst.py

@@ -0,0 +1,4 @@
+a = 1
+<caret>
+
+b = 2

python/testData/copyPaste/Indent7709.src.py

@@ -0,0 +1,4 @@
+a = 1
+<selection># some comment</selection>
+
+b = 2

python/testData/copyPaste/Indent8982.after.py

@@ -0,0 +1,4 @@
+a = 1
+a = 1
+b = 1
+d = 1

python/testData/copyPaste/Indent8982.dst.py

@@ -0,0 +1,4 @@
+a = 1
+<selection>b = 1
+c = 1
+</selection><caret>d = 1

python/testData/copyPaste/Indent8982.src.py

@@ -0,0 +1,4 @@
+<selection>a = 1
+b = 1
+</selection>c = 1
+d = 1

python/testData/copyPaste/IndentTab.after.py

@@ -1,5 +1,5 @@
 class Test:
-    def __init__(self):
+	def __init__(self):
 		print "Line 1"
 
-        print "Line 1"
+		print "Line 1"

python/testData/copyPaste/IndentTab.dst.py

@@ -1,4 +1,4 @@
 class Test:
-    def __init__(self):
-        <caret>
-        print "Line 1"
+	def __init__(self):
+		<caret>
+		print "Line 1"

python/testData/copyPaste/IndentTab.src.py

@@ -1,4 +1,4 @@
 class Test:
-    def __init__(self):
-<selection>        print "Line 1"
+	def __init__(self):
+<selection>		print "Line 1"
 </selection><caret>

python/testData/copyPaste/NonRectangleTopLevel.after.py

@@ -6,4 +6,3 @@ class A(object):
 
     def foo(self):
         pass
-

python/testData/copyPaste/Selection1.after.py

@@ -6,4 +6,4 @@ if True:
 def f():
     if True:
         a = 1
-        b = 2
+        b = 2

python/testData/copyPaste/Selection2.after.py

@@ -6,4 +6,4 @@ if True:
 def f():
     if True:
         a = 1
-        b = 2
+        b = 2

python/testData/copyPaste/TwoIndentedLines.after.py

@@ -3,5 +3,4 @@ filter(Q(myfield=123),
     Q(myanotherfield=321))
 
 a = filter(Q(myfield=123),
-           Q(myanotherfield=321))
-
+    Q(myanotherfield=321))

python/testData/copyPaste/UnfinishedCompound.after.py

@@ -2,4 +2,5 @@ def f():
     if True:
         c = 1
 
+
 a = 1

python/testData/copyPaste/multiLine/DecreaseIndent.after.py

@@ -2,6 +2,8 @@ class C:
     def foo(self):
         x = 1
         y = 2
+
+
 def foo(self):
     x = 1
     y = 2

python/testData/copyPaste/multiLine/IndentFunction.after.py

@@ -6,5 +6,6 @@ class C:
         x = 1
         y = 2
 
+
     def bar(self):
         y = 2

python/testData/copyPaste/multiLine/IndentInnerFunction1.after.py

@@ -1,6 +1,7 @@
 def foo(self):
     x = 1
     y = 2
+
     def foo(self):
         x = 1
         y = 2

python/testData/copyPaste/multiLine/IndentInnerFunction2.after.py

@@ -1,8 +1,10 @@
 def foo(self):
     x = 1
     y = 2
-       def foo(self):
-           x = 1
-           y = 2
+
+    def foo(self):
+        x = 1
+        y = 2
+
 
 z = 3

python/testData/copyPaste/singleLine/IndentInIfInDef.after.py

@@ -2,4 +2,4 @@ def f(self):
     x = 1
     if True:
         b = 2
-        x = 1
+        x = 1

python/testData/copyPaste/singleLine/IndentOnTopLevel.after.py

@@ -3,6 +3,7 @@ class C:
         x = 1
         y = 2
 
+
 x = 1
 def foo():
     pass

python/testSrc/com/jetbrains/python/PyCopyPasteTest.java

@@ -277,10 +277,25 @@ public class PyCopyPasteTest extends PyTestCase {
     doTestTabs();
   }
 
+  public void testIndent8982() {
+    doTest();
+  }
+
+  public void testIndent7709() {
+    doTest();
+  }
+
+  public void testIndent6994() {
+    doTest();
+  }
+
+  public void testEmpty() {
+    doTest();
+  }
+
   private void doTestTabs() {
     final CommonCodeStyleSettings.IndentOptions indentOptions =
-      CodeStyleSettingsManager.getSettings(myFixture.getProject())
-        .getIndentOptions(PythonFileType.INSTANCE);
+      CodeStyleSettingsManager.getSettings(myFixture.getProject()).getIndentOptions(PythonFileType.INSTANCE);
     indentOptions.USE_TAB_CHARACTER = true;
     try {
       doTest();
@@ -294,33 +309,33 @@ public class PyCopyPasteTest extends PyTestCase {
     doTestTabs();
   }
 
-  private void doTest() {
-    String name = getTestName(false);
+  private void doTest(String prefix) {
+    int oldReformat = CodeInsightSettings.getInstance().REFORMAT_ON_PASTE;
+    try {
+      CodeInsightSettings.getInstance().REFORMAT_ON_PASTE = CodeInsightSettings.NO_REFORMAT;
+      String name = getTestName(false);
 
-    myFixture.configureByFile("copyPaste/" + name + ".src.py");
-    myFixture.performEditorAction(IdeActions.ACTION_EDITOR_COPY);
-    myFixture.configureByFile("copyPaste/" + name + ".dst.py");
-    myFixture.performEditorAction(IdeActions.ACTION_EDITOR_PASTE);
-    myFixture.checkResultByFile("copyPaste/" + name + ".after.py", true);
+      myFixture.configureByFile("copyPaste/" + prefix + name + ".src.py");
+      myFixture.performEditorAction(IdeActions.ACTION_EDITOR_COPY);
+      myFixture.configureByFile("copyPaste/" + prefix + name + ".dst.py");
+      myFixture.performEditorAction(IdeActions.ACTION_EDITOR_PASTE);
+      myFixture.checkResultByFile("copyPaste/" + prefix + name + ".after.py", true);
+    }
+    finally {
+      CodeInsightSettings.getInstance().REFORMAT_ON_PASTE = oldReformat;
+    }
+
+  }
+
+  private void doTest() {
+    doTest("");
   }
 
   private void doTestSingleLine() {
-    String name = getTestName(false);
-
-    myFixture.configureByFile("copyPaste/singleLine/" + name + ".src.py");
-    myFixture.performEditorAction(IdeActions.ACTION_EDITOR_COPY);
-    myFixture.configureByFile("copyPaste/singleLine/" + name + ".dst.py");
-    myFixture.performEditorAction(IdeActions.ACTION_EDITOR_PASTE);
-    myFixture.checkResultByFile("copyPaste/singleLine/" + name + ".after.py", true);
+    doTest("singleLine/");
   }
 
   private void doTestMultiLine() {
-    String name = getTestName(false);
-
-    myFixture.configureByFile("copyPaste/multiLine/" + name + ".src.py");
-    myFixture.performEditorAction(IdeActions.ACTION_EDITOR_COPY);
-    myFixture.configureByFile("copyPaste/multiLine/" + name + ".dst.py");
-    myFixture.performEditorAction(IdeActions.ACTION_EDITOR_PASTE);
-    myFixture.checkResultByFile("copyPaste/multiLine/" + name + ".after.py", true);
+    doTest("multiLine/");
   }
 }