PY-9795 Strip leading indent from description of a field

python/psi-api/src/com/jetbrains/python/toolbox/Substring.java

@@ -17,7 +17,6 @@ package com.jetbrains.python.toolbox;
 
 import com.intellij.openapi.util.TextRange;
 import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -26,7 +25,7 @@ import java.util.regex.Pattern;
 
 /**
  * Substring with explicit offsets within its parent string.
- * <p>
+ * <p/>
  * Regular java.lang.String objects share a single char buffer for results of substring(), trim(), etc., but the offset and count
  * fields of Strings are unfortunately private.
  *
@@ -42,7 +41,7 @@ public class Substring implements CharSequence {
     }
     return new Substring(s, matcher.start(groupNumber), matcher.end(groupNumber));
   }
-  
+
   @NotNull
   public static Substring fromMatcherGroup(@NotNull Substring s, @NotNull Matcher matcher, int groupNumber) {
     if (matcher.groupCount() < groupNumber || matcher.end(groupNumber) > s.length()) {
@@ -54,7 +53,7 @@ public class Substring implements CharSequence {
   @NotNull private final String myString;
   private final int myStartOffset;
   private final int myEndOffset;
-  
+
   public Substring(@NotNull String s) {
     this(s, 0, s.length());
   }
@@ -106,7 +105,7 @@ public class Substring implements CharSequence {
   public List<Substring> split(@NotNull String regex) {
     return split(regex, Integer.MAX_VALUE);
   }
-  
+
   @NotNull
   public List<Substring> split(@NotNull String regex, int maxSplits) {
     return split(Pattern.compile(regex), maxSplits);
@@ -116,7 +115,7 @@ public class Substring implements CharSequence {
   public List<Substring> split(@NotNull Pattern pattern) {
     return split(pattern, Integer.MAX_VALUE);
   }
-  
+
   @NotNull
   public List<Substring> split(@NotNull Pattern pattern, int maxSplits) {
     final List<Substring> result = new ArrayList<Substring>();
@@ -130,11 +129,13 @@ public class Substring implements CharSequence {
         end = m.start();
         result.add(createAnotherSubstring(start, Math.min(end, myEndOffset)));
         start = m.end();
-      } while (end < myEndOffset && m.find() && splitCount < maxSplits);
+      }
+      while (end < myEndOffset && m.find() && splitCount < maxSplits);
       if (start < myEndOffset) {
         result.add(createAnotherSubstring(start, myEndOffset));
       }
-    } else {
+    }
+    else {
       result.add(createAnotherSubstring(start, end));
     }
     return result;
@@ -147,13 +148,21 @@ public class Substring implements CharSequence {
 
   @NotNull
   public Substring trim() {
+    return trimLeft().trimRight();
+  }
+  
+  @NotNull
+  public Substring trimLeft() {
     int start;
+    for (start = myStartOffset; start < myEndOffset && myString.charAt(start) <= '\u0020'; start++) { /*empty*/ }
+    return createAnotherSubstring(start, myEndOffset);
+  }
+
+  @NotNull
+  public Substring trimRight() {
     int end;
-    for (start = myStartOffset; start < myEndOffset && myString.charAt(start) <= '\u0020'; start++) {
-    }
-    for (end = myEndOffset - 1; end > start && myString.charAt(end) <= '\u0020'; end--) {
-    }
-    return createAnotherSubstring(start, end + 1);
+    for (end = myEndOffset - 1; end > myStartOffset && myString.charAt(end) <= '\u0020'; end--) { /* empty */ }
+    return createAnotherSubstring(myStartOffset, end + 1);
   }
 
   @NotNull
@@ -173,7 +182,7 @@ public class Substring implements CharSequence {
 
   @Override
   public CharSequence subSequence(int start, int end) {
-    return substring(start,  end);
+    return substring(start, end);
   }
 
   public boolean startsWith(@NotNull String prefix) {
@@ -231,18 +240,4 @@ public class Substring implements CharSequence {
   public int getEndOffset() {
     return myEndOffset;
   }
-
-  /**
-   * If both substrings share the same origin, returns new substring that includes both of them. Otherwise return {@code null}.
-   *
-   * @param other substring to concat with
-   * @return new substring as described
-   */
-  @Nullable
-  public Substring getSmallestInclusiveSubstring(@NotNull Substring other) {
-    if (myString.equals(other.myString)) {
-      return new Substring(myString, Math.min(myStartOffset, other.myStartOffset), Math.max(myEndOffset, other.myEndOffset));
-    }
-    return null;
-  }
 }

python/src/com/jetbrains/python/documentation/GoogleCodeStyleDocString.java

@@ -30,7 +30,7 @@ import java.util.regex.Pattern;
  * @author Mikhail Golubev
  */
 public class GoogleCodeStyleDocString extends SectionBasedDocString {
-  private static final Pattern SECTION_HEADER_RE = Pattern.compile("^\\s*(.+?):\\s*$");
+  private static final Pattern SECTION_HEADER_RE = Pattern.compile("^\\s*(\\w[\\s\\w]*):\\s*$");
   private static final Pattern FIELD_NAME_AND_TYPE_RE = Pattern.compile("\\s*(.+?)\\s*\\(\\s*(.+?)\\s*\\)\\s*");
   private static final Pattern SPHINX_REFERENCE_RE = Pattern.compile("(:\\w+:\\S+:`.+?`|:\\S+:`.+?`|`.+?`)"); 
 
@@ -85,11 +85,11 @@ public class GoogleCodeStyleDocString extends SectionBasedDocString {
       }
     }
     description = parts.get(1);
-    final Pair<List<Substring>, Integer> pair = parseIndentedBlock(lineNum + 1, getLineIndent(lineNum), sectionIndent);
+    final Pair<List<Substring>, Integer> pair = parseIndentedBlock(lineNum + 1, getIndent(getLine(lineNum)), sectionIndent);
     final List<Substring> nestedBlock = pair.getFirst();
     if (!nestedBlock.isEmpty()) {
       //noinspection ConstantConditions
-      description = description.getSmallestInclusiveSubstring(ContainerUtil.getLastItem(nestedBlock));
+      description = mergeSubstrings(description, ContainerUtil.getLastItem(nestedBlock));
     }
     assert description != null;
     description = description.trim();

python/src/com/jetbrains/python/documentation/SectionBasedDocString.java

@@ -20,6 +20,7 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.intellij.openapi.util.Pair;
 import com.intellij.openapi.util.text.StringUtil;
+import com.intellij.util.Function;
 import com.intellij.util.containers.ContainerUtil;
 import com.jetbrains.python.psi.StructuredDocString;
 import com.jetbrains.python.toolbox.Substring;
@@ -107,7 +108,7 @@ public abstract class SectionBasedDocString implements StructuredDocString {
     }
     int lineNum = skipEmptyLines(pair.getSecond());
     final List<SectionField> fields = new ArrayList<SectionField>();
-    final int sectionIndent = getLineIndent(sectionStartLine);
+    final int sectionIndent = getIndent(getLine(sectionStartLine));
     while (!isSectionBreak(lineNum, sectionIndent)) {
       final Pair<SectionField, Integer> result = parseField(lineNum, title, sectionIndent);
       if (result.getFirst() == null) {
@@ -136,8 +137,8 @@ public abstract class SectionBasedDocString implements StructuredDocString {
     final Substring firstLine = ContainerUtil.getFirstItem(pair.getFirst());
     final Substring lastLine = ContainerUtil.getLastItem(pair.getFirst());
     if (firstLine != null && lastLine != null) {
-      final Substring mergedSubstring = new Substring(firstLine.getSuperString(), firstLine.getStartOffset(), lastLine.getEndOffset());
-      return Pair.create(new SectionField(null, null, mergedSubstring), pair.getSecond());
+      //noinspection ConstantConditions
+      return Pair.create(new SectionField(null, null, mergeSubstrings(firstLine, lastLine).trim()), pair.getSecond());
     }
     return Pair.create(null, pair.getSecond());
   }
@@ -151,16 +152,6 @@ public abstract class SectionBasedDocString implements StructuredDocString {
   @NotNull
   protected abstract Pair<String, Integer> parseSectionHeader(int lineNum);
 
-  protected int getLineIndent(int lineNum) {
-    final Substring line = getLine(lineNum);
-    for (int i = 0; i < line.length(); i++) {
-      if (!Character.isSpaceChar(line.charAt(i))) {
-        return i;
-      }
-    }
-    return 0;
-  }
-
   private int skipEmptyLines(int lineNum) {
     while (lineNum < myLines.size() && isEmpty(lineNum)) {
       lineNum++;
@@ -189,7 +180,7 @@ public abstract class SectionBasedDocString implements StructuredDocString {
   private boolean isSectionBreak(int lineNum, int curSectionIndent) {
     return lineNum >= myLines.size() ||
            isSectionStart(lineNum) ||
-           (!isEmpty(lineNum) && getLineIndent(lineNum) <= curSectionIndent);
+           (!isEmpty(lineNum) && getIndent(getLine(lineNum)) <= curSectionIndent);
   }
 
   /**
@@ -201,7 +192,7 @@ public abstract class SectionBasedDocString implements StructuredDocString {
     final List<Substring> result = new ArrayList<Substring>();
     int lastNonEmpty = lineNum - 1;
     while (!isSectionBreak(lineNum, sectionIndent)) {
-      if (getLineIndent(lineNum) > blockIndent) {
+      if (getIndent(getLine(lineNum)) > blockIndent) {
         // copy all lines after the last non empty including the current one
         for (int i = lastNonEmpty + 1; i <= lineNum; i++) {
           result.add(getLine(lineNum));
@@ -216,6 +207,59 @@ public abstract class SectionBasedDocString implements StructuredDocString {
     return Pair.create(result, lineNum);
   }
 
+  /**
+   * If both substrings share the same origin, returns new substring that includes both of them. Otherwise return {@code null}.
+   *
+   * @param s1
+   * @param s2 substring to concat with
+   * @return new substring as described
+   */
+  @Nullable
+  public static Substring mergeSubstrings(@NotNull Substring s1, @NotNull Substring s2) {
+    if (s1.getSuperString().equals(s2.getSuperString())) {
+      return new Substring(s1.getSuperString(), 
+                           Math.min(s1.getStartOffset(), s2.getStartOffset()), 
+                           Math.max(s1.getEndOffset(), s2.getEndOffset()));
+    }
+    return null;
+  }
+
+  // like Python's textwrap.dedent()
+  @NotNull
+  protected static String stripCommonIndent(@NotNull Substring text, boolean ignoreFirstStringIfNonEmpty) {
+    final List<Substring> lines = text.splitLines();
+    if (lines.isEmpty()) {
+      return "";
+    }
+    final String firstLine = lines.get(0).toString();
+    final boolean skipFirstLine = ignoreFirstStringIfNonEmpty && !StringUtil.isEmptyOrSpaces(firstLine);
+    final Iterable<Substring> workList = lines.subList(skipFirstLine ? 1 : 0, lines.size());
+    int curMinIndent = Integer.MAX_VALUE;
+    for (Substring line : workList) {
+      if (StringUtil.isEmptyOrSpaces(line)) {
+        continue;
+      }
+      curMinIndent = Math.min(curMinIndent, getIndent(line));
+    }
+    final int minIndent = curMinIndent;
+    final List<String> dedentedLines = ContainerUtil.map(workList, new Function<Substring, String>() {
+      @Override
+      public String fun(Substring line) {
+        return line.substring(Math.min(line.length(), minIndent)).toString();
+      }
+    });
+    return StringUtil.join(skipFirstLine ? ContainerUtil.prepend(dedentedLines, firstLine) : dedentedLines, "\n");
+  }
+
+  protected static int getIndent(@NotNull CharSequence line) {
+    for (int i = 0; i < line.length(); i++) {
+      if (!Character.isSpaceChar(line.charAt(i))) {
+        return i;
+      }
+    }
+    return 0;
+  }
+
   @NotNull
   protected Substring getLine(int indent) {
     return myLines.get(indent);
@@ -428,7 +472,7 @@ public abstract class SectionBasedDocString implements StructuredDocString {
 
     @NotNull
     public String getDescription() {
-      return myDescription == null ? "" : myDescription.toString();
+      return myDescription == null ? "" : stripCommonIndent(myDescription, true);
     }
 
     @Nullable

python/testData/docstrings/nestedIndentation.py

@@ -0,0 +1,13 @@
+def func(x):
+    """
+    Parameters:
+
+      x(int): first line of the description
+        second line
+          third line
+
+        Example::
+
+            assert func(42) is None
+    
+    """

python/testSrc/com/jetbrains/python/PyGoogleCodeStyleDocStringTest.java

@@ -51,7 +51,7 @@ public class PyGoogleCodeStyleDocStringTest extends PyTestCase {
     assertEquals("y", param2.getName());
     assertEmpty(param2.getType());
     assertEquals("second parameter\n" +
-                 "        with longer description", param2.getDescription());
+                 "with longer description", param2.getDescription());
 
     assertEquals("raises", sections.get(1).getTitle());
     final List<SectionField> exceptionFields = sections.get(1).getFields();
@@ -111,8 +111,8 @@ public class PyGoogleCodeStyleDocStringTest extends PyTestCase {
     final SectionField example1 = examplesSection.getFields().get(0);
     assertEmpty(example1.getName());
     assertEmpty(example1.getType());
-    assertEquals("    Useless call\n" +
-                 "    func() == func()", example1.getDescription());
+    assertEquals("Useless call\n" +
+                 "func() == func()", example1.getDescription());
     
     final Section notesSection = docString.getSections().get(1);
     assertEquals("notes", notesSection.getTitle());
@@ -120,8 +120,8 @@ public class PyGoogleCodeStyleDocStringTest extends PyTestCase {
     final SectionField note1 = notesSection.getFields().get(0);
     assertEmpty(note1.getName());
     assertEmpty(note1.getType());
-    assertEquals("      some\n" +
-                 "        notes", note1.getDescription());
+    assertEquals("some\n" +
+                 "notes", note1.getDescription());
   }
 
   public void testTypeReferences() {
@@ -144,6 +144,24 @@ public class PyGoogleCodeStyleDocStringTest extends PyTestCase {
     assertEquals("thrown in case of any error", exception1.getDescription());
   }
 
+  public void testNestedIndentation() {
+    final GoogleCodeStyleDocString docString = findAndParseDocString();
+    assertSize(1, docString.getSections());
+    final Section section1 = docString.getSections().get(0);
+    assertEquals("parameters", section1.getTitle());
+    assertSize(1, section1.getFields());
+    final SectionField param1 = section1.getFields().get(0);
+    assertEquals("x", param1.getName());
+    assertEquals("int", param1.getType());
+    assertEquals("first line of the description\n" +
+                 "second line\n" +
+                 "  third line\n" +
+                 "\n" +
+                 "Example::\n" +
+                 "\n" +
+                 "    assert func(42) is None", param1.getDescription());
+  }
+
   @Override
   protected String getTestDataPath() {
     return super.getTestDataPath() + "/docstrings";