MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

Better parsing of Numpy docstring with empty section indent

Browse Source 
Also field parsing is stricter now and it can't be parsed if parameter
name isn't valid Python identifier.

As soon as I fixed parsing of Numpy docstring format it caused errors in
multiple tests that used types of function parameters, because
previously in these places docstrings couldn't have been parsed
successfully (they were treated as Epydoc docstrings) and
PyNamedParameterImpl#getType delegated to NumpyDocStringTypeProvider.
I explicitly prohibited using of NumpyDocString for this purpose before
NumpyDocStringTypeProvider is migrated to the newer docstring API.
This commit is contained in:
Mikhail Golubev
2015-09-03 18:57:41 +03:00
parent d234cafc38
commit feb4206ce3
10 changed files with 81 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/src/com/jetbrains/python/documentation/DocStringLineParser.java
+6 -3
View File
@@ -82,11 +82,14 @@ public abstract class DocStringLineParser {
* it means that no block with requested indentation exist. Block can contain intermediate empty lines or lines that consists
* solely of spaces: their indentation is ignored, but it always ends with non-empty line. Non-empty lines in a block must
* have indentation greater than indentThreshold and for them {@link #isBlockEnd(int)} must return false.
*
* @param indentThreshold indentation size of non-empty line that will break block once reached, i.e. minimum possible indentation
* inside a block is {@code indentThreshold + 1}
*/
public int parseIndentedBlock(int startLine, int indentThreshold) {
public int consumeIndentedBlock(int startLine, int indentThreshold) {
int blockEnd = startLine - 1;
int lineNum = startLine;
while (!(lineNum >= getLineCount() || isBlockEnd(lineNum))) {
while (lineNum < getLineCount() && !isBlockEnd(lineNum)) {
if (!isEmpty(lineNum)) {
if (getLineIndentSize(lineNum) > indentThreshold) {
blockEnd = lineNum;
@@ -100,7 +103,7 @@ public abstract class DocStringLineParser {
return blockEnd + 1;
}
public int skipEmptyLines(int lineNum) {
public int consumeEmptyLines(int lineNum) {
while (lineNum < getLineCount() && isEmpty(lineNum)) {
lineNum++;
}
python/src/com/jetbrains/python/documentation/GoogleCodeStyleDocString.java
+5 -1
View File
@@ -17,6 +17,7 @@ package com.jetbrains.python.documentation;
import com.intellij.openapi.util.Pair;
import com.intellij.util.containers.ContainerUtil;
import com.jetbrains.python.PyNames;
import com.jetbrains.python.toolbox.Substring;
import org.jetbrains.annotations.NonNls;
import org.jetbrains.annotations.NotNull;
@@ -98,9 +99,12 @@ public class GoogleCodeStyleDocString extends SectionBasedDocString {
type = name;
name = null;
}
if (name != null ? !PyNames.isIdentifierString(name.toString()) : type.isEmpty()) {
return Pair.create(null, lineNum);
}
description = colonSeparatedParts.get(1);
// parse line with indentation at least one space greater than indentation of the field
final Pair<List<Substring>, Integer> pair = parseIndentedBlock(lineNum + 1, getLineIndentSize(lineNum), sectionIndent);
final Pair<List<Substring>, Integer> pair = parseIndentedBlock(lineNum + 1, getLineIndentSize(lineNum));
final List<Substring> nestedBlock = pair.getFirst();
if (!nestedBlock.isEmpty()) {
//noinspection ConstantConditions
python/src/com/jetbrains/python/documentation/NumpyDocString.java
+6 -2
View File
@@ -16,6 +16,7 @@
package com.jetbrains.python.documentation;
import com.intellij.openapi.util.Pair;
import com.jetbrains.python.PyNames;
import com.jetbrains.python.toolbox.Substring;
import org.jetbrains.annotations.NonNls;
import org.jetbrains.annotations.NotNull;
@@ -39,7 +40,7 @@ public class NumpyDocString extends SectionBasedDocString {
@Override
protected int parseHeader(int startLine) {
final int nextNonEmptyLineNum = skipEmptyLines(startLine);
final int nextNonEmptyLineNum = consumeEmptyLines(startLine);
final Substring line = getLineOrNull(nextNonEmptyLineNum);
if (line != null && SIGNATURE.matcher(line).matches()) {
mySignature = line.trim();
@@ -82,7 +83,10 @@ public class NumpyDocString extends SectionBasedDocString {
type = name;
name = null;
}
final Pair<List<Substring>, Integer> parsedDescription = parseIndentedBlock(lineNum + 1, getLineIndentSize(lineNum), sectionIndent);
if (name != null? !PyNames.isIdentifierString(name.toString()) : type.isEmpty()) {
return Pair.create(null, lineNum);
}
final Pair<List<Substring>, Integer> parsedDescription = parseIndentedBlock(lineNum + 1, getLineIndentSize(lineNum));
final List<Substring> descriptionLines = parsedDescription.getFirst();
if (!descriptionLines.isEmpty()) {
description = descriptionLines.get(0).union(descriptionLines.get(descriptionLines.size() - 1));
python/src/com/jetbrains/python/documentation/PlainDocString.java
+1 -1
View File
@@ -36,7 +36,7 @@ public class PlainDocString extends DocStringLineParser implements StructuredDoc
super(content);
if (!isEmpty(0) && isEmptyOrDoesNotExist(1)) {
mySummary = getLine(0).trim().toString();
final int next = skipEmptyLines(1);
final int next = consumeEmptyLines(1);
if (next != 1) {
final String remaining = getLine(next).union(getLine(getLineCount() - 1)).toString();
myDescription = PyIndentUtil.removeCommonIndent(remaining, false);
python/src/com/jetbrains/python/documentation/SectionBasedDocString.java
+21 -16
View File
@@ -100,7 +100,7 @@ public abstract class SectionBasedDocString extends DocStringLineParser implemen
protected SectionBasedDocString(@NotNull Substring text) {
super(text);
List<Substring> summary = Collections.emptyList();
int startLine = skipEmptyLines(parseHeader(0));
int startLine = consumeEmptyLines(parseHeader(0));
int lineNum = startLine;
while (lineNum < getLineCount()) {
final Pair<Section, Integer> parsedSection = parseSection(lineNum);
@@ -117,7 +117,7 @@ public abstract class SectionBasedDocString extends DocStringLineParser implemen
myOtherContent.add(getLine(lineNum));
lineNum++;
}
lineNum = skipEmptyLines(lineNum);
lineNum = consumeEmptyLines(lineNum);
}
//noinspection ConstantConditions
mySummary = summary.isEmpty() ? null : summary.get(0).union(summary.get(summary.size() - 1)).trim();
@@ -144,29 +144,32 @@ public abstract class SectionBasedDocString extends DocStringLineParser implemen
@NotNull
protected Pair<Section, Integer> parseSection(int sectionStartLine) {
final Pair<Substring, Integer> pair = parseSectionHeader(sectionStartLine);
if (pair.getFirst() == null) {
final Pair<Substring, Integer> parsedHeader = parseSectionHeader(sectionStartLine);
if (parsedHeader.getFirst() == null) {
return Pair.create(null, sectionStartLine);
}
final String normalized = getNormalizedSectionTitle(pair.getFirst().toString());
final String normalized = getNormalizedSectionTitle(parsedHeader.getFirst().toString());
if (normalized == null) {
return Pair.create(null, sectionStartLine);
}
int lineNum = skipEmptyLines(pair.getSecond());
final List<SectionField> fields = new ArrayList<SectionField>();
final int sectionIndent = getLineIndentSize(sectionStartLine);
int lineNum = consumeEmptyLines(parsedHeader.getSecond());
while (!isSectionBreak(lineNum, sectionIndent)) {
final Pair<SectionField, Integer> parsedField = parseSectionField(lineNum, normalized, sectionIndent);
if (!isEmpty(lineNum)) {
final Pair<SectionField, Integer> result = parseSectionField(lineNum, normalized, sectionIndent);
if (result.getFirst() != null) {
fields.add(result.getFirst());
lineNum = result.getSecond();
if (parsedField.getFirst() != null) {
fields.add(parsedField.getFirst());
lineNum = parsedField.getSecond();
continue;
}
else {
myOtherContent.add(getLine(lineNum));
}
}
lineNum++;
}
return Pair.create(new Section(pair.getFirst(), fields), lineNum);
return Pair.create(new Section(parsedHeader.getFirst(), fields), lineNum);
}
@NotNull
@@ -193,7 +196,8 @@ public abstract class SectionBasedDocString extends DocStringLineParser implemen
@NotNull
protected Pair<SectionField, Integer> parseGenericField(int lineNum, int sectionIndent) {
final Pair<List<Substring>, Integer> pair = parseIndentedBlock(lineNum, sectionIndent, sectionIndent);
// We want to let section content has the same indent as section header, in particular for Numpy
final Pair<List<Substring>, Integer> pair = parseIndentedBlock(lineNum, Math.max(sectionIndent - 1, 0));
final Substring firstLine = ContainerUtil.getFirstItem(pair.getFirst());
final Substring lastLine = ContainerUtil.getLastItem(pair.getFirst());
if (firstLine != null && lastLine != null) {
@@ -212,8 +216,9 @@ public abstract class SectionBasedDocString extends DocStringLineParser implemen
protected boolean isSectionBreak(int lineNum, int curSectionIndent) {
return lineNum >= getLineCount() ||
(!isEmpty(lineNum) && getLineIndentSize(lineNum) <= curSectionIndent) ||
isBlockEnd(lineNum);
// note that field may have the same indent as its containing section
(!isEmpty(lineNum) && getLineIndentSize(lineNum) < curSectionIndent) ||
isSectionStart(lineNum);
}
/**
@@ -223,8 +228,8 @@ public abstract class SectionBasedDocString extends DocStringLineParser implemen
* @param blockIndent indentation threshold, block ends with a line that has greater indentation
*/
@NotNull
protected Pair<List<Substring>, Integer> parseIndentedBlock(int lineNum, int blockIndent, int sectionIndent) {
final int blockEnd = parseIndentedBlock(lineNum, blockIndent);
protected Pair<List<Substring>, Integer> parseIndentedBlock(int lineNum, int blockIndent) {
final int blockEnd = consumeIndentedBlock(lineNum, blockIndent);
return Pair.create(myLines.subList(lineNum, blockEnd), blockEnd);
}
python/src/com/jetbrains/python/documentation/docstrings/DocStringUpdater.java
+1 -1
View File
@@ -79,7 +79,7 @@ public abstract class DocStringUpdater<T extends DocStringLineParser> {
}
private int skipEmptyLines(int startLine) {
return Math.min(myOriginalDocString.skipEmptyLines(startLine), myOriginalDocString.getLineCount() - 1);
return Math.min(myOriginalDocString.consumeEmptyLines(startLine), myOriginalDocString.getLineCount() - 1);
}
protected final void removeLine(int line) {
python/src/com/jetbrains/python/documentation/docstrings/TagBasedDocStringUpdater.java
+1 -1
View File
@@ -65,7 +65,7 @@ public class TagBasedDocStringUpdater extends DocStringUpdater<TagBasedDocString
for (Substring sub : nameSubs) {
if (sub.toString().equals(name)) {
final int startLine = sub.getStartLine();
final int nextAfterBlock = myOriginalDocString.parseIndentedBlock(startLine + 1, getLineIndentSize(startLine));
final int nextAfterBlock = myOriginalDocString.consumeIndentedBlock(startLine + 1, getLineIndentSize(startLine));
removeLinesAndSpacesAfter(startLine, nextAfterBlock);
}
}
python/src/com/jetbrains/python/psi/impl/PyNamedParameterImpl.java
+3 -1
View File
@@ -35,6 +35,7 @@ import com.jetbrains.python.PyTokenTypes;
import com.jetbrains.python.PythonDialectsTokenSetProvider;
import com.jetbrains.python.codeInsight.controlflow.ScopeOwner;
import com.jetbrains.python.codeInsight.dataflow.scope.ScopeUtil;
import com.jetbrains.python.documentation.NumpyDocString;
import com.jetbrains.python.psi.*;
import com.jetbrains.python.psi.resolve.PyResolveContext;
import com.jetbrains.python.psi.stubs.PyNamedParameterStub;
@@ -208,7 +209,8 @@ public class PyNamedParameterImpl extends PyBaseElementImpl<PyNamedParameterStub
docString = pyClass.getStructuredDocString();
}
}
if (docString != null) {
// FIXME Temporarily disable type extraction directly from NumpyDocString in favor of NumpyDocStringTypeProvider
if (docString != null && !(docString instanceof NumpyDocString)) {
String typeName = docString.getParamType(getName());
if (typeName != null) {
return PyTypeParser.getTypeByName(this, typeName);
python/testData/docstrings/numpyEmptySectionIndent.py
+22
View File
@@ -0,0 +1,22 @@
def func(param1):
"""
Parameters
----------
x : int
y : str
description
with continuation
Example
-------
First sentence.
Second sentence.
Returns
-------
Something
"""
pass
python/testSrc/com/jetbrains/python/PySectionBasedDocStringTest.java
+15
View File
@@ -304,6 +304,21 @@ public class PySectionBasedDocStringTest extends PyTestCase {
" Returns:\n"));
}
public void testNumpyEmptySectionIndent() {
final NumpyDocString docString = findAndParseNumpyStyleDocString();
assertSize(3, docString.getSections());
final Section paramSection = docString.getSections().get(0);
assertEquals("parameters", paramSection.getNormalizedTitle());
assertSize(2, paramSection.getFields());
final Section exampleSection = docString.getSections().get(1);
assertSize(1, exampleSection.getFields());
assertEquals("First sentence.\n" +
"Second sentence.", exampleSection.getFields().get(0).getDescription());
final Section returnSection = docString.getSections().get(2);
assertSize(1, returnSection.getFields());
assertEquals("Something", returnSection.getFields().get(0).getType());
}
@Override
protected String getTestDataPath() {
return super.getTestDataPath() + "/docstrings";