PY-16908 Support combined parameter declarations in Numpy docstrings

This commit is contained in:
Mikhail Golubev
2015-09-17 18:48:46 +03:00
parent 0adcb1f478
commit e28c441a73
25 changed files with 341 additions and 69 deletions
@@ -130,10 +130,11 @@ public class DocStringReferenceProvider extends PsiReferenceProvider {
@Nullable DocStringParameterReference.ReferenceType nameRefType) {
final List<PsiReference> result = new ArrayList<PsiReference>();
for (SectionBasedDocString.SectionField field : fields) {
final Substring nameSub = field.getNameAsSubstring();
if (nameRefType != null && nameSub != null && !nameSub.isEmpty()) {
final TextRange range = nameSub.getTextRange().shiftRight(offset);
result.add(new DocStringParameterReference(element, range, nameRefType));
for (Substring nameSub: field.getNamesAsSubstrings()) {
if (nameRefType != null && nameSub != null && !nameSub.isEmpty()) {
final TextRange range = nameSub.getTextRange().shiftRight(offset);
result.add(new DocStringParameterReference(element, range, nameRefType));
}
}
final Substring typeSub = field.getTypeAsSubstring();
if (typeSub != null && !typeSub.isEmpty()) {
@@ -20,6 +20,7 @@ import com.jetbrains.python.toolbox.Substring;
import org.jetbrains.annotations.NonNls;
import org.jetbrains.annotations.NotNull;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Pattern;
@@ -30,6 +31,7 @@ import java.util.regex.Pattern;
*/
public class NumpyDocString extends SectionBasedDocString {
private static final Pattern SIGNATURE = Pattern.compile("^[ \t]*([\\w., ]+=)?[ \t]*[\\w\\.]+\\(.*\\)[ \t]*$", Pattern.MULTILINE);
private static final Pattern NAME_SEPARATOR = Pattern.compile("[ \t]*,[ \t]*");
public static final Pattern SECTION_HEADER = Pattern.compile("^[ \t]*[-=]{2,}[ \t]*$", Pattern.MULTILINE);
private Substring mySignature;
@@ -67,25 +69,35 @@ public class NumpyDocString extends SectionBasedDocString {
boolean mayHaveType,
boolean preferType) {
final Substring line = getLine(lineNum);
Substring name, type = null, description = null;
Substring namesPart, type = null, description = null;
if (mayHaveType) {
final List<Substring> colonSeparatedParts = splitByFirstColon(line);
name = colonSeparatedParts.get(0).trim();
namesPart = colonSeparatedParts.get(0).trim();
if (colonSeparatedParts.size() == 2) {
type = colonSeparatedParts.get(1).trim();
}
}
else {
name = line.trim();
namesPart = line.trim();
}
if (preferType && type == null) {
type = name;
name = null;
type = namesPart;
namesPart = null;
}
if (name != null) {
name = cleanUpName(name);
final List<Substring> names = new ArrayList<Substring>();
if (namesPart != null) {
// Unlike Google code style, Numpydoc allows to list several parameter with same file together, e.g.
// x1, x2 : array_like
// Input arrays, description of `x1`, `x2`.
for (Substring name : namesPart.split(NAME_SEPARATOR)) {
final Substring identifier = cleanUpName(name);
if (!isValidName(identifier.toString())) {
return Pair.create(null, lineNum);
}
names.add(identifier);
}
}
if (name != null ? !isValidName(name.toString()) : !isValidType(type.toString())) {
if (namesPart == null && !isValidType(type.toString())) {
return Pair.create(null, lineNum);
}
final Pair<List<Substring>, Integer> parsedDescription = parseIndentedBlock(lineNum + 1, getLineIndentSize(lineNum));
@@ -93,7 +105,7 @@ public class NumpyDocString extends SectionBasedDocString {
if (!descriptionLines.isEmpty()) {
description = descriptionLines.get(0).union(descriptionLines.get(descriptionLines.size() - 1));
}
return Pair.create(new SectionField(name, type, description != null ? description.trim() : null), parsedDescription.getSecond());
return Pair.create(new SectionField(names, type, description != null ? description.trim() : null), parsedDescription.getSecond());
}
@NotNull
@@ -30,7 +30,7 @@ public class NumpyDocStringUpdater extends SectionBasedDocStringUpdater {
@Override
protected void updateParamDeclarationWithType(@NotNull Substring nameSubstring, @NotNull String type) {
insert(nameSubstring.getEndOffset(), " : " + type);
insert(myOriginalDocString.getLine(nameSubstring.getEndLine()).trimRight().getEndOffset(), " : " + type);
}
@Override
@@ -209,7 +209,7 @@ public abstract class SectionBasedDocString extends DocStringLineParser implemen
final Substring firstLine = ContainerUtil.getFirstItem(pair.getFirst());
final Substring lastLine = ContainerUtil.getLastItem(pair.getFirst());
if (firstLine != null && lastLine != null) {
return Pair.create(new SectionField(null, null, firstLine.union(lastLine).trim()), pair.getSecond());
return Pair.create(new SectionField((Substring)null, null, firstLine.union(lastLine).trim()), pair.getSecond());
}
return Pair.create(null, pair.getSecond());
}
@@ -297,10 +297,10 @@ public abstract class SectionBasedDocString extends DocStringLineParser implemen
@NotNull
@Override
public List<String> getParameters() {
return ContainerUtil.map(getParameterFields(), new Function<SectionField, String>() {
return ContainerUtil.map(getParameterSubstrings(), new Function<Substring, String>() {
@Override
public String fun(SectionField field) {
return field.getName();
public String fun(Substring substring) {
return substring.toString();
}
});
}
@@ -308,12 +308,11 @@ public abstract class SectionBasedDocString extends DocStringLineParser implemen
@NotNull
@Override
public List<Substring> getParameterSubstrings() {
return ContainerUtil.mapNotNull(getParameterFields(), new Function<SectionField, Substring>() {
@Override
public Substring fun(SectionField field) {
return field.getNameAsSubstring();
}
});
final List<Substring> result = new ArrayList<Substring>();
for (SectionField field : getParameterFields()) {
ContainerUtil.addAllNotNull(result, field.getNamesAsSubstrings());
}
return result;
}
@Nullable
@@ -357,7 +356,7 @@ public abstract class SectionBasedDocString extends DocStringLineParser implemen
return ContainerUtil.find(getParameterFields(), new Condition<SectionField>() {
@Override
public boolean value(SectionField field) {
return name.equals(field.getName());
return field.getNames().contains(name);
}
});
}
@@ -379,25 +378,23 @@ public abstract class SectionBasedDocString extends DocStringLineParser implemen
@NotNull
@Override
public List<String> getKeywordArguments() {
return ContainerUtil.mapNotNull(getKeywordArgumentFields(), new Function<SectionField, String>() {
@Override
public String fun(SectionField field) {
return field.getName();
}
});
final List<String> result = new ArrayList<String>();
for (SectionField field : getKeywordArgumentFields()) {
result.addAll(field.getNames());
}
return result;
}
@NotNull
@Override
public List<Substring> getKeywordArgumentSubstrings() {
return ContainerUtil.mapNotNull(getKeywordArgumentFields(), new Function<SectionField, Substring>() {
@Override
public Substring fun(SectionField field) {
return field.getNameAsSubstring();
}
});
final List<Substring> result = new ArrayList<Substring>();
for (SectionField field : getKeywordArgumentFields()) {
ContainerUtil.addAllNotNull(field.getNamesAsSubstrings());
}
return result;
}
@Nullable
@Override
public String getKeywordArgumentDescription(@Nullable String paramName) {
@@ -424,7 +421,7 @@ public abstract class SectionBasedDocString extends DocStringLineParser implemen
return ContainerUtil.find(getKeywordArgumentFields(), new Condition<SectionField>() {
@Override
public boolean value(SectionField field) {
return name.equals(field.getName());
return field.getNames().contains(name);
}
});
}
@@ -598,24 +595,43 @@ public abstract class SectionBasedDocString extends DocStringLineParser implemen
}
public static class SectionField {
private final Substring myName;
private final List<Substring> myNames;
private final Substring myType;
private final Substring myDescription;
public SectionField(@Nullable Substring name, @Nullable Substring type, @Nullable Substring description) {
myName = name;
this(name == null ? Collections.<Substring>emptyList() : Collections.singletonList(name), type, description);
}
public SectionField(@NotNull List<Substring> names, @Nullable Substring type, @Nullable Substring description) {
myNames = names;
myType = type;
myDescription = description;
}
@NotNull
public String getName() {
return myName == null ? "" : myName.toString();
return myNames.isEmpty() ? "" : myNames.get(0).toString();
}
@Nullable
public Substring getNameAsSubstring() {
return myName;
return myNames.isEmpty() ? null : myNames.get(0);
}
@NotNull
public List<Substring> getNamesAsSubstrings() {
return myNames;
}
@NotNull
public List<String> getNames() {
return ContainerUtil.map(myNames, new Function<Substring, String>() {
@Override
public String fun(Substring substring) {
return substring.toString();
}
});
}
@NotNull
@@ -645,7 +661,7 @@ public abstract class SectionBasedDocString extends DocStringLineParser implemen
SectionField field = (SectionField)o;
if (myName != null ? !myName.equals(field.myName) : field.myName != null) return false;
if (myNames != null ? !myNames.equals(field.myNames) : field.myNames != null) return false;
if (myType != null ? !myType.equals(field.myType) : field.myType != null) return false;
if (myDescription != null ? !myDescription.equals(field.myDescription) : field.myDescription != null) return false;
@@ -654,7 +670,7 @@ public abstract class SectionBasedDocString extends DocStringLineParser implemen
@Override
public int hashCode() {
int result = myName != null ? myName.hashCode() : 0;
int result = myNames != null ? myNames.hashCode() : 0;
result = 31 * result + (myType != null ? myType.hashCode() : 0);
result = 31 * result + (myDescription != null ? myDescription.hashCode() : 0);
return result;
@@ -91,30 +91,82 @@ public abstract class SectionBasedDocStringUpdater extends DocStringUpdater<Sect
}
@Override
public void removeParameter(@NotNull String name) {
public void removeParameter(@NotNull final String name) {
for (Section section : myOriginalDocString.getParameterSections()) {
final List<SectionField> sectionFields = section.getFields();
for (SectionField param : sectionFields) {
if (param.getName().equals(name)) {
final int endLine = getFieldEndLine(param);
if (sectionFields.size() == 1) {
removeLinesAndSpacesAfter(getSectionStartLine(section), endLine + 1);
for (SectionField field : sectionFields) {
final Substring nameSub = ContainerUtil.find(field.getNamesAsSubstrings(), new Condition<Substring>() {
@Override
public boolean value(Substring substring) {
return substring.toString().equals(name);
}
else {
final int startLine = getFieldStartLine(param);
if (ContainerUtil.getLastItem(sectionFields) == param) {
removeLines(startLine, endLine + 1);
});
if (nameSub != null) {
if (field.getNamesAsSubstrings().size() == 1) {
final int endLine = getFieldEndLine(field);
if (sectionFields.size() == 1) {
removeLinesAndSpacesAfter(getSectionStartLine(section), endLine + 1);
}
else {
removeLinesAndSpacesAfter(startLine, endLine + 1);
final int startLine = getFieldStartLine(field);
if (ContainerUtil.getLastItem(sectionFields) == field) {
removeLines(startLine, endLine + 1);
}
else {
removeLinesAndSpacesAfter(startLine, endLine + 1);
}
}
}
else {
final Substring wholeParamName = expandParamNameSubstring(nameSub);
remove(wholeParamName.getStartOffset(), wholeParamName.getEndOffset());
}
break;
}
}
}
}
@NotNull
private static Substring expandParamNameSubstring(@NotNull Substring name) {
final String superString = name.getSuperString();
int startWithStars = name.getStartOffset();
int prevNonWhitespace;
do {
prevNonWhitespace = skipWhitespaces(superString, startWithStars - 1, true);
if (prevNonWhitespace >= 0 && superString.charAt(prevNonWhitespace) == '*') {
startWithStars = prevNonWhitespace;
}
else {
break;
}
}
while (startWithStars >= 0);
if (prevNonWhitespace >= 0 && superString.charAt(prevNonWhitespace) == ',') {
return new Substring(superString, prevNonWhitespace, name.getEndOffset());
}
// end offset is always exclusive
final int nextNonWhitespace = skipWhitespaces(superString, name.getEndOffset(), false);
if (nextNonWhitespace < superString.length() && superString.charAt(nextNonWhitespace) == ',') {
// if we remove parameter with trailing comma (i.e. first parameter) remove whitespaces after it as well
return new Substring(superString, startWithStars, skipWhitespaces(superString, nextNonWhitespace + 1, false));
}
return name;
}
private static int skipWhitespaces(@NotNull String s, int start, boolean backward) {
int result = start;
while (start >= 0 && start < s.length() && " \t".indexOf(s.charAt(result)) >= 0) {
if (backward) {
result--;
}
else{
result++;
}
}
return result;
}
@Override
protected void beforeApplyingModifications() {
final List<AddParameter> newParams = new ArrayList<AddParameter>();
@@ -0,0 +1,8 @@
def f(x, *args, y, **kwargs):
"""
Parameters
----------
x, *args, **kwargs, y: Any
description
"""
@@ -0,0 +1,7 @@
def f():
"""
Parameters
==========
x, *ar<caret>gs, **kwargs
no one writes like that
"""
@@ -0,0 +1,7 @@
def f():
"""
Parameters
==========
x, **kwargs
no one writes like that
"""
@@ -0,0 +1,8 @@
def f():
"""
Parameters
==========
<caret>x, y, z : type
description
"""
@@ -0,0 +1,8 @@
def f():
"""
Parameters
==========
y, z : type
description
"""
@@ -0,0 +1,8 @@
def f():
"""
Parameters
==========
x, y, <caret>z : type
description
"""
@@ -0,0 +1,8 @@
def f():
"""
Parameters
==========
x, y : type
description
"""
@@ -0,0 +1,8 @@
def f():
"""
Parameters
==========
x, <caret>y, z : type
description
"""
@@ -0,0 +1,8 @@
def f():
"""
Parameters
==========
x, z : type
description
"""
@@ -0,0 +1,8 @@
def f(x, y, z):
"""
Parameters
----------
x, y, z : object
Description
"""
@@ -0,0 +1,8 @@
def f(x, y, z):
"""
Parameters
----------
x, y, z : object
Description
"""
@@ -0,0 +1,8 @@
def f(x, <caret>y, z):
"""
Parameters
----------
x, y, z
Description
"""
@@ -0,0 +1,8 @@
def f(x, <caret>y, z):
"""
Parameters
----------
x, y, z :
Description
"""
@@ -0,0 +1,6 @@
def f(foo, b<caret>az, quux):
"""
Parameters
----------
foo, baz, quux : str
"""
@@ -0,0 +1,6 @@
def f(foo, bar, quux):
"""
Parameters
----------
foo, bar, quux : str
"""
@@ -217,6 +217,7 @@ public class Py3QuickFixTest extends PyTestCase {
* @param available true if the fix should be available, false if it should be explicitly not available.
* @throws Exception
*/
@SuppressWarnings("Duplicates")
protected void doInspectionTest(@NonNls @NotNull String[] testFiles,
@NotNull Class inspectionClass,
@NonNls @NotNull String quickFixName,
@@ -227,10 +228,14 @@ public class Py3QuickFixTest extends PyTestCase {
myFixture.checkHighlighting(true, false, false);
final List<IntentionAction> intentionActions = myFixture.filterAvailableIntentions(quickFixName);
if (available) {
assertOneElement(intentionActions);
if (intentionActions.isEmpty()) {
throw new AssertionError("Quickfix \"" + quickFixName + "\" is not available");
}
if (intentionActions.size() > 1) {
throw new AssertionError("There are more than one quickfix with the name \"" + quickFixName + "\"");
}
if (applyFix) {
myFixture.launchAction(intentionActions.get(0));
myFixture.checkResultByFile(graftBeforeExt(testFiles[0], "_after"));
}
}
@@ -534,6 +534,46 @@ public class PyQuickFixTest extends PyTestCase {
});
}
// PY-16908
public void testNumpyDocStringRemoveFirstOfCombinedParams() {
runWithDocStringFormat(DocStringFormat.NUMPY, new Runnable() {
@Override
public void run() {
doInspectionTest(PyDocstringInspection.class, PyBundle.message("QFIX.docstring.remove.$0", "x"), true, true);
}
});
}
// PY-16908
public void testNumpyDocStringRemoveMidOfCombinedParams() {
runWithDocStringFormat(DocStringFormat.NUMPY, new Runnable() {
@Override
public void run() {
doInspectionTest(PyDocstringInspection.class, PyBundle.message("QFIX.docstring.remove.$0", "y"), true, true);
}
});
}
// PY-16908
public void testNumpyDocStringRemoveLastOfCombinedParams() {
runWithDocStringFormat(DocStringFormat.NUMPY, new Runnable() {
@Override
public void run() {
doInspectionTest(PyDocstringInspection.class, PyBundle.message("QFIX.docstring.remove.$0", "z"), true, true);
}
});
}
// PY-16908
public void testNumpyDocStringRemoveCombinedVarargParam() {
runWithDocStringFormat(DocStringFormat.NUMPY, new Runnable() {
@Override
public void run() {
doInspectionTest(PyDocstringInspection.class, PyBundle.message("QFIX.docstring.remove.$0", "args"), true, true);
}
});
}
public void testUnnecessaryBackslash() {
String[] testFiles = new String[]{"UnnecessaryBackslash.py"};
myFixture.enableInspections(PyUnnecessaryBackslashInspection.class);
@@ -625,6 +665,7 @@ public class PyQuickFixTest extends PyTestCase {
* @param available true if the fix should be available, false if it should be explicitly not available.
* @throws Exception
*/
@SuppressWarnings("Duplicates")
protected void doInspectionTest(@NonNls @NotNull String[] testFiles,
@NotNull Class inspectionClass,
@NonNls @NotNull String quickFixName,
@@ -635,10 +676,14 @@ public class PyQuickFixTest extends PyTestCase {
myFixture.checkHighlighting(true, false, false);
final List<IntentionAction> intentionActions = myFixture.filterAvailableIntentions(quickFixName);
if (available) {
assertOneElement(intentionActions);
if (intentionActions.isEmpty()) {
throw new AssertionError("Quickfix \"" + quickFixName + "\" is not available");
}
if (intentionActions.size() > 1) {
throw new AssertionError("There are more than one quickfix with the name \"" + quickFixName + "\"");
}
if (applyFix) {
myFixture.launchAction(intentionActions.get(0));
myFixture.checkResultByFile(graftBeforeExt(testFiles[0], "_after"));
}
}
@@ -351,6 +351,18 @@ public class PySectionBasedDocStringTest extends PyTestCase {
assertSize(2, returnSection.getFields());
}
// PY-16908
public void testNumpyCombinedParamDeclarations() {
final NumpyDocString docString = findAndParseNumpyStyleDocString();
assertSize(1, docString.getSections());
final Section paramSection = docString.getSections().get(0);
assertSize(1, paramSection.getFields());
final SectionField firstField = paramSection.getFields().get(0);
assertSameElements(firstField.getNames(), "x", "y", "args", "kwargs");
assertEquals(firstField.getType(), "Any");
assertEquals(firstField.getDescription(), "description");
}
@Override
protected String getTestDataPath() {
return super.getTestDataPath() + "/docstrings";
@@ -619,7 +619,17 @@ public class PyIntentionTest extends PyTestCase {
public void testParamTypeInNumpyDocStringOtherSectionExists() {
doDocParamTypeTest(DocStringFormat.NUMPY);
}
// PY-16908
public void testParamTypeInNumpyDocStringCombinedParams() {
doDocParamTypeTest(DocStringFormat.NUMPY);
}
// PY-16908
public void testParamTypeInNumpyDocStringCombinedParamsColon() {
doDocParamTypeTest(DocStringFormat.NUMPY);
}
// PY-4717
public void testReturnTypeInEmptyNumpyDocString() {
doDocReturnTypeTest(DocStringFormat.NUMPY);
@@ -203,36 +203,41 @@ public class PyRenameTest extends PyTestCase {
// PY-9795
public void testGoogleDocStringParam() {
renameWithDocStringFormat("bar");
renameWithDocStringFormat(DocStringFormat.GOOGLE, "bar");
}
// PY-9795
public void testGoogleDocStringAttribute() {
renameWithDocStringFormat("bar");
renameWithDocStringFormat(DocStringFormat.GOOGLE, "bar");
}
// PY-9795
public void testGoogleDocStringParamType() {
renameWithDocStringFormat("Bar");
renameWithDocStringFormat(DocStringFormat.GOOGLE, "Bar");
}
// PY-9795
public void testGoogleDocStringReturnType() {
renameWithDocStringFormat("Bar");
renameWithDocStringFormat(DocStringFormat.GOOGLE, "Bar");
}
// PY-16761
public void testGoogleDocStringPositionalVararg() {
renameWithDocStringFormat("bar");
renameWithDocStringFormat(DocStringFormat.GOOGLE, "bar");
}
// PY-16761
public void testGoogleDocStringKeywordVararg() {
renameWithDocStringFormat("bar");
renameWithDocStringFormat(DocStringFormat.GOOGLE, "bar");
}
private void renameWithDocStringFormat(final String newName) {
runWithDocStringFormat(DocStringFormat.GOOGLE, new Runnable() {
// PY-16908
public void testNumpyDocStringCombinedParam() {
renameWithDocStringFormat(DocStringFormat.NUMPY, "bar");
}
private void renameWithDocStringFormat(DocStringFormat format, final String newName) {
runWithDocStringFormat(format, new Runnable() {
public void run() {
doTest(newName);
}