merge numpy support into main pycharm-community code

This commit is contained in:
Dmitry Jemerov
2013-09-23 16:07:28 +02:00
parent 9cc2deb327
commit 72b7ace1de
5 changed files with 620 additions and 0 deletions
@@ -577,6 +577,9 @@
<!-- PyQt -->
<typeProvider implementation="com.jetbrains.pyqt.PyQtTypeProvider"/>
<!-- NumPy -->
<pyModuleMembersProvider implementation="com.jetbrains.numpy.codeInsight.NumpyModuleMembersProvider"/>
<typeProvider implementation="com.jetbrains.numpy.codeInsight.NumpyDocStringTypeProvider"/>
</extensions>
<project-components>
@@ -0,0 +1,173 @@
/*
* Copyright 2000-2012 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.jetbrains.numpy.codeInsight;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.psi.PsiElement;
import com.intellij.psi.PsiFile;
import com.jetbrains.numpy.documentation.NumPyDocString;
import com.jetbrains.numpy.documentation.NumPyDocStringParameter;
import com.jetbrains.python.psi.PyFunction;
import com.jetbrains.python.psi.PyNamedParameter;
import com.jetbrains.python.psi.PyPsiFacade;
import com.jetbrains.python.psi.PyQualifiedExpression;
import com.jetbrains.python.psi.types.PyType;
import com.jetbrains.python.psi.types.PyTypeProviderBase;
import com.jetbrains.python.psi.types.TypeEvalContext;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.util.*;
/**
* Provides type information extracted from NumPy docstring format.
*
* @author avereshchagin
* @author vlan
*/
public class NumpyDocStringTypeProvider extends PyTypeProviderBase {
private static final Map<String, String> NUMPY_ALIAS_TO_REAL_TYPE = new HashMap<String, String>();
static {
NUMPY_ALIAS_TO_REAL_TYPE.put("ndarray", "numpy.core.multiarray.ndarray");
// 184 occurrences
NUMPY_ALIAS_TO_REAL_TYPE.put("array_like", "collections.Iterable or int or long or float or complex");
// Parameters marked as 'data-type' actually get any Python type identifier such as 'bool' or
// an instance of 'numpy.core.multiarray.dtype', however the type checker isn't able to check it.
// 30 occurrences
NUMPY_ALIAS_TO_REAL_TYPE.put("data-type", "object");
// 16 occurrences
NUMPY_ALIAS_TO_REAL_TYPE.put("scalar", "int or long or float or complex");
// 10 occurrences
NUMPY_ALIAS_TO_REAL_TYPE.put("array", "collections.Iterable");
// 9 occurrences
NUMPY_ALIAS_TO_REAL_TYPE.put("any", "object");
// 5 occurrences
NUMPY_ALIAS_TO_REAL_TYPE.put("Standard Python scalar object", "int or long or float or complex");
// 4 occurrences
NUMPY_ALIAS_TO_REAL_TYPE.put("Python type", "object");
// 3 occurrences
NUMPY_ALIAS_TO_REAL_TYPE.put("callable", "collections.Callable");
// 3 occurrences
NUMPY_ALIAS_TO_REAL_TYPE.put("number", "int or long or float or complex");
}
@Nullable
@Override
public PyType getReturnType(@NotNull PyFunction function, @Nullable PyQualifiedExpression callSite, @NotNull TypeEvalContext context) {
if (isInsideNumPy(function)) {
final NumPyDocString docString = NumPyDocString.forFunction(function, callSite);
if (docString != null) {
final List<NumPyDocStringParameter> returns = docString.getReturns();
final PyPsiFacade facade = getPsiFacade(function);
switch (returns.size()) {
case 0:
// Function returns nothing
return facade.parseTypeAnnotation("None", function);
case 1:
// Function returns single value
final String typeName = returns.get(0).getType();
if (typeName != null) {
return parseNumpyDocType(function, typeName);
}
return null;
default:
// Function returns a tuple
final List<PyType> members = new ArrayList<PyType>();
for (NumPyDocStringParameter ret : returns) {
final String memberTypeName = ret.getType();
members.add(memberTypeName != null ? parseNumpyDocType(function, memberTypeName) : null);
}
return facade.createTupleType(members, function);
}
}
}
return null;
}
@Nullable
@Override
public PyType getParameterType(@NotNull PyNamedParameter parameter, @NotNull PyFunction function, @NotNull TypeEvalContext context) {
if (isInsideNumPy(function)) {
final String name = parameter.getName();
if (name != null) {
return getParameterType(function, name);
}
}
return null;
}
private static boolean isInsideNumPy(@NotNull PsiElement element) {
final PsiFile file = element.getContainingFile();
if (file != null) {
final PyPsiFacade facade = getPsiFacade(element);
final VirtualFile virtualFile = file.getVirtualFile();
if (virtualFile != null) {
final String name = facade.findShortestImportableName(virtualFile, element);
return name != null && name.startsWith("numpy.");
}
}
return false;
}
private static PyPsiFacade getPsiFacade(@NotNull PsiElement anchor) {
return PyPsiFacade.getInstance(anchor.getProject());
}
@Nullable
private static PyType parseSingleNumpyDocType(@NotNull PsiElement anchor, @NotNull String typeString) {
final PyPsiFacade facade = getPsiFacade(anchor);
final String realTypeName = NUMPY_ALIAS_TO_REAL_TYPE.get(typeString);
if (realTypeName != null) {
final PyType type = facade.parseTypeAnnotation(realTypeName, anchor);
if (type != null) {
return type;
}
}
return facade.parseTypeAnnotation(typeString, anchor);
}
@Nullable
private static PyType parseNumpyDocType(@NotNull PsiElement anchor, @NotNull String typeString) {
typeString = NumPyDocString.cleanupOptional(typeString);
final Set<PyType> types = new LinkedHashSet<PyType>();
for (String typeName : NumPyDocString.getNumpyUnionType(typeString)) {
PyType parsedType = parseSingleNumpyDocType(anchor, typeName);
if (parsedType != null) {
types.add(parsedType);
}
}
return getPsiFacade(anchor).createUnionType(types);
}
@Nullable
private PyType getParameterType(@NotNull PyFunction function, @NotNull String parameterName) {
final NumPyDocString docString = NumPyDocString.forFunction(function, function);
if (docString != null) {
NumPyDocStringParameter parameter = docString.getNamedParameter(parameterName);
// If parameter name starts with "p_", and we failed to obtain it from the docstring,
// try to obtain parameter named without such prefix.
if (parameter == null && parameterName.startsWith("p_")) {
parameter = docString.getNamedParameter(parameterName.substring(2));
}
if (parameter != null) {
return parseNumpyDocType(function, parameter.getType());
}
}
return null;
}
}
@@ -0,0 +1,52 @@
/*
* Copyright 2000-2012 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.jetbrains.numpy.codeInsight;
import com.jetbrains.python.codeInsight.PyDynamicMember;
import com.jetbrains.python.psi.PyFile;
import com.jetbrains.python.psi.types.PyModuleMembersProvider;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
/**
* Provides 'numpy' module dynamic members for numeric types.
*
* @author avereshchagin
* @author vlan
*/
public class NumpyModuleMembersProvider extends PyModuleMembersProvider {
private static final String[] NUMERIC_TYPES = {
"int8", "int16", "int32", "int64", "int128",
"uint8", "uint16", "uint32", "uint64", "uint128",
"float16", "float32", "float64", "float80", "float96", "float128", "float256",
"complex32", "complex64", "complex128", "complex160", "complex192", "complex256", "complex512"
};
@Override
protected Collection<PyDynamicMember> getMembersByQName(PyFile module, String qName) {
if ("numpy".equals(qName)) {
final List<PyDynamicMember> members = new ArrayList<PyDynamicMember>();
for (String type : NUMERIC_TYPES) {
members.add(new PyDynamicMember(type, "numpy.core.multiarray.dtype", false));
}
return members;
}
return Collections.emptyList();
}
}
@@ -0,0 +1,344 @@
/*
* Copyright 2000-2012 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.jetbrains.numpy.documentation;
import com.intellij.psi.PsiDirectory;
import com.intellij.psi.PsiElement;
import com.jetbrains.python.PyNames;
import com.jetbrains.python.psi.PyClass;
import com.jetbrains.python.psi.PyFile;
import com.jetbrains.python.psi.PyFunction;
import com.jetbrains.python.psi.PyPsiFacade;
import com.jetbrains.python.psi.impl.PyQualifiedName;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* @author avereshchagin
* @author vlan
*/
public class NumPyDocString {
private static final Pattern LINE_SEPARATOR = Pattern.compile("\n|\r|\r\n");
private static final Pattern WHITE_SPACED_LINE = Pattern.compile("^[ \t]+$");
private static final Pattern ANY_INDENT = Pattern.compile("(^[ \t]*)[^ \t\r\n]");
private static final Pattern HAS_INDENT = Pattern.compile("(^[ \t]+)[^ \t\r\n]");
private static final Pattern SIGNATURE = Pattern.compile("^([\\w., ]+=)?\\s*[\\w\\.]+\\(.*\\)$");
private static final Pattern SECTION_HEADER = Pattern.compile("^[-=]+");
private static final Pattern PARAMETER_WITH_TYPE = Pattern.compile("^(.+) : (.+)$");
private static final Pattern PARAMETER_WITHOUT_TYPE = Pattern.compile("^([^ :,]+)$");
private static final Pattern REDIRECT = Pattern.compile("^Refer to `(.*)` for full documentation.$");
private static final Pattern NUMPY_UNION_PATTERN = Pattern.compile("^\\{(.*)\\}$");
private static final Pattern QUOTED_STRING_PATTERN = Pattern.compile("^(?:\\\"(.*)\\\")|(?:\\'(.*)\\')$");
private final String mySignature;
private final List<NumPyDocStringParameter> myParameters = new ArrayList<NumPyDocStringParameter>();
private final List<NumPyDocStringParameter> myReturns = new ArrayList<NumPyDocStringParameter>();
private NumPyDocString(@Nullable String signature, @NotNull List<String> lines)
throws NotNumpyDocStringException {
mySignature = signature;
parseSections(lines);
if (myReturns.size() == 0 && myParameters.size() == 0) {
throw new NotNumpyDocStringException(signature);
}
}
@Nullable
public String getSignature() {
return mySignature;
}
@NotNull
public List<NumPyDocStringParameter> getParameters() {
return myParameters;
}
@NotNull
public List<NumPyDocStringParameter> getReturns() {
return myReturns;
}
@Nullable
public NumPyDocStringParameter getNamedParameter(@NotNull String name) {
for (NumPyDocStringParameter parameter : getParameters()) {
if (name.equals(parameter.getName())) {
return parameter;
}
}
return null;
}
/**
* Returns PyFunction object for specified fully qualified name accessible from specified reference.
*
* @param redirect A fully qualified name of function that is redirected to.
* @param reference An original reference element.
* @return Resolved function or null if it was not resolved.
*/
@Nullable
private static PyFunction resolveRedirectToFunction(@NotNull String redirect, @NotNull PsiElement reference) {
final PyQualifiedName qualifiedName = PyQualifiedName.fromDottedString(redirect);
final String functionName = qualifiedName.getLastComponent();
final PyPsiFacade facade = PyPsiFacade.getInstance(reference.getProject());
final List<PsiElement> items = facade.qualifiedNameResolver(qualifiedName.removeLastComponent()).fromElement(reference).resultsAsList();
for (PsiElement item : items) {
if (item instanceof PsiDirectory) {
item = ((PsiDirectory)item).findFile(PyNames.INIT_DOT_PY);
}
if (item instanceof PyFile) {
final PsiElement element = ((PyFile)item).getElementNamed(functionName);
if (element instanceof PyFunction) {
return (PyFunction)element;
}
}
}
return null;
}
@Nullable
private static NumPyDocString forFunction(@NotNull PyFunction function, @Nullable PsiElement reference, @Nullable String knownSignature) {
String docString = function.getDocStringValue();
if (docString == null && "__init__".equals(function.getName())) {
// Docstring for constructor can be found in the docstring of class
PyClass cls = function.getContainingClass();
if (cls != null) {
docString = cls.getDocStringValue();
}
}
if (docString != null) {
List<String> lines = splitByLines(docString);
dedent(lines);
String signature = null;
if (!lines.isEmpty() && SIGNATURE.matcher(lines.get(0)).matches()) {
signature = lines.get(0);
lines.remove(0);
dedent(lines);
}
String redirect = findRedirect(lines);
if (redirect != null && reference != null) {
PyFunction resolvedFunction = resolveRedirectToFunction(redirect, reference);
if (resolvedFunction != null) {
return forFunction(resolvedFunction, reference, knownSignature != null ? knownSignature : signature);
}
}
try {
return new NumPyDocString(knownSignature != null ? knownSignature : signature, lines);
}
catch (NotNumpyDocStringException e) {
return null;
}
}
return null;
}
/**
* Returns NumPyDocString object confirming to Numpy-style formatted docstring of specified function.
*
* @param function Function containing docstring for which Numpy wrapper object is to be obtained.
* @param reference An original reference element to specified function.
* @return Numpy docstring wrapper object for specified function.
*/
@Nullable
public static NumPyDocString forFunction(@NotNull PyFunction function, @Nullable PsiElement reference) {
return forFunction(function, reference, null);
}
@NotNull
private static List<String> splitByLines(@NotNull String text) {
List<String> lines = new ArrayList<String>();
for (String line : LINE_SEPARATOR.split(text)) {
if (!line.isEmpty() && !WHITE_SPACED_LINE.matcher(line).matches()) {
lines.add(line);
}
}
return lines;
}
private static void dedent(@NotNull List<String> lines) {
String margin = null;
for (String line : lines) {
Matcher matcher = ANY_INDENT.matcher(line);
if (matcher.find() && matcher.groupCount() != 0) {
String indent = matcher.group(1);
if (margin == null || (margin.startsWith(indent) && margin.length() != indent.length())) {
// update margin
margin = indent;
} else if (!indent.startsWith(margin)) {
// lines have no common margin
margin = "";
break;
}
}
}
if (margin != null && !margin.isEmpty()) {
for (int i = 0; i < lines.size(); i++) {
lines.set(i, lines.get(i).substring(margin.length()));
}
}
}
private static int indexOfMatch(@NotNull List<String> lines, @NotNull Pattern pattern, int start) {
for (int i = start; i < lines.size(); i++) {
if (pattern.matcher(lines.get(i)).matches()) {
return i;
}
}
return -1;
}
@NotNull
private static <T> List<T> copyOfRange(@NotNull List<T> src, int start, int end) {
List<T> dest = new ArrayList<T>();
if (start < 0) {
start = 0;
}
if (end < 0) {
end = src.size();
}
for (int i = start; i < end; i++) {
dest.add(src.get(i));
}
return dest;
}
@Nullable
private static String findRedirect(@NotNull List<String> lines) {
for (String line : lines) {
Matcher matcher = REDIRECT.matcher(line);
if (matcher.matches() && matcher.groupCount() > 0) {
return matcher.group(1);
}
}
return null;
}
private void parseSections(@NotNull List<String> lines) {
int current = indexOfMatch(lines, SECTION_HEADER, 1);
while (current != -1) {
int next = indexOfMatch(lines, SECTION_HEADER, current + 1);
String sectionName = lines.get(current - 1);
if ("Parameters".equalsIgnoreCase(sectionName)) {
parseParametersSection(copyOfRange(lines, current + 1, next - 1), myParameters);
} else if ("Returns".equalsIgnoreCase(sectionName)) {
parseParametersSection(copyOfRange(lines, current + 1, next - 1), myReturns);
}
current = next;
}
}
private static void parseParametersSection(@NotNull List<String> lines, List<NumPyDocStringParameter> parameters) {
DocStringParameterBuilder builder = null;
for (String line : lines) {
if (!HAS_INDENT.matcher(line).find()) {
if (builder != null) {
parameters.add(builder.build());
}
builder = new DocStringParameterBuilder();
Matcher parameterWithTypeMatcher = PARAMETER_WITH_TYPE.matcher(line);
if (parameterWithTypeMatcher.matches()) {
builder.setName(parameterWithTypeMatcher.group(1));
builder.setType(parameterWithTypeMatcher.group(2));
} else {
Matcher parameterWithoutTypeMatcher = PARAMETER_WITHOUT_TYPE.matcher(line);
if (parameterWithoutTypeMatcher.matches()) {
builder.setName(parameterWithoutTypeMatcher.group(1));
builder.setType("object");
}
}
} else {
if (builder != null) {
builder.appendDescription(line.trim());
}
}
}
if (builder != null) {
parameters.add(builder.build());
}
}
@NotNull
public static String cleanupOptional(@NotNull String typeString) {
int index = typeString.indexOf(", optional");
if (index >= 0) {
return typeString.substring(0, index);
}
return typeString;
}
@NotNull
public static List<String> getNumpyUnionType(@NotNull String typeString) {
Matcher matcher = NUMPY_UNION_PATTERN.matcher(typeString);
if (matcher.matches()) {
typeString = matcher.group(1);
}
return Arrays.asList(typeString.split(" *, *"));
}
@NotNull
public static Set<String> extractPermissibleArgumentsFromNumpyDocType(String typeString) {
List<String> elements = getNumpyUnionType(cleanupOptional(typeString));
Set<String> result = new LinkedHashSet<String>();
for (String element : elements) {
Matcher matcher = QUOTED_STRING_PATTERN.matcher(element);
if (matcher.matches()) {
if (matcher.group(1) != null) {
result.add(matcher.group(1));
} else if (matcher.group(2) != null) {
result.add(matcher.group(2));
}
}
}
return result;
}
public static class NotNumpyDocStringException extends Exception {
public NotNumpyDocStringException(String signature) {
super("Function " + signature + " is not containing docstring of Numpy format.");
}
}
public static class DocStringParameterBuilder {
private String myName = "";
private String myType = "";
private StringBuilder myDescription = new StringBuilder();
public void setName(String name) {
myName = name;
}
public void setType(String type) {
myType = type;
}
public void appendDescription(String text) {
myDescription.append(" ");
myDescription.append(text);
}
public NumPyDocStringParameter build() {
return new NumPyDocStringParameter(myName, myType, myDescription.toString());
}
}
}
@@ -0,0 +1,48 @@
/*
* Copyright 2000-2012 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.jetbrains.numpy.documentation;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
/**
* @author avereshchagin
* @author vlan
*/
public class NumPyDocStringParameter {
private final String myName;
private final String myType;
private final String myDescription;
public NumPyDocStringParameter(@NotNull String name, @Nullable String type, @Nullable String description) {
myName = name;
myType = type;
myDescription = description;
}
public String getName() {
return myName;
}
public String getType() {
return myType;
}
public String getDescription() {
return myDescription;
}
}