diff --git a/python/src/META-INF/python-plugin-core.xml b/python/src/META-INF/python-plugin-core.xml index 5919af8f1e17..42f8bb100ebf 100644 --- a/python/src/META-INF/python-plugin-core.xml +++ b/python/src/META-INF/python-plugin-core.xml @@ -577,6 +577,9 @@ + + + diff --git a/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java b/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java new file mode 100644 index 000000000000..7b17cf424c42 --- /dev/null +++ b/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java @@ -0,0 +1,173 @@ +/* + * Copyright 2000-2012 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.jetbrains.numpy.codeInsight; + +import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.psi.PsiElement; +import com.intellij.psi.PsiFile; +import com.jetbrains.numpy.documentation.NumPyDocString; +import com.jetbrains.numpy.documentation.NumPyDocStringParameter; +import com.jetbrains.python.psi.PyFunction; +import com.jetbrains.python.psi.PyNamedParameter; +import com.jetbrains.python.psi.PyPsiFacade; +import com.jetbrains.python.psi.PyQualifiedExpression; +import com.jetbrains.python.psi.types.PyType; +import com.jetbrains.python.psi.types.PyTypeProviderBase; +import com.jetbrains.python.psi.types.TypeEvalContext; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.*; + +/** + * Provides type information extracted from NumPy docstring format. + * + * @author avereshchagin + * @author vlan + */ +public class NumpyDocStringTypeProvider extends PyTypeProviderBase { + private static final Map NUMPY_ALIAS_TO_REAL_TYPE = new HashMap(); + + static { + NUMPY_ALIAS_TO_REAL_TYPE.put("ndarray", "numpy.core.multiarray.ndarray"); + // 184 occurrences + NUMPY_ALIAS_TO_REAL_TYPE.put("array_like", "collections.Iterable or int or long or float or complex"); + // Parameters marked as 'data-type' actually get any Python type identifier such as 'bool' or + // an instance of 'numpy.core.multiarray.dtype', however the type checker isn't able to check it. + // 30 occurrences + NUMPY_ALIAS_TO_REAL_TYPE.put("data-type", "object"); + // 16 occurrences + NUMPY_ALIAS_TO_REAL_TYPE.put("scalar", "int or long or float or complex"); + // 10 occurrences + NUMPY_ALIAS_TO_REAL_TYPE.put("array", "collections.Iterable"); + // 9 occurrences + NUMPY_ALIAS_TO_REAL_TYPE.put("any", "object"); + // 5 occurrences + NUMPY_ALIAS_TO_REAL_TYPE.put("Standard Python scalar object", "int or long or float or complex"); + // 4 occurrences + NUMPY_ALIAS_TO_REAL_TYPE.put("Python type", "object"); + // 3 occurrences + NUMPY_ALIAS_TO_REAL_TYPE.put("callable", "collections.Callable"); + // 3 occurrences + NUMPY_ALIAS_TO_REAL_TYPE.put("number", "int or long or float or complex"); + } + + @Nullable + @Override + public PyType getReturnType(@NotNull PyFunction function, @Nullable PyQualifiedExpression callSite, @NotNull TypeEvalContext context) { + if (isInsideNumPy(function)) { + final NumPyDocString docString = NumPyDocString.forFunction(function, callSite); + if (docString != null) { + final List returns = docString.getReturns(); + final PyPsiFacade facade = getPsiFacade(function); + switch (returns.size()) { + case 0: + // Function returns nothing + return facade.parseTypeAnnotation("None", function); + case 1: + // Function returns single value + final String typeName = returns.get(0).getType(); + if (typeName != null) { + return parseNumpyDocType(function, typeName); + } + return null; + default: + // Function returns a tuple + final List members = new ArrayList(); + for (NumPyDocStringParameter ret : returns) { + final String memberTypeName = ret.getType(); + members.add(memberTypeName != null ? parseNumpyDocType(function, memberTypeName) : null); + } + return facade.createTupleType(members, function); + } + } + } + return null; + } + + @Nullable + @Override + public PyType getParameterType(@NotNull PyNamedParameter parameter, @NotNull PyFunction function, @NotNull TypeEvalContext context) { + if (isInsideNumPy(function)) { + final String name = parameter.getName(); + if (name != null) { + return getParameterType(function, name); + } + } + return null; + } + + private static boolean isInsideNumPy(@NotNull PsiElement element) { + final PsiFile file = element.getContainingFile(); + if (file != null) { + final PyPsiFacade facade = getPsiFacade(element); + final VirtualFile virtualFile = file.getVirtualFile(); + if (virtualFile != null) { + final String name = facade.findShortestImportableName(virtualFile, element); + return name != null && name.startsWith("numpy."); + } + } + return false; + } + + private static PyPsiFacade getPsiFacade(@NotNull PsiElement anchor) { + return PyPsiFacade.getInstance(anchor.getProject()); + } + + @Nullable + private static PyType parseSingleNumpyDocType(@NotNull PsiElement anchor, @NotNull String typeString) { + final PyPsiFacade facade = getPsiFacade(anchor); + final String realTypeName = NUMPY_ALIAS_TO_REAL_TYPE.get(typeString); + if (realTypeName != null) { + final PyType type = facade.parseTypeAnnotation(realTypeName, anchor); + if (type != null) { + return type; + } + } + return facade.parseTypeAnnotation(typeString, anchor); + } + + @Nullable + private static PyType parseNumpyDocType(@NotNull PsiElement anchor, @NotNull String typeString) { + typeString = NumPyDocString.cleanupOptional(typeString); + final Set types = new LinkedHashSet(); + for (String typeName : NumPyDocString.getNumpyUnionType(typeString)) { + PyType parsedType = parseSingleNumpyDocType(anchor, typeName); + if (parsedType != null) { + types.add(parsedType); + } + } + return getPsiFacade(anchor).createUnionType(types); + } + + @Nullable + private PyType getParameterType(@NotNull PyFunction function, @NotNull String parameterName) { + final NumPyDocString docString = NumPyDocString.forFunction(function, function); + if (docString != null) { + NumPyDocStringParameter parameter = docString.getNamedParameter(parameterName); + + // If parameter name starts with "p_", and we failed to obtain it from the docstring, + // try to obtain parameter named without such prefix. + if (parameter == null && parameterName.startsWith("p_")) { + parameter = docString.getNamedParameter(parameterName.substring(2)); + } + if (parameter != null) { + return parseNumpyDocType(function, parameter.getType()); + } + } + return null; + } +} diff --git a/python/src/com/jetbrains/numpy/codeInsight/NumpyModuleMembersProvider.java b/python/src/com/jetbrains/numpy/codeInsight/NumpyModuleMembersProvider.java new file mode 100644 index 000000000000..0bc98e97467e --- /dev/null +++ b/python/src/com/jetbrains/numpy/codeInsight/NumpyModuleMembersProvider.java @@ -0,0 +1,52 @@ +/* + * Copyright 2000-2012 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.jetbrains.numpy.codeInsight; + +import com.jetbrains.python.codeInsight.PyDynamicMember; +import com.jetbrains.python.psi.PyFile; +import com.jetbrains.python.psi.types.PyModuleMembersProvider; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +/** + * Provides 'numpy' module dynamic members for numeric types. + * + * @author avereshchagin + * @author vlan + */ +public class NumpyModuleMembersProvider extends PyModuleMembersProvider { + private static final String[] NUMERIC_TYPES = { + "int8", "int16", "int32", "int64", "int128", + "uint8", "uint16", "uint32", "uint64", "uint128", + "float16", "float32", "float64", "float80", "float96", "float128", "float256", + "complex32", "complex64", "complex128", "complex160", "complex192", "complex256", "complex512" + }; + + @Override + protected Collection getMembersByQName(PyFile module, String qName) { + if ("numpy".equals(qName)) { + final List members = new ArrayList(); + for (String type : NUMERIC_TYPES) { + members.add(new PyDynamicMember(type, "numpy.core.multiarray.dtype", false)); + } + return members; + } + return Collections.emptyList(); + } +} diff --git a/python/src/com/jetbrains/numpy/documentation/NumPyDocString.java b/python/src/com/jetbrains/numpy/documentation/NumPyDocString.java new file mode 100644 index 000000000000..ea811a6afc6a --- /dev/null +++ b/python/src/com/jetbrains/numpy/documentation/NumPyDocString.java @@ -0,0 +1,344 @@ +/* + * Copyright 2000-2012 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.jetbrains.numpy.documentation; + +import com.intellij.psi.PsiDirectory; +import com.intellij.psi.PsiElement; +import com.jetbrains.python.PyNames; +import com.jetbrains.python.psi.PyClass; +import com.jetbrains.python.psi.PyFile; +import com.jetbrains.python.psi.PyFunction; +import com.jetbrains.python.psi.PyPsiFacade; +import com.jetbrains.python.psi.impl.PyQualifiedName; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * @author avereshchagin + * @author vlan + */ +public class NumPyDocString { + private static final Pattern LINE_SEPARATOR = Pattern.compile("\n|\r|\r\n"); + private static final Pattern WHITE_SPACED_LINE = Pattern.compile("^[ \t]+$"); + private static final Pattern ANY_INDENT = Pattern.compile("(^[ \t]*)[^ \t\r\n]"); + private static final Pattern HAS_INDENT = Pattern.compile("(^[ \t]+)[^ \t\r\n]"); + private static final Pattern SIGNATURE = Pattern.compile("^([\\w., ]+=)?\\s*[\\w\\.]+\\(.*\\)$"); + private static final Pattern SECTION_HEADER = Pattern.compile("^[-=]+"); + private static final Pattern PARAMETER_WITH_TYPE = Pattern.compile("^(.+) : (.+)$"); + private static final Pattern PARAMETER_WITHOUT_TYPE = Pattern.compile("^([^ :,]+)$"); + private static final Pattern REDIRECT = Pattern.compile("^Refer to `(.*)` for full documentation.$"); + private static final Pattern NUMPY_UNION_PATTERN = Pattern.compile("^\\{(.*)\\}$"); + private static final Pattern QUOTED_STRING_PATTERN = Pattern.compile("^(?:\\\"(.*)\\\")|(?:\\'(.*)\\')$"); + + private final String mySignature; + private final List myParameters = new ArrayList(); + private final List myReturns = new ArrayList(); + + private NumPyDocString(@Nullable String signature, @NotNull List lines) + throws NotNumpyDocStringException { + mySignature = signature; + parseSections(lines); + if (myReturns.size() == 0 && myParameters.size() == 0) { + throw new NotNumpyDocStringException(signature); + } + } + + @Nullable + public String getSignature() { + return mySignature; + } + + @NotNull + public List getParameters() { + return myParameters; + } + + @NotNull + public List getReturns() { + return myReturns; + } + + @Nullable + public NumPyDocStringParameter getNamedParameter(@NotNull String name) { + for (NumPyDocStringParameter parameter : getParameters()) { + if (name.equals(parameter.getName())) { + return parameter; + } + } + return null; + } + + /** + * Returns PyFunction object for specified fully qualified name accessible from specified reference. + * + * @param redirect A fully qualified name of function that is redirected to. + * @param reference An original reference element. + * @return Resolved function or null if it was not resolved. + */ + @Nullable + private static PyFunction resolveRedirectToFunction(@NotNull String redirect, @NotNull PsiElement reference) { + final PyQualifiedName qualifiedName = PyQualifiedName.fromDottedString(redirect); + final String functionName = qualifiedName.getLastComponent(); + final PyPsiFacade facade = PyPsiFacade.getInstance(reference.getProject()); + final List items = facade.qualifiedNameResolver(qualifiedName.removeLastComponent()).fromElement(reference).resultsAsList(); + for (PsiElement item : items) { + if (item instanceof PsiDirectory) { + item = ((PsiDirectory)item).findFile(PyNames.INIT_DOT_PY); + } + if (item instanceof PyFile) { + final PsiElement element = ((PyFile)item).getElementNamed(functionName); + if (element instanceof PyFunction) { + return (PyFunction)element; + } + } + } + return null; + } + + @Nullable + private static NumPyDocString forFunction(@NotNull PyFunction function, @Nullable PsiElement reference, @Nullable String knownSignature) { + String docString = function.getDocStringValue(); + if (docString == null && "__init__".equals(function.getName())) { + // Docstring for constructor can be found in the docstring of class + PyClass cls = function.getContainingClass(); + if (cls != null) { + docString = cls.getDocStringValue(); + } + } + + if (docString != null) { + List lines = splitByLines(docString); + dedent(lines); + + String signature = null; + if (!lines.isEmpty() && SIGNATURE.matcher(lines.get(0)).matches()) { + signature = lines.get(0); + lines.remove(0); + dedent(lines); + } + + String redirect = findRedirect(lines); + if (redirect != null && reference != null) { + PyFunction resolvedFunction = resolveRedirectToFunction(redirect, reference); + if (resolvedFunction != null) { + return forFunction(resolvedFunction, reference, knownSignature != null ? knownSignature : signature); + } + } + try { + return new NumPyDocString(knownSignature != null ? knownSignature : signature, lines); + } + catch (NotNumpyDocStringException e) { + return null; + } + } + return null; + } + + /** + * Returns NumPyDocString object confirming to Numpy-style formatted docstring of specified function. + * + * @param function Function containing docstring for which Numpy wrapper object is to be obtained. + * @param reference An original reference element to specified function. + * @return Numpy docstring wrapper object for specified function. + */ + @Nullable + public static NumPyDocString forFunction(@NotNull PyFunction function, @Nullable PsiElement reference) { + return forFunction(function, reference, null); + } + + @NotNull + private static List splitByLines(@NotNull String text) { + List lines = new ArrayList(); + for (String line : LINE_SEPARATOR.split(text)) { + if (!line.isEmpty() && !WHITE_SPACED_LINE.matcher(line).matches()) { + lines.add(line); + } + } + return lines; + } + + private static void dedent(@NotNull List lines) { + String margin = null; + for (String line : lines) { + Matcher matcher = ANY_INDENT.matcher(line); + if (matcher.find() && matcher.groupCount() != 0) { + String indent = matcher.group(1); + if (margin == null || (margin.startsWith(indent) && margin.length() != indent.length())) { + // update margin + margin = indent; + } else if (!indent.startsWith(margin)) { + // lines have no common margin + margin = ""; + break; + } + } + } + + if (margin != null && !margin.isEmpty()) { + for (int i = 0; i < lines.size(); i++) { + lines.set(i, lines.get(i).substring(margin.length())); + } + } + } + + private static int indexOfMatch(@NotNull List lines, @NotNull Pattern pattern, int start) { + for (int i = start; i < lines.size(); i++) { + if (pattern.matcher(lines.get(i)).matches()) { + return i; + } + } + return -1; + } + + @NotNull + private static List copyOfRange(@NotNull List src, int start, int end) { + List dest = new ArrayList(); + if (start < 0) { + start = 0; + } + if (end < 0) { + end = src.size(); + } + for (int i = start; i < end; i++) { + dest.add(src.get(i)); + } + return dest; + } + + @Nullable + private static String findRedirect(@NotNull List lines) { + for (String line : lines) { + Matcher matcher = REDIRECT.matcher(line); + if (matcher.matches() && matcher.groupCount() > 0) { + return matcher.group(1); + } + } + return null; + } + + private void parseSections(@NotNull List lines) { + int current = indexOfMatch(lines, SECTION_HEADER, 1); + while (current != -1) { + int next = indexOfMatch(lines, SECTION_HEADER, current + 1); + String sectionName = lines.get(current - 1); + if ("Parameters".equalsIgnoreCase(sectionName)) { + parseParametersSection(copyOfRange(lines, current + 1, next - 1), myParameters); + } else if ("Returns".equalsIgnoreCase(sectionName)) { + parseParametersSection(copyOfRange(lines, current + 1, next - 1), myReturns); + } + current = next; + } + } + + private static void parseParametersSection(@NotNull List lines, List parameters) { + DocStringParameterBuilder builder = null; + for (String line : lines) { + if (!HAS_INDENT.matcher(line).find()) { + if (builder != null) { + parameters.add(builder.build()); + } + builder = new DocStringParameterBuilder(); + Matcher parameterWithTypeMatcher = PARAMETER_WITH_TYPE.matcher(line); + if (parameterWithTypeMatcher.matches()) { + builder.setName(parameterWithTypeMatcher.group(1)); + builder.setType(parameterWithTypeMatcher.group(2)); + } else { + Matcher parameterWithoutTypeMatcher = PARAMETER_WITHOUT_TYPE.matcher(line); + if (parameterWithoutTypeMatcher.matches()) { + builder.setName(parameterWithoutTypeMatcher.group(1)); + builder.setType("object"); + } + } + } else { + if (builder != null) { + builder.appendDescription(line.trim()); + } + } + } + if (builder != null) { + parameters.add(builder.build()); + } + } + + @NotNull + public static String cleanupOptional(@NotNull String typeString) { + int index = typeString.indexOf(", optional"); + if (index >= 0) { + return typeString.substring(0, index); + } + return typeString; + } + + @NotNull + public static List getNumpyUnionType(@NotNull String typeString) { + Matcher matcher = NUMPY_UNION_PATTERN.matcher(typeString); + if (matcher.matches()) { + typeString = matcher.group(1); + } + return Arrays.asList(typeString.split(" *, *")); + } + + @NotNull + public static Set extractPermissibleArgumentsFromNumpyDocType(String typeString) { + List elements = getNumpyUnionType(cleanupOptional(typeString)); + Set result = new LinkedHashSet(); + for (String element : elements) { + Matcher matcher = QUOTED_STRING_PATTERN.matcher(element); + if (matcher.matches()) { + if (matcher.group(1) != null) { + result.add(matcher.group(1)); + } else if (matcher.group(2) != null) { + result.add(matcher.group(2)); + } + } + } + return result; + } + + public static class NotNumpyDocStringException extends Exception { + + public NotNumpyDocStringException(String signature) { + super("Function " + signature + " is not containing docstring of Numpy format."); + } + } + + public static class DocStringParameterBuilder { + private String myName = ""; + private String myType = ""; + private StringBuilder myDescription = new StringBuilder(); + + public void setName(String name) { + myName = name; + } + + public void setType(String type) { + myType = type; + } + + public void appendDescription(String text) { + myDescription.append(" "); + myDescription.append(text); + } + + public NumPyDocStringParameter build() { + return new NumPyDocStringParameter(myName, myType, myDescription.toString()); + } + } +} diff --git a/python/src/com/jetbrains/numpy/documentation/NumPyDocStringParameter.java b/python/src/com/jetbrains/numpy/documentation/NumPyDocStringParameter.java new file mode 100644 index 000000000000..4502c48a58d4 --- /dev/null +++ b/python/src/com/jetbrains/numpy/documentation/NumPyDocStringParameter.java @@ -0,0 +1,48 @@ +/* + * Copyright 2000-2012 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.jetbrains.numpy.documentation; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + + +/** + * @author avereshchagin + * @author vlan + */ +public class NumPyDocStringParameter { + private final String myName; + private final String myType; + private final String myDescription; + + public NumPyDocStringParameter(@NotNull String name, @Nullable String type, @Nullable String description) { + myName = name; + myType = type; + myDescription = description; + } + + public String getName() { + return myName; + } + + public String getType() { + return myType; + } + + public String getDescription() { + return myDescription; + } +}