PY-84033 type engine: callable representation parsing

GitOrigin-RevId: c2926c5d507c2380af394593785297d0a5ddad18
This commit is contained in:
Morgan Bartholomew
2026-01-14 17:37:57 +00:00
committed by intellij-monorepo-bot
parent 6742bf163d
commit bc0940e05d
39 changed files with 1154 additions and 3 deletions
@@ -93,6 +93,10 @@
<lang.parserDefinition language="PyFunctionTypeComment"
implementationClass="com.jetbrains.python.codeInsight.functionTypeComments.PyFunctionTypeAnnotationParserDefinition"/>
<!-- PyTypeRepresentation -->
<lang.parserDefinition language="PyTypeRepresentation"
implementationClass="com.jetbrains.python.codeInsight.typeRepresentation.PyTypeRepresentationParserDefinition"/>
<lang.parserDefinition language="PyTypeHint"
implementationClass="com.jetbrains.python.codeInsight.typeHints.PyTypeHintParserDefinition"/>
@@ -516,6 +516,7 @@ ACT.from.some.module.import=Import from\u2026
filetype.python.docstring.description=Python docstring
filetype.python.function.type.annotation.description=Python PEP-484 function type comment
filetype.python.type.representation.description=Type representation language
filetype.python.type.hint.description=Python PEP-484 type hint
python.docstring.format=Docstring format:
python.docstring.select.type=Select Docstring Type
@@ -0,0 +1,27 @@
/*
* Copyright 2000-2025 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.jetbrains.python.codeInsight.typeRepresentation
import com.intellij.lang.DependentLanguage
import com.intellij.lang.Language
import com.jetbrains.python.PythonLanguage
/**
* Used to represent types that are not possible to express in the language like callable types
*
* This is used to serialize types, and for type engine communication
*/
object PyTypeRepresentationDialect : Language(PythonLanguage.getInstance(), "PyTypeRepresentation"), DependentLanguage
@@ -0,0 +1,30 @@
/*
* Copyright 2000-2025 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.jetbrains.python.codeInsight.typeRepresentation
import com.intellij.psi.tree.IElementType
import com.jetbrains.python.codeInsight.typeRepresentation.psi.PyFunctionTypeRepresentation
import com.jetbrains.python.codeInsight.typeRepresentation.psi.PyNamedParameterTypeRepresentation
import com.jetbrains.python.codeInsight.typeRepresentation.psi.PyParameterListRepresentation
import com.jetbrains.python.psi.PyElementType
import com.jetbrains.python.psi.impl.PyElementImpl
object PyTypeRepresentationElementTypes {
val FUNCTION_SIGNATURE: PyElementType = PyElementType("FUNCTION_SIGNATURE") { node -> PyFunctionTypeRepresentation(node) }
val PARAMETER_TYPE_LIST: PyElementType = PyElementType("PARAMETER_TYPE_LIST") { node -> PyParameterListRepresentation(node) }
val NAMED_PARAMETER_TYPE: PyElementType = PyElementType("NAMED_PARAMETER_TYPE") { node -> PyNamedParameterTypeRepresentation(node) }
val PLACEHOLDER: IElementType = PyElementType("PLACEHOLDER") { node -> PyElementImpl(node) }
}
@@ -0,0 +1,22 @@
/*
* Copyright 2000-2025 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.jetbrains.python.codeInsight.typeRepresentation
import com.jetbrains.python.psi.PyFileElementType
object PyTypeRepresentationFileElementType : PyFileElementType(PyTypeRepresentationDialect) {
override fun getExternalId(): String = "PyTypeRepresentation.ID"
}
@@ -0,0 +1,19 @@
package com.jetbrains.python.codeInsight.typeRepresentation
import com.jetbrains.python.PyPsiBundle
import com.jetbrains.python.PythonFileType
import org.jetbrains.annotations.NonNls
object PyTypeRepresentationFileType : PythonFileType(PyTypeRepresentationDialect) {
override fun getName(): @NonNls String {
return "PythonTypeRepresentation"
}
override fun getDescription(): String {
return PyPsiBundle.message("filetype.python.type.representation.description")
}
override fun getDefaultExtension(): String {
return "pythonTypeRepresentation"
}
}
@@ -0,0 +1,263 @@
/*
* Copyright 2000-2025 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.jetbrains.python.codeInsight.typeRepresentation
import com.intellij.lang.SyntaxTreeBuilder
import com.intellij.openapi.util.NlsContexts
import com.intellij.psi.tree.IElementType
import com.jetbrains.python.PyElementTypes
import com.jetbrains.python.PyParsingBundle
import com.jetbrains.python.PyTokenTypes
import com.jetbrains.python.parsing.ExpressionParsing
import com.jetbrains.python.parsing.ParsingContext
import com.jetbrains.python.parsing.PyParser
import com.jetbrains.python.parsing.StatementParsing
import com.jetbrains.python.psi.LanguageLevel
class PyTypeRepresentationParser : PyParser() {
override fun createParsingContext(builder: SyntaxTreeBuilder?, languageLevel: LanguageLevel?): ParsingContext {
return object : ParsingContext(builder, languageLevel) {
private val myExpressionParsing = TypeRepresentationParser(this)
private val myStatementParsing = object : StatementParsing(this) {
override fun parseStatement() {
expressionParser.parseExpression()
}
}
override fun getExpressionParser(): ExpressionParsing = myExpressionParsing
override fun getStatementParser(): StatementParsing = myStatementParsing
}
}
private class TypeRepresentationParser(context: ParsingContext?) : ExpressionParsing(context) {
override fun parseExpression() {
if (myBuilder.eof()) {
return
}
// Skip statement breaks - they're not part of type expressions
if (atToken(PyTokenTypes.STATEMENT_BREAK)) {
myBuilder.advanceLexer()
return
}
val startOffset = myBuilder.currentOffset
// Handle @Todo(...) annotations for unsupported type constructs
if (atToken(PyTokenTypes.AT)) {
parseTodoAnnotation()
return
}
// Try to parse function type (callable syntax) first
if (!parseFunctionType()) {
// Not a function type, parse as regular expression
super.parseSingleExpression(false)
// Ensure we made progress
if (myBuilder.currentOffset == startOffset && !myBuilder.eof()) {
// We didn't consume anything, advance to avoid infinite loop
myBuilder.error("Unexpected token") // NON-NLS
myBuilder.advanceLexer()
}
}
}
/**
* Parses @Todo(...) style annotations used to mark unsupported type constructs.
* These are opaque markers and their content is not interpreted.
*/
private fun parseTodoAnnotation() {
assert(atToken(PyTokenTypes.AT))
val annotationMarker = myBuilder.mark()
myBuilder.advanceLexer() // consume @
// Consume tokens until we hit a structural boundary
var depth = 0
while (!myBuilder.eof()) {
val token = myBuilder.getTokenType()
when {
token == PyTokenTypes.LPAR -> {
depth++
myBuilder.advanceLexer()
}
token == PyTokenTypes.RPAR -> {
if (depth == 0) break // Structural boundary (parameter list close, etc.)
depth--
myBuilder.advanceLexer()
}
token == PyTokenTypes.COMMA && depth == 0 -> break // Structural boundary
token == PyTokenTypes.RARROW && depth == 0 -> break // Structural boundary (->)
token == PyTokenTypes.STATEMENT_BREAK -> break // Structural boundary
else -> myBuilder.advanceLexer()
}
}
annotationMarker.done(PyTypeRepresentationElementTypes.PLACEHOLDER)
}
fun parseFunctionType(): Boolean {
if (atToken(PyTokenTypes.LPAR)) {
val mark = myBuilder.mark()
parseParameterTypeList()
// Check if this is a function type (has ->) or just a parenthesized expression/tuple
if (atToken(PyTokenTypes.RARROW)) {
myBuilder.advanceLexer() // consume ->
// Use parseExpression to handle nested function types in return position
parseExpression()
mark.done(PyTypeRepresentationElementTypes.FUNCTION_SIGNATURE)
return true
}
else {
// Not a function type, rollback
mark.rollbackTo()
return false
}
}
return false
}
fun parseParameterTypeList() {
assert(atToken(PyTokenTypes.LPAR))
val listMark = myBuilder.mark()
myBuilder.advanceLexer()
var paramCount = 0
while (!(atAnyOfTokens(PyTokenTypes.RPAR, PyTokenTypes.RARROW, PyTokenTypes.STATEMENT_BREAK) || myBuilder.eof())) {
val currentOffset = myBuilder.currentOffset
if (paramCount > 0) {
if (!checkMatches(PyTokenTypes.COMMA, PyParsingBundle.message("PARSE.expected.comma"))) {
// No comma found, break out of parameter parsing
break
}
}
val parsed: Boolean
if (atToken(PyTokenTypes.DIV)) {
// Handle positional-only separator /
val slashMarker = myBuilder.mark()
myBuilder.advanceLexer()
slashMarker.done(PyElementTypes.SLASH_PARAMETER)
parsed = true
}
else if (atToken(PyTokenTypes.MULT)) {
val starMarker = myBuilder.mark()
myBuilder.advanceLexer()
// Check if this is a named *args parameter (*name: type)
if (atToken(PyTokenTypes.IDENTIFIER) && myBuilder.lookAhead(1) == PyTokenTypes.COLON) {
val namedVarargMarker = myBuilder.mark()
myBuilder.advanceLexer() // consume identifier
myBuilder.advanceLexer() // consume colon
// Parse the type expression, handling * prefix
if (atToken(PyTokenTypes.MULT)) {
val innerStarMarker = myBuilder.mark()
myBuilder.advanceLexer()
parseExpression()
innerStarMarker.done(PyElementTypes.STAR_EXPRESSION)
}
else {
parseExpression()
}
namedVarargMarker.done(PyTypeRepresentationElementTypes.NAMED_PARAMETER_TYPE)
}
else {
// Unnamed *args, just parse the type
parseExpression()
}
parsed = true
starMarker.done(PyElementTypes.STAR_EXPRESSION)
}
else if (atToken(PyTokenTypes.EXP)) {
val doubleStarMarker = myBuilder.mark()
myBuilder.advanceLexer()
// Check if this is a named **kwargs parameter (**name: type)
if (atToken(PyTokenTypes.IDENTIFIER) && myBuilder.lookAhead(1) == PyTokenTypes.COLON) {
val namedKwargMarker = myBuilder.mark()
myBuilder.advanceLexer() // consume identifier
myBuilder.advanceLexer() // consume colon
// Parse the type expression, handling ** prefix
if (atToken(PyTokenTypes.EXP)) {
val innerDoubleStarMarker = myBuilder.mark()
myBuilder.advanceLexer()
parseExpression()
innerDoubleStarMarker.done(PyElementTypes.DOUBLE_STAR_EXPRESSION)
}
else {
parseExpression()
}
namedKwargMarker.done(PyTypeRepresentationElementTypes.NAMED_PARAMETER_TYPE)
}
else {
// Unnamed **kwargs, just parse the type
parseExpression()
}
parsed = true
doubleStarMarker.done(PyElementTypes.DOUBLE_STAR_EXPRESSION)
}
else {
if (atToken(PyTokenTypes.AT)) {
parseTodoAnnotation()
parsed = true
}
// Check if this is a named parameter (name: type) or (name: type = default)
else if (atToken(PyTokenTypes.IDENTIFIER) && myBuilder.lookAhead(1) == PyTokenTypes.COLON) {
val namedParamMarker = myBuilder.mark()
myBuilder.advanceLexer() // consume identifier
myBuilder.advanceLexer() // consume colon
// Parse the type expression
parseExpression()
// Check for default value
if (atToken(PyTokenTypes.EQ)) {
myBuilder.advanceLexer() // consume =
parseExpression() // parse default value
}
namedParamMarker.done(PyTypeRepresentationElementTypes.NAMED_PARAMETER_TYPE)
parsed = myBuilder.currentOffset > currentOffset
}
else {
// Use parseExpression to handle nested function types and positional types
parseExpression()
parsed = myBuilder.currentOffset > currentOffset // Check we made progress
}
}
if (!parsed) {
myBuilder.error(PyParsingBundle.message("PARSE.expected.expression"))
recoverUntilMatches(PyParsingBundle.message("PARSE.expected.expression"), PyTokenTypes.COMMA, PyTokenTypes.RPAR,
PyTokenTypes.RARROW, PyTokenTypes.STATEMENT_BREAK)
}
paramCount++
}
checkMatches(PyTokenTypes.RPAR, PyParsingBundle.message("PARSE.expected.rpar"))
listMark.done(PyTypeRepresentationElementTypes.PARAMETER_TYPE_LIST)
}
fun recoverUntilMatches(errorMessage: @NlsContexts.ParsingError String, vararg types: IElementType) {
val errorMarker = myBuilder.mark()
var hasNonWhitespaceTokens = false
while (!(atAnyOfTokens(*types) || myBuilder.eof())) {
// Regular whitespace tokens are already skipped by advancedLexer()
if (!atToken(PyTokenTypes.STATEMENT_BREAK)) {
hasNonWhitespaceTokens = true
}
myBuilder.advanceLexer()
}
if (hasNonWhitespaceTokens) {
errorMarker.error(errorMessage)
}
else {
errorMarker.drop()
}
}
}
}
@@ -0,0 +1,35 @@
/*
* Copyright 2000-2025 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.jetbrains.python.codeInsight.typeRepresentation
import com.intellij.lang.PsiParser
import com.intellij.openapi.project.Project
import com.intellij.psi.FileViewProvider
import com.intellij.psi.PsiFile
import com.intellij.psi.tree.IFileElementType
import com.intellij.psi.tree.TokenSet
import com.jetbrains.python.PythonParserDefinition
import com.jetbrains.python.codeInsight.typeRepresentation.psi.PyTypeRepresentationFile
class PyTypeRepresentationParserDefinition : PythonParserDefinition() {
override fun getCommentTokens(): TokenSet = TokenSet.EMPTY
override fun createFile(viewProvider: FileViewProvider): PsiFile = PyTypeRepresentationFile(viewProvider)
override fun getFileNodeType(): IFileElementType = PyTypeRepresentationFileElementType
override fun createParser(project: Project?): PsiParser = PyTypeRepresentationParser()
}
@@ -0,0 +1,111 @@
/*
* Copyright 2000-2025 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.jetbrains.python.codeInsight.typeRepresentation.psi
import com.intellij.lang.ASTNode
import com.jetbrains.python.ast.findChildByClass
import com.jetbrains.python.codeInsight.typing.PyTypingTypeProvider
import com.jetbrains.python.psi.PyDoubleStarExpression
import com.jetbrains.python.psi.PyExpression
import com.jetbrains.python.psi.PySlashParameter
import com.jetbrains.python.psi.PyStarExpression
import com.jetbrains.python.psi.impl.PyBuiltinCache
import com.jetbrains.python.psi.impl.PyElementImpl
import com.jetbrains.python.psi.types.*
class PyFunctionTypeRepresentation(astNode: ASTNode) : PyElementImpl(astNode), PyExpression {
val parameterList: PyParameterListRepresentation
get() = findNotNullChildByClass(PyParameterListRepresentation::class.java)
val returnType: PyExpression?
get() = findChildByClass(PyExpression::class.java)
override fun getType(context: TypeEvalContext, key: TypeEvalContext.Key): PyType? {
val returnType = returnType ?: return null
val params = parameterList.parameters
val callableParams = params.map { param ->
when (param) {
is PySlashParameter -> {
// Positional-only separator
PyCallableParameterImpl.psi(param)
}
is PyNamedParameterTypeRepresentation -> {
val paramName = param.parameterName
val paramType = param.typeExpression?.let { resolveTypeExpression(it, context) }
PyCallableParameterImpl.nonPsi(paramName, paramType, param.defaultValue)
}
is PyStarExpression -> {
// *args parameter
// Check if it contains a named parameter
val namedParam = param.findChildByClass(PyNamedParameterTypeRepresentation::class.java)
if (namedParam != null) {
// *args: type
val paramName = namedParam.parameterName
val paramType = namedParam.typeExpression?.let { resolveTypeExpression(it, context) }
PyCallableParameterImpl.positionalNonPsi(paramName, paramType)
}
else {
// Unnamed *args: *type
val innerExpr = param.expression
val paramType = innerExpr?.let { resolveTypeExpression(it, context) }
PyCallableParameterImpl.positionalNonPsi(null, paramType)
}
}
is PyDoubleStarExpression -> {
// **kwargs parameter
// Check if it contains a named parameter
val namedParam = param.findChildByClass(PyNamedParameterTypeRepresentation::class.java)
if (namedParam != null) {
val paramName = namedParam.parameterName
val paramType = namedParam.typeExpression?.let {
if (it is PyDoubleStarExpression)
// Named kwargs unpacked: `**name: **type`
resolveTypeExpression(it.expression!!, context)
else {
// Named kwargs: `**name: type`, adapt to `dict`
val builtins = PyBuiltinCache.getInstance(it)
PyCollectionTypeImpl(
builtins.dictType!!.pyClass, false, listOf(builtins.strType, resolveTypeExpression(it, context))
)
}
}
PyCallableParameterImpl.keywordNonPsi(paramName, paramType)
}
else {
// Unnamed kwargs: `**type`
val innerExpr = param.expression
val paramType = innerExpr?.let { resolveTypeExpression(it, context) }
PyCallableParameterImpl.keywordNonPsi(null, paramType)
}
}
is PyExpression -> {
val paramType = resolveTypeExpression(param, context)
PyCallableParameterImpl.nonPsi(paramType)
}
else -> {
PyCallableParameterImpl.nonPsi(null)
}
}
}
val retType = resolveTypeExpression(returnType, context)
return PyCallableTypeImpl(callableParams, retType)
}
private fun resolveTypeExpression(expr: PyExpression, context: TypeEvalContext): PyType? = when (expr) {
is PyDoubleStarExpression -> PyTypingTypeProvider.getType(expr.expression!!, context)?.get()
else -> PyTypingTypeProvider.getType(expr, context)?.get()
}
}
@@ -0,0 +1,46 @@
/*
* Copyright 2000-2025 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.jetbrains.python.codeInsight.typeRepresentation.psi
import com.intellij.lang.ASTNode
import com.intellij.psi.PsiElement
import com.jetbrains.python.PyTokenTypes
import com.jetbrains.python.psi.PyExpression
import com.jetbrains.python.psi.impl.PyElementImpl
class PyNamedParameterTypeRepresentation(astNode: ASTNode) : PyElementImpl(astNode), PsiElement {
val parameterName: String?
get() = node.findChildByType(PyTokenTypes.IDENTIFIER)?.text
val typeExpression: PyExpression?
get() = findChildByClass(PyExpression::class.java)
val defaultValue: PyExpression?
get() {
// Find the expression after the = token
val children = node.getChildren(null)
var foundEquals = false
for (child in children) {
if (foundEquals && child.psi is PyExpression) {
return child.psi as PyExpression
}
if (child.elementType == PyTokenTypes.EQ) {
foundEquals = true
}
}
return null
}
}
@@ -0,0 +1,14 @@
package com.jetbrains.python.codeInsight.typeRepresentation.psi
import com.intellij.lang.ASTNode
import com.intellij.psi.PsiElement
import com.intellij.psi.util.elementType
import com.jetbrains.python.codeInsight.typeRepresentation.PyTypeRepresentationElementTypes
import com.jetbrains.python.psi.PyExpression
import com.jetbrains.python.psi.PySlashParameter
import com.jetbrains.python.psi.impl.PyElementImpl
class PyParameterListRepresentation(astNode: ASTNode) : PyElementImpl(astNode) {
val parameters: List<PsiElement>
get() = children.filter { it is PyExpression || it is PySlashParameter || it is PyNamedParameterTypeRepresentation || it.elementType == PyTypeRepresentationElementTypes.PLACEHOLDER }
}
@@ -0,0 +1,50 @@
package com.jetbrains.python.codeInsight.typeRepresentation.psi
import com.intellij.openapi.fileTypes.FileType
import com.intellij.psi.FileViewProvider
import com.intellij.psi.PsiElement
import com.intellij.psi.impl.PsiManagerEx
import com.intellij.testFramework.LightVirtualFile
import com.jetbrains.python.codeInsight.typeRepresentation.PyTypeRepresentationDialect
import com.jetbrains.python.codeInsight.typeRepresentation.PyTypeRepresentationFileType
import com.jetbrains.python.psi.LanguageLevel
import com.jetbrains.python.psi.PyExpression
import com.jetbrains.python.psi.PyExpressionCodeFragment
import com.jetbrains.python.psi.impl.PyFileImpl
class PyTypeRepresentationFile : PyFileImpl, PyExpressionCodeFragment {
constructor(text: String, context: PsiElement) : super(
PsiManagerEx.getInstanceEx(context.project)
.fileManager
.createFileViewProvider(
LightVirtualFile(
"foo.bar",
PyTypeRepresentationFileType,
// workaround for PY-86754: strip def types
text.replace(Regex("(?:^|[^\"'])def [\\w.]+"), "")
),
false)
) {
myContext = context
}
constructor(viewProvider: FileViewProvider) : super(viewProvider, PyTypeRepresentationDialect) {
myContext = null
}
private val myContext: PsiElement?
override fun getFileType(): FileType = PyTypeRepresentationFileType
override fun toString(): String = "TypeRepresentation:$name"
override fun getLanguageLevel(): LanguageLevel =
// The same as for .pyi files
LanguageLevel.getLatest()
override fun getContext(): PsiElement? = if (myContext?.isValid == true) myContext else super.getContext()
val type: PyExpression?
get() = findChildByClass(PyExpression::class.java)
}
@@ -29,6 +29,7 @@ import com.jetbrains.python.codeInsight.dataflow.scope.ScopeUtil;
import com.jetbrains.python.codeInsight.functionTypeComments.psi.PyFunctionTypeAnnotation;
import com.jetbrains.python.codeInsight.functionTypeComments.psi.PyFunctionTypeAnnotationFile;
import com.jetbrains.python.codeInsight.typeHints.PyTypeHintFile;
import com.jetbrains.python.codeInsight.typeRepresentation.psi.PyFunctionTypeRepresentation;
import com.jetbrains.python.psi.*;
import com.jetbrains.python.psi.impl.PyBuiltinCache;
import com.jetbrains.python.psi.impl.PyEvaluator;
@@ -1018,8 +1019,16 @@ public final class PyTypingTypeProvider extends PyTypeProviderWithCustomContext<
if (classType != null) {
return classType;
}
if (context.myUseFqn && resolved.getText().equals("Unknown")) {
return Ref.create();
if (context.myUseFqn) {
if (resolved.getText().equals("Unknown")) {
return Ref.create();
}
if (resolved instanceof PyFunctionTypeRepresentation function) {
var result = context.myContext.getType(function);
if (result != null) {
return Ref.create(result);
}
}
}
return null;
}
@@ -0,0 +1,42 @@
TypeRepresentation:a.pythonTypeRepresentation
PyFunctionTypeRepresentation
PyParameterListRepresentation
PsiElement(Py:LPAR)('(')
PyStarExpression
PsiElement(Py:MULT)('*')
PyNamedParameterTypeRepresentation
PsiElement(Py:IDENTIFIER)('a')
PsiElement(Py:COLON)(':')
PsiWhiteSpace(' ')
PyStarExpression
PsiElement(Py:MULT)('*')
PySubscriptionExpression
PyReferenceExpression: tuple
PsiElement(Py:IDENTIFIER)('tuple')
PsiElement(Py:LBRACKET)('[')
PyTupleExpression
PyReferenceExpression: int
PsiElement(Py:IDENTIFIER)('int')
PsiElement(Py:COMMA)(',')
PsiWhiteSpace(' ')
PyReferenceExpression: str
PsiElement(Py:IDENTIFIER)('str')
PsiElement(Py:RBRACKET)(']')
PsiElement(Py:COMMA)(',')
PsiWhiteSpace(' ')
PyDoubleStarExpression
PsiElement(Py:EXP)('**')
PyNamedParameterTypeRepresentation
PsiElement(Py:IDENTIFIER)('b')
PsiElement(Py:COLON)(':')
PsiWhiteSpace(' ')
PyDoubleStarExpression
PsiElement(Py:EXP)('**')
PyReferenceExpression: B
PsiElement(Py:IDENTIFIER)('B')
PsiElement(Py:RPAR)(')')
PsiWhiteSpace(' ')
PsiElement(Py:RARROW)('->')
PsiWhiteSpace(' ')
PyNoneLiteralExpression
PsiElement(Py:NONE_KEYWORD)('None')
@@ -0,0 +1,17 @@
TypeRepresentation:a.pythonTypeRepresentation
PyFunctionTypeRepresentation
PyParameterListRepresentation
PsiElement(Py:LPAR)('(')
PyReferenceExpression: int
PsiElement(Py:IDENTIFIER)('int')
PsiElement(Py:COMMA)(',')
PsiErrorElement:Unexpected token
<empty list>
PsiElement(Py:RPAR)(')')
PsiErrorElement:')' expected
<empty list>
PsiWhiteSpace(' ')
PsiElement(Py:RARROW)('->')
PsiWhiteSpace(' ')
PyNoneLiteralExpression
PsiElement(Py:NONE_KEYWORD)('None')
@@ -0,0 +1,23 @@
TypeRepresentation:a.pythonTypeRepresentation
PyFunctionTypeRepresentation
PyParameterListRepresentation
PsiElement(Py:LPAR)('(')
PyNamedParameterTypeRepresentation
PsiElement(Py:IDENTIFIER)('a')
PsiElement(Py:COLON)(':')
PsiWhiteSpace(' ')
PyReferenceExpression: int
PsiElement(Py:IDENTIFIER)('int')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyEllipsisLiteralExpression
PsiElement(Py:DOT)('.')
PsiElement(Py:DOT)('.')
PsiElement(Py:DOT)('.')
PsiElement(Py:RPAR)(')')
PsiWhiteSpace(' ')
PsiElement(Py:RARROW)('->')
PsiWhiteSpace(' ')
PyNoneLiteralExpression
PsiElement(Py:NONE_KEYWORD)('None')
@@ -0,0 +1,14 @@
TypeRepresentation:a.pythonTypeRepresentation
PyFunctionTypeRepresentation
PyParameterListRepresentation
PsiElement(Py:LPAR)('(')
PyEllipsisLiteralExpression
PsiElement(Py:DOT)('.')
PsiElement(Py:DOT)('.')
PsiElement(Py:DOT)('.')
PsiElement(Py:RPAR)(')')
PsiWhiteSpace(' ')
PsiElement(Py:RARROW)('->')
PsiWhiteSpace(' ')
PyNoneLiteralExpression
PsiElement(Py:NONE_KEYWORD)('None')
@@ -0,0 +1,2 @@
TypeRepresentation:a.pythonTypeRepresentation
<empty list>
@@ -0,0 +1,10 @@
TypeRepresentation:a.pythonTypeRepresentation
PyFunctionTypeRepresentation
PyParameterListRepresentation
PsiElement(Py:LPAR)('(')
PsiElement(Py:RPAR)(')')
PsiWhiteSpace(' ')
PsiElement(Py:RARROW)('->')
PsiWhiteSpace(' ')
PyNoneLiteralExpression
PsiElement(Py:NONE_KEYWORD)('None')
@@ -0,0 +1,16 @@
TypeRepresentation:a.pythonTypeRepresentation
PyFunctionTypeRepresentation
PyParameterListRepresentation
PsiElement(Py:LPAR)('(')
PyNamedParameterTypeRepresentation
PsiElement(Py:IDENTIFIER)('a')
PsiElement(Py:COLON)(':')
PsiWhiteSpace(' ')
PyReferenceExpression: int
PsiElement(Py:IDENTIFIER)('int')
PsiElement(Py:RPAR)(')')
PsiWhiteSpace(' ')
PsiElement(Py:RARROW)('->')
PsiWhiteSpace(' ')
PyNoneLiteralExpression
PsiElement(Py:NONE_KEYWORD)('None')
@@ -0,0 +1,38 @@
TypeRepresentation:a.pythonTypeRepresentation
PyFunctionTypeRepresentation
PyParameterListRepresentation
PsiElement(Py:LPAR)('(')
PyReferenceExpression: int
PsiElement(Py:IDENTIFIER)('int')
PsiElement(Py:COMMA)(',')
PsiWhiteSpace(' ')
PyFunctionTypeRepresentation
PyParameterListRepresentation
PsiElement(Py:LPAR)('(')
PyReferenceExpression: str
PsiElement(Py:IDENTIFIER)('str')
PsiElement(Py:RPAR)(')')
PsiWhiteSpace(' ')
PsiElement(Py:RARROW)('->')
PsiWhiteSpace(' ')
PyNoneLiteralExpression
PsiElement(Py:NONE_KEYWORD)('None')
PsiElement(Py:COMMA)(',')
PsiWhiteSpace(' ')
PyReferenceExpression: int
PsiElement(Py:IDENTIFIER)('int')
PsiElement(Py:RPAR)(')')
PsiWhiteSpace(' ')
PsiElement(Py:RARROW)('->')
PsiWhiteSpace(' ')
PyFunctionTypeRepresentation
PyParameterListRepresentation
PsiElement(Py:LPAR)('(')
PyReferenceExpression: str
PsiElement(Py:IDENTIFIER)('str')
PsiElement(Py:RPAR)(')')
PsiWhiteSpace(' ')
PsiElement(Py:RARROW)('->')
PsiWhiteSpace(' ')
PyNoneLiteralExpression
PsiElement(Py:NONE_KEYWORD)('None')
@@ -0,0 +1,10 @@
TypeRepresentation:a.pythonTypeRepresentation
PyFunctionTypeRepresentation
PyParameterListRepresentation
PsiElement(Py:LPAR)('(')
PyReferenceExpression: int
PsiElement(Py:IDENTIFIER)('int')
PsiElement(Py:RPAR)(')')
PsiWhiteSpace(' ')
PsiElement(Py:RARROW)('->')
PsiWhiteSpace(' ')
@@ -0,0 +1,16 @@
TypeRepresentation:a.pythonTypeRepresentation
PyFunctionTypeRepresentation
PyParameterListRepresentation
PsiElement(Py:LPAR)('(')
PyStarExpression
PsiElement(Py:MULT)('*')
PsiErrorElement:Unexpected token
<empty list>
PsiElement(Py:RPAR)(')')
PsiErrorElement:')' expected
<empty list>
PsiWhiteSpace(' ')
PsiElement(Py:RARROW)('->')
PsiWhiteSpace(' ')
PyReferenceExpression: int
PsiElement(Py:IDENTIFIER)('int')
@@ -0,0 +1,16 @@
TypeRepresentation:a.pythonTypeRepresentation
PyFunctionTypeRepresentation
PyParameterListRepresentation
PsiElement(Py:LPAR)('(')
PyReferenceExpression: int
PsiElement(Py:IDENTIFIER)('int')
PsiElement(Py:COMMA)(',')
PsiWhiteSpace(' ')
PyReferenceExpression: str
PsiElement(Py:IDENTIFIER)('str')
PsiElement(Py:RPAR)(')')
PsiWhiteSpace(' ')
PsiElement(Py:RARROW)('->')
PsiWhiteSpace(' ')
PyNoneLiteralExpression
PsiElement(Py:NONE_KEYWORD)('None')
@@ -0,0 +1,20 @@
TypeRepresentation:a.pythonTypeRepresentation
PyFunctionTypeRepresentation
PyParameterListRepresentation
PsiElement(Py:LPAR)('(')
PyStarExpression
PsiElement(Py:MULT)('*')
PyReferenceExpression: int
PsiElement(Py:IDENTIFIER)('int')
PsiElement(Py:COMMA)(',')
PsiWhiteSpace(' ')
PyDoubleStarExpression
PsiElement(Py:EXP)('**')
PyReferenceExpression: str
PsiElement(Py:IDENTIFIER)('str')
PsiElement(Py:RPAR)(')')
PsiWhiteSpace(' ')
PsiElement(Py:RARROW)('->')
PsiWhiteSpace(' ')
PyNoneLiteralExpression
PsiElement(Py:NONE_KEYWORD)('None')
@@ -0,0 +1,24 @@
TypeRepresentation:a.pythonTypeRepresentation
PyFunctionTypeRepresentation
PyParameterListRepresentation
PsiElement(Py:LPAR)('(')
PyElement
PsiElement(Py:AT)('@')
PsiElement(Py:IDENTIFIER)('Todo')
PsiElement(Py:LPAR)('(')
PsiElement(Py:TICK)('`')
PsiElement(Py:IDENTIFIER)('Unpack')
PsiElement(Py:LBRACKET)('[')
PsiElement(Py:RBRACKET)(']')
PsiElement(Py:TICK)('`')
PsiWhiteSpace(' ')
PsiElement(Py:IDENTIFIER)('special')
PsiWhiteSpace(' ')
PsiElement(Py:IDENTIFIER)('form')
PsiElement(Py:RPAR)(')')
PsiElement(Py:RPAR)(')')
PsiWhiteSpace(' ')
PsiElement(Py:RARROW)('->')
PsiWhiteSpace(' ')
PyNoneLiteralExpression
PsiElement(Py:NONE_KEYWORD)('None')
@@ -0,0 +1,15 @@
TypeRepresentation:a.pythonTypeRepresentation
PyElement
PsiElement(Py:AT)('@')
PsiElement(Py:IDENTIFIER)('Todo')
PsiElement(Py:LPAR)('(')
PsiElement(Py:TICK)('`')
PsiElement(Py:IDENTIFIER)('Unpack')
PsiElement(Py:LBRACKET)('[')
PsiElement(Py:RBRACKET)(']')
PsiElement(Py:TICK)('`')
PsiWhiteSpace(' ')
PsiElement(Py:IDENTIFIER)('special')
PsiWhiteSpace(' ')
PsiElement(Py:IDENTIFIER)('form')
PsiElement(Py:RPAR)(')')
@@ -0,0 +1,17 @@
TypeRepresentation:a.pythonTypeRepresentation
PyFunctionTypeRepresentation
PyParameterTypeListRepresentation
PsiElement(Py:LPAR)('(')
PyReferenceExpression: int
PsiElement(Py:IDENTIFIER)('int')
PsiElement(Py:COMMA)(',')
PsiErrorElement:Unexpected token
<empty list>
PsiElement(Py:RPAR)(')')
PsiErrorElement:')' expected
<empty list>
PsiWhiteSpace(' ')
PsiElement(Py:RARROW)('->')
PsiWhiteSpace(' ')
PyNoneLiteralExpression
PsiElement(Py:NONE_KEYWORD)('None')
@@ -0,0 +1,14 @@
TypeRepresentation:a.pythonTypeRepresentation
PyParenthesizedExpression
PsiElement(Py:LPAR)('(')
PsiErrorElement:')' expected
<empty list>
PsiErrorElement:Unexpected token
<empty list>
PsiElement(Py:DEF_KEYWORD)('def')
PsiWhiteSpace(' ')
PyReferenceExpression: foo
PsiElement(Py:IDENTIFIER)('foo')
PsiErrorElement:Unexpected token
<empty list>
PsiElement(Py:RPAR)(')')
@@ -0,0 +1,14 @@
TypeRepresentation:a.pythonTypeRepresentation
PyFunctionTypeRepresentation
PyParameterTypeListRepresentation
PsiElement(Py:LPAR)('(')
PyEllipsisLiteralExpression
PsiElement(Py:DOT)('.')
PsiElement(Py:DOT)('.')
PsiElement(Py:DOT)('.')
PsiElement(Py:RPAR)(')')
PsiWhiteSpace(' ')
PsiElement(Py:RARROW)('->')
PsiWhiteSpace(' ')
PyNoneLiteralExpression
PsiElement(Py:NONE_KEYWORD)('None')
@@ -0,0 +1,2 @@
TypeRepresentation:a.typeRepresentation
<empty list>
@@ -0,0 +1,2 @@
TypeRepresentation:a.pythonTypeRepresentation
<empty list>
@@ -0,0 +1,13 @@
TypeRepresentation:a.pythonTypeRepresentation
PyParenthesizedExpression
PsiElement(Py:LPAR)('(')
PyLambdaExpression
PsiElement(Py:LAMBDA_KEYWORD)('lambda')
PyParameterList
<empty list>
PsiElement(Py:COLON)(':')
PsiWhiteSpace(' ')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('42')
PsiErrorElement:')' expected
<empty list>
@@ -0,0 +1,6 @@
TypeRepresentation:a.pythonTypeRepresentation
PyParenthesizedExpression
PsiElement(Py:LPAR)('(')
PyReferenceExpression: int
PsiElement(Py:IDENTIFIER)('int')
PsiElement(Py:RPAR)(')')
@@ -0,0 +1,7 @@
TypeRepresentation:a.pythonTypeRepresentation
PyParenthesizedExpression
PsiElement(Py:LPAR)('(')
PyReferenceExpression: int
PsiElement(Py:IDENTIFIER)('int')
PsiErrorElement:')' expected
<empty list>
@@ -0,0 +1,10 @@
TypeRepresentation:a.pythonTypeRepresentation
PyFunctionTypeRepresentation
PyParameterTypeListRepresentation
PsiElement(Py:LPAR)('(')
PyReferenceExpression: int
PsiElement(Py:IDENTIFIER)('int')
PsiElement(Py:RPAR)(')')
PsiWhiteSpace(' ')
PsiElement(Py:RARROW)('->')
PsiWhiteSpace(' ')
@@ -0,0 +1,16 @@
TypeRepresentation:a.pythonTypeRepresentation
PyFunctionTypeRepresentation
PyParameterTypeListRepresentation
PsiElement(Py:LPAR)('(')
PyStarExpression
PsiElement(Py:MULT)('*')
PsiErrorElement:Unexpected token
<empty list>
PsiElement(Py:RPAR)(')')
PsiErrorElement:')' expected
<empty list>
PsiWhiteSpace(' ')
PsiElement(Py:RARROW)('->')
PsiWhiteSpace(' ')
PyReferenceExpression: int
PsiElement(Py:IDENTIFIER)('int')
@@ -10,7 +10,6 @@ import com.jetbrains.python.codeInsight.functionTypeComments.psi.PyFunctionTypeA
import com.jetbrains.python.documentation.doctest.PyDocstringTokenSetContributor;
import com.jetbrains.python.psi.PyEllipsisLiteralExpression;
import com.jetbrains.python.psi.PyExpression;
import com.jetbrains.python.psi.PyNoneLiteralExpression;
import com.jetbrains.python.psi.PythonVisitorFilter;
import com.jetbrains.python.psi.impl.PythonASTFactory;
import org.jetbrains.annotations.NotNull;
@@ -30,6 +29,7 @@ public class PyFunctionTypeAnnotationParsingTest extends ParsingTestCase {
@Override
protected void setUp() throws Exception {
super.setUp();
getApplication().registerService(PyElementTypesFacade.class, PyElementTypesFacadeImpl.class);
registerExtensionPoint(PythonDialectsTokenSetContributor.EP_NAME, PythonDialectsTokenSetContributor.class);
registerExtension(PythonDialectsTokenSetContributor.EP_NAME, new PythonTokenSetContributor());
registerExtension(PythonDialectsTokenSetContributor.EP_NAME, new PyDocstringTokenSetContributor());
@@ -0,0 +1,156 @@
// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.jetbrains.python
import com.intellij.lang.LanguageASTFactory
import com.intellij.testFramework.ParsingTestCase
import com.jetbrains.python.codeInsight.typeRepresentation.PyTypeRepresentationDialect
import com.jetbrains.python.codeInsight.typeRepresentation.PyTypeRepresentationParserDefinition
import com.jetbrains.python.codeInsight.typeRepresentation.psi.PyFunctionTypeRepresentation
import com.jetbrains.python.codeInsight.typeRepresentation.psi.PyTypeRepresentationFile
import com.jetbrains.python.documentation.doctest.PyDocstringTokenSetContributor
import com.jetbrains.python.psi.PyEllipsisLiteralExpression
import com.jetbrains.python.psi.PyExpression
import com.jetbrains.python.psi.PythonVisitorFilter
import com.jetbrains.python.psi.impl.PythonASTFactory
import junit.framework.TestCase
import java.io.IOException
import org.junit.jupiter.api.assertInstanceOf as assertInstanceOfJunit5
class PyTypeRepresentationParsingTest : ParsingTestCase("typeRepresentation/parsing", "pythonTypeRepresentation",
PyTypeRepresentationParserDefinition(), PythonParserDefinition()) {
@Throws(Exception::class)
override fun setUp() {
super.setUp()
application.registerService(PyElementTypesFacade::class.java, PyElementTypesFacadeImpl::class.java)
registerExtensionPoint(PythonDialectsTokenSetContributor.EP_NAME,
PythonDialectsTokenSetContributor::class.java)
registerExtension(PythonDialectsTokenSetContributor.EP_NAME, PythonTokenSetContributor())
registerExtension(PythonDialectsTokenSetContributor.EP_NAME, PyDocstringTokenSetContributor())
addExplicitExtension(LanguageASTFactory.INSTANCE, PythonLanguage.getInstance(), PythonASTFactory())
}
@Throws(Exception::class)
override fun tearDown() {
// clear cached extensions
try {
PythonVisitorFilter.INSTANCE.removeExplicitExtension(PythonLanguage.INSTANCE,
PythonVisitorFilter { _, _ -> false })
PythonVisitorFilter.INSTANCE.removeExplicitExtension(PyTypeRepresentationDialect,
PythonVisitorFilter { _, _ -> false })
}
catch (e: Throwable) {
addSuppressedException(e)
}
finally {
super.tearDown()
}
}
override fun doCodeTest(code: String) {
try {
super.doCodeTest(code)
}
catch (e: IOException) {
throw AssertionError(e)
}
}
fun parseCallable(code: String): PyFunctionTypeRepresentation {
doCodeTest(code)
return assertInstanceOfJunit5<PyFunctionTypeRepresentation>(parsedType)
}
private val parsedType: PyExpression? get() = assertInstanceOfJunit5<PyTypeRepresentationFile>(myFile).type
fun `test todo`() {
doCodeTest("@Todo(`Unpack[]` special form)")
}
fun `test todo in callable`() {
val callable = parseCallable("(@Todo(`Unpack[]` special form)) -> None")
assertNotNull(callable)
assertSize(1, callable.parameterList.parameters)
}
fun `test callable empty`() {
val callable = parseCallable("() -> None")
assertEmpty(callable.parameterList.parameters)
val returnType = callable.returnType
assertNotNull(returnType)
assertEquals("None", returnType!!.text)
}
fun `test callable simple`() {
parseCallable("(int, str) -> None")
}
fun `test callable named parameter`() {
parseCallable("(a: int) -> None")
}
fun `test callable default parameter`() {
parseCallable("(a: int = ...) -> None")
}
fun `test callable varargs`() {
parseCallable("(*int, **str) -> None")
}
fun `test callable complex varargs`() {
parseCallable("(*a: *tuple[int, str], **b: **B) -> None")
}
fun `test callable ellipsis`() {
val callable = parseCallable("(...) -> None")
val paramTypes = callable.parameterList.parameters
assertSize(1, paramTypes)
assertInstanceOfJunit5<PyEllipsisLiteralExpression>(paramTypes[0])
val returnType = callable.returnType
assertNotNull(returnType)
}
fun `test callable nested`() {
val callable = parseCallable("(int, (str) -> None, int) -> (str) -> None")
val paramTypes = callable.parameterList.parameters
assertSize(3, paramTypes)
val nestedCallable = assertInstanceOfJunit5<PyFunctionTypeRepresentation>(paramTypes[1])
assertSize(1, nestedCallable.parameterList.parameters)
val nestedReturnType = nestedCallable.returnType
assertNotNull(nestedReturnType)
TestCase.assertEquals(
"None", nestedReturnType!!.text
)
val returnType = assertInstanceOfJunit5<PyFunctionTypeRepresentation>(paramTypes[1])
assertSize(1, returnType.parameterList.parameters)
val returnTypeNestedReturnType = returnType.returnType
assertNotNull(returnTypeNestedReturnType)
TestCase.assertEquals(
"None", nestedReturnType.text
)
}
fun `test callable no return type`() {
val callable = parseCallable("(int) -> ")
val paramTypes = callable.parameterList.parameters
assertSize(1, paramTypes)
assertNull(callable.returnType)
}
fun `test callable dangling comma`() {
parseCallable("(int,) -> None")
}
fun `test callable empty function type`() {
doCodeTest("")
assertNull(parsedType)
}
fun `test callable no type after star`() {
parseCallable("(*) -> int")
}
override fun getTestDataPath(): String {
return PythonTestUtil.getTestDataPath()
}
}