From 4d84bd6b7578f6e29fa08d6c2ddc4fdb0fab685b Mon Sep 17 00:00:00 2001 From: Max Medvedev Date: Wed, 28 May 2025 12:20:45 +0200 Subject: [PATCH] syntax: rework lazy parseables GitOrigin-RevId: 2c535eeabeb0f9dcc4cfd49cd80c1eeb52ae2995 --- .../syntax/element/JavaCodeBlockParser.kt | 62 ++++------------- .../element/JavaLanguageLevelProvider.kt | 6 +- .../java/syntax/element/JavaSyntaxUtil.kt | 50 -------------- .../syntax-api/api-dump-experimental.txt | 28 ++++---- .../intellij/platform/syntax/LazyParser.kt | 69 ++++++++++++++----- .../platform/syntax/SyntaxElementType.kt | 2 +- .../platform/syntax/tree/SyntaxTree.kt | 13 +--- 7 files changed, 85 insertions(+), 145 deletions(-) delete mode 100644 java/java-syntax/src/com/intellij/java/syntax/element/JavaSyntaxUtil.kt diff --git a/java/java-syntax/src/com/intellij/java/syntax/element/JavaCodeBlockParser.kt b/java/java-syntax/src/com/intellij/java/syntax/element/JavaCodeBlockParser.kt index 0b573a8586fb..f889a315425e 100644 --- a/java/java-syntax/src/com/intellij/java/syntax/element/JavaCodeBlockParser.kt +++ b/java/java-syntax/src/com/intellij/java/syntax/element/JavaCodeBlockParser.kt @@ -2,65 +2,27 @@ package com.intellij.java.syntax.element import com.intellij.java.syntax.parser.JavaParser -import com.intellij.platform.syntax.CancellationProvider import com.intellij.platform.syntax.LazyParser import com.intellij.platform.syntax.LazyParsingContext -import com.intellij.platform.syntax.lexer.TokenList import com.intellij.platform.syntax.parser.ProductionResult import com.intellij.platform.syntax.parser.prepareProduction -import com.intellij.platform.syntax.tree.SyntaxNode -import com.intellij.platform.syntax.util.cancellation.cancellationProvider import com.intellij.platform.syntax.util.parser.SyntaxBuilderUtil -import com.intellij.pom.java.LanguageLevel internal class JavaCodeBlockParser : LazyParser { override fun parse(parsingContext: LazyParsingContext): ProductionResult { - return doParse( - node = parsingContext.node, - cachedLexemes = parsingContext.tokenList, - text = parsingContext.text, - level = getLanguageLevel(parsingContext), - cancellationProvider = cancellationProvider() - ) - } - - override fun tryReparse(parsingContext: LazyParsingContext): ProductionResult? { - val cancellationProvider = cancellationProvider() - val level = getLanguageLevel(parsingContext) - val tokens = cachedOrLex( - cachedLexemes = parsingContext.tokenList, - text = parsingContext.text, - languageLevel = level, - cancellationProvider = cancellationProvider - ) - - val hasProperBraceBalance = SyntaxBuilderUtil.hasProperBraceBalance( - tokenList = tokens, - leftBrace = JavaSyntaxTokenType.LBRACE, - rightBrace = JavaSyntaxTokenType.RBRACE, - cancellationProvider = cancellationProvider - ) - if (!hasProperBraceBalance) return null - - return doParse( - node = parsingContext.node, - cachedLexemes = tokens, - text = parsingContext.text, - level = level, - cancellationProvider = cancellationProvider - ) - } - - private fun doParse( - node: SyntaxNode, - cachedLexemes: TokenList? = null, - text: CharSequence, - level: LanguageLevel, - cancellationProvider: CancellationProvider?, - ): ProductionResult { - val builder = createSyntaxBuilder(node, text, level, cachedLexemes, cancellationProvider) - JavaParser(level).statementParser.parseCodeBlockDeep(builder, true) + val javaParser = JavaParser(languageLevel = level) + val builder = parsingContext.syntaxTreeBuilder + javaParser.statementParser.parseCodeBlockDeep(builder, true) return prepareProduction(builder) } + + override fun canBeReparsedIncrementally(parsingContext: LazyParsingContext): Boolean { + return SyntaxBuilderUtil.hasProperBraceBalance( + tokenList = parsingContext.tokenList, + leftBrace = JavaSyntaxTokenType.LBRACE, + rightBrace = JavaSyntaxTokenType.RBRACE, + cancellationProvider = parsingContext.cancellationProvider, + ) + } } \ No newline at end of file diff --git a/java/java-syntax/src/com/intellij/java/syntax/element/JavaLanguageLevelProvider.kt b/java/java-syntax/src/com/intellij/java/syntax/element/JavaLanguageLevelProvider.kt index f39f8f441650..f2fe6d8684cd 100644 --- a/java/java-syntax/src/com/intellij/java/syntax/element/JavaLanguageLevelProvider.kt +++ b/java/java-syntax/src/com/intellij/java/syntax/element/JavaLanguageLevelProvider.kt @@ -4,7 +4,7 @@ package com.intellij.java.syntax.element import com.intellij.platform.syntax.LazyParsingContext import com.intellij.platform.syntax.extensions.ExtensionPointKey import com.intellij.platform.syntax.extensions.ExtensionSupport -import com.intellij.platform.syntax.tree.SyntaxTree +import com.intellij.platform.syntax.tree.SyntaxNode import com.intellij.pom.java.LanguageLevel import org.jetbrains.annotations.ApiStatus @@ -13,12 +13,12 @@ import org.jetbrains.annotations.ApiStatus */ @ApiStatus.OverrideOnly interface JavaLanguageLevelProvider { - fun getLanguageLevel(syntaxTree: SyntaxTree): LanguageLevel + fun getLanguageLevel(node: SyntaxNode): LanguageLevel } internal fun getLanguageLevel(parsingContext: LazyParsingContext): LanguageLevel { val languageLevelProvider = ExtensionSupport().getExtensions(languageLevelExtensionPoint).firstOrNull() - val languageLevel = languageLevelProvider?.getLanguageLevel(parsingContext.tree) ?: LanguageLevel.HIGHEST + val languageLevel = languageLevelProvider?.getLanguageLevel(parsingContext.node) ?: LanguageLevel.HIGHEST return languageLevel } diff --git a/java/java-syntax/src/com/intellij/java/syntax/element/JavaSyntaxUtil.kt b/java/java-syntax/src/com/intellij/java/syntax/element/JavaSyntaxUtil.kt deleted file mode 100644 index d8b7d6cef63a..000000000000 --- a/java/java-syntax/src/com/intellij/java/syntax/element/JavaSyntaxUtil.kt +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. -@file:JvmName("JavaSyntaxUtil") - -package com.intellij.java.syntax.element - -import com.intellij.java.syntax.JavaSyntaxDefinition -import com.intellij.java.syntax.JavaSyntaxLog -import com.intellij.platform.syntax.CancellationProvider -import com.intellij.platform.syntax.lexer.TokenList -import com.intellij.platform.syntax.lexer.performLexing -import com.intellij.platform.syntax.parser.SyntaxTreeBuilder -import com.intellij.platform.syntax.parser.SyntaxTreeBuilderFactory -import com.intellij.platform.syntax.tree.SyntaxNode -import com.intellij.pom.java.LanguageLevel -import kotlin.jvm.JvmName - -internal fun createSyntaxBuilder( - chameleon: SyntaxNode, - text: CharSequence, - languageLevel: LanguageLevel, - cachedLexemes: TokenList? = null, - cancellationProvider: CancellationProvider?, -): SyntaxTreeBuilder { - val lexemes = cachedOrLex(cachedLexemes, text, languageLevel, cancellationProvider) - val builder = SyntaxTreeBuilderFactory.builder( - text = text, - whitespaces = JavaSyntaxDefinition.whitespaces, - comments = JavaSyntaxDefinition.commentSet, - tokenList = lexemes, - ) - .withStartOffset(chameleon.startOffset) - .withLanguage(JavaSyntaxDefinition.language.id) - .withWhitespaceOrCommentBindingPolicy(JavaBindingPolicy) - .withCancellationProvider(cancellationProvider) - .withLogger(JavaSyntaxLog.log) - - return builder.build() -} - -internal fun cachedOrLex( - cachedLexemes: TokenList?, - text: CharSequence, - languageLevel: LanguageLevel, - cancellationProvider: CancellationProvider?, -): TokenList { - return cachedLexemes ?: run { - val lexer = JavaSyntaxDefinition.createLexer(languageLevel) - performLexing(text, lexer, cancellationProvider, JavaSyntaxLog.log) - } -} diff --git a/platform/syntax/syntax-api/api-dump-experimental.txt b/platform/syntax/syntax-api/api-dump-experimental.txt index 9ce323c1515a..52dcfa9315bb 100644 --- a/platform/syntax/syntax-api/api-dump-experimental.txt +++ b/platform/syntax/syntax-api/api-dump-experimental.txt @@ -1,17 +1,23 @@ *:com.intellij.platform.syntax.CancellationProvider - a:checkCancelled():V -*:com.intellij.platform.syntax.LazyParser -- a:parse(com.intellij.platform.syntax.LazyParsingContext):com.intellij.platform.syntax.parser.ProductionResult -- tryReparse(com.intellij.platform.syntax.LazyParsingContext):com.intellij.platform.syntax.parser.ProductionResult -*f:com.intellij.platform.syntax.LazyParserKt -- *sf:parseLazyNode(com.intellij.platform.syntax.LazyParsingContext):com.intellij.platform.syntax.parser.ProductionResult -- *sf:tryReparseLazyNode(com.intellij.platform.syntax.LazyParsingContext):com.intellij.platform.syntax.parser.ProductionResult -*f:com.intellij.platform.syntax.LazyParsingContext -- (com.intellij.platform.syntax.tree.SyntaxTree,com.intellij.platform.syntax.tree.SyntaxNode,java.lang.CharSequence,com.intellij.platform.syntax.lexer.TokenList):V +*f:com.intellij.platform.syntax.LazyLexingContext +- (com.intellij.platform.syntax.tree.SyntaxNode,com.intellij.platform.syntax.CancellationProvider):V +- f:getCancellationProvider():com.intellij.platform.syntax.CancellationProvider - f:getNode():com.intellij.platform.syntax.tree.SyntaxNode +*:com.intellij.platform.syntax.LazyParser +- canBeReparsedIncrementally(com.intellij.platform.syntax.LazyParsingContext):Z +- createLexer(com.intellij.platform.syntax.LazyLexingContext):com.intellij.platform.syntax.lexer.Lexer +- a:parse(com.intellij.platform.syntax.LazyParsingContext):com.intellij.platform.syntax.parser.ProductionResult +*f:com.intellij.platform.syntax.LazyParserKt +- *sf:canLazyNodeBeReparsedIncrementally(com.intellij.platform.syntax.LazyParsingContext):Z +- *sf:parseLazyNode(com.intellij.platform.syntax.LazyParsingContext):com.intellij.platform.syntax.parser.ProductionResult +*f:com.intellij.platform.syntax.LazyParsingContext +- (com.intellij.platform.syntax.tree.SyntaxNode,com.intellij.platform.syntax.lexer.TokenList,com.intellij.platform.syntax.parser.SyntaxTreeBuilder,com.intellij.platform.syntax.CancellationProvider):V +- f:getCancellationProvider():com.intellij.platform.syntax.CancellationProvider +- f:getNode():com.intellij.platform.syntax.tree.SyntaxNode +- f:getSyntaxTreeBuilder():com.intellij.platform.syntax.parser.SyntaxTreeBuilder - f:getText():java.lang.CharSequence - f:getTokenList():com.intellij.platform.syntax.lexer.TokenList -- f:getTree():com.intellij.platform.syntax.tree.SyntaxTree *:com.intellij.platform.syntax.Logger - a:debug(java.lang.String,java.lang.Throwable):V - bs:debug$default(com.intellij.platform.syntax.Logger,java.lang.String,java.lang.Throwable,I,java.lang.Object):V @@ -210,6 +216,7 @@ - a:getEndOffset():I - a:getErrorMessage():java.lang.String - a:getFirstChild():com.intellij.platform.syntax.tree.SyntaxNode +- a:getLanguage():java.lang.Object - a:getLastChild():com.intellij.platform.syntax.tree.SyntaxNode - a:getNextSibling():com.intellij.platform.syntax.tree.SyntaxNode - a:getParent():com.intellij.platform.syntax.tree.SyntaxNode @@ -217,6 +224,3 @@ - a:getStartOffset():I - a:getText():java.lang.CharSequence - a:getType():com.intellij.platform.syntax.SyntaxElementType -*:com.intellij.platform.syntax.tree.SyntaxTree -- a:getRoot():com.intellij.platform.syntax.tree.SyntaxNode -- a:getText():java.lang.CharSequence diff --git a/platform/syntax/syntax-api/src/com/intellij/platform/syntax/LazyParser.kt b/platform/syntax/syntax-api/src/com/intellij/platform/syntax/LazyParser.kt index 3c96370fd359..47dc0fa0c265 100644 --- a/platform/syntax/syntax-api/src/com/intellij/platform/syntax/LazyParser.kt +++ b/platform/syntax/syntax-api/src/com/intellij/platform/syntax/LazyParser.kt @@ -3,24 +3,28 @@ package com.intellij.platform.syntax +import com.intellij.platform.syntax.lexer.Lexer import com.intellij.platform.syntax.lexer.TokenList import com.intellij.platform.syntax.parser.ProductionResult +import com.intellij.platform.syntax.parser.SyntaxTreeBuilder import com.intellij.platform.syntax.tree.SyntaxNode -import com.intellij.platform.syntax.tree.SyntaxTree import org.jetbrains.annotations.ApiStatus /** - * A parser that is attached to so-called chameleon nodes which allows to parse them lazily on demand. + * A parser that is attached to chameleon (lazy parseable) nodes which allows to parse them lazily on demand. + * This parser also allows reparsing the node incrementally. + * + * It is guaranteed that the passed node has the element type corresponding to the lazy parser. * * ### Implementation note: * - * Provided [SyntaxTree] and [SyntaxNode] might be backed by different tree implementations, + * Provided [SyntaxNode] might be backed by different tree implementations, * depending on the syntax-lib client. So please don't make any assumptions on the actual types of the passed instances. * * If you want to add platform-specific code, introduce an extension point, see [com.intellij.platform.syntax.extensions.ExtensionSupport]. * * @see parseLazyNode - * @see tryReparseLazyNode + * @see canBeReparsedIncrementally */ @ApiStatus.Experimental @ApiStatus.OverrideOnly @@ -33,12 +37,23 @@ interface LazyParser { fun parse(parsingContext: LazyParsingContext): ProductionResult /** - * Called when the node is requested to be reparsed. + * Called when the node is requested to be reparsed incrementally. + * The provided [parsingContext] contains a token list corresponding to the new node's text. + * The method should decide if the new token list can be used as a valid input for the parser. * - * @return the result of the parsing operation or `null` if reparsing is not possible - * (e.g., when braces got unbalanced in the next) + * An example: + * This lazy parser corresponds to a code-block in Java. The method should check that the provided token list + * represents a valid brace structure, i.e. the braces are matched and there are no extra tokens before the opening brace and + * after the closing brace. + * + * @return true if the new token list can be used as a valid input for the parser, false otherwise. */ - fun tryReparse(parsingContext: LazyParsingContext): ProductionResult? = null + fun canBeReparsedIncrementally(parsingContext: LazyParsingContext): Boolean = false + + /** + * Creates a lexer for the given node. + */ + fun createLexer(lexingContext: LazyLexingContext): Lexer? = null } /** @@ -52,28 +67,46 @@ fun parseLazyNode(parsingContext: LazyParsingContext): ProductionResult { } /** - * Tries to reparse the given node and returns [ProductionResult] if possible. + * Checks if the given node can be reparsed incrementally. * - * @see LazyParser.tryReparse + * @see LazyParser.canBeReparsedIncrementally */ @ApiStatus.Experimental -fun tryReparseLazyNode(parsingContext: LazyParsingContext): ProductionResult? { - return parsingContext.lazyParser.tryReparse(parsingContext) +fun canLazyNodeBeReparsedIncrementally(parsingContext: LazyParsingContext): Boolean { + return parsingContext.lazyParser.canBeReparsedIncrementally(parsingContext) } /** - * @param tree the tree being parsed * @param node the node being parsed - * @param text the text of the node being parsed * @param tokenList the token list being parsed. Might be missing if the parsing engine does not store this information. + * @param syntaxTreeBuilder a syntax tree builder for the node to be parsed + * @param cancellationProvider a cancellation provider for the parser */ @ApiStatus.Experimental class LazyParsingContext( - val tree: SyntaxTree, val node: SyntaxNode, - val text: CharSequence, - val tokenList: TokenList?, + val tokenList: TokenList, + val syntaxTreeBuilder: SyntaxTreeBuilder, + val cancellationProvider: CancellationProvider, ) { + /** + * text of the node to be reparsed + */ + val text: CharSequence get() = node.text + + /** + * parser for the node to be reparsed + */ internal val lazyParser: LazyParser get() = node.type.lazyParser ?: error("Node ${node} has non-lazy element type ${node.type}") -} \ No newline at end of file +} + +/** + * @param node the node being lexed + * @param cancellationProvider a cancellation provider for the lexer + */ +@ApiStatus.Experimental +class LazyLexingContext( + val node: SyntaxNode, + val cancellationProvider: CancellationProvider, +) diff --git a/platform/syntax/syntax-api/src/com/intellij/platform/syntax/SyntaxElementType.kt b/platform/syntax/syntax-api/src/com/intellij/platform/syntax/SyntaxElementType.kt index 0cf7e21b6f7a..76947f26849c 100644 --- a/platform/syntax/syntax-api/src/com/intellij/platform/syntax/SyntaxElementType.kt +++ b/platform/syntax/syntax-api/src/com/intellij/platform/syntax/SyntaxElementType.kt @@ -25,7 +25,7 @@ class SyntaxElementType internal constructor( /** * Checks if this element type is lazy-parseable. - * For performing reparse, use [parseLazyNode] and [tryReparseLazyNode] functions. + * For performing reparse, use [parseLazyNode] and [canLazyNodeBeReparsedIncrementally] functions. * * @return `true` if this element type is lazy-parseable. */ diff --git a/platform/syntax/syntax-api/src/com/intellij/platform/syntax/tree/SyntaxTree.kt b/platform/syntax/syntax-api/src/com/intellij/platform/syntax/tree/SyntaxTree.kt index eeeed8e3876f..fe92c89cea08 100644 --- a/platform/syntax/syntax-api/src/com/intellij/platform/syntax/tree/SyntaxTree.kt +++ b/platform/syntax/syntax-api/src/com/intellij/platform/syntax/tree/SyntaxTree.kt @@ -4,17 +4,6 @@ package com.intellij.platform.syntax.tree import com.intellij.platform.syntax.SyntaxElementType import org.jetbrains.annotations.ApiStatus -/** - * API for a Syntax Tree. - * - * The tree is passed to [com.intellij.platform.syntax.LazyParser] as the context for parsing. - */ -@ApiStatus.Experimental -interface SyntaxTree { - val text: CharSequence - val root: SyntaxNode -} - /** * API for a Syntax Node. * @@ -37,4 +26,6 @@ interface SyntaxNode { val lastChild: SyntaxNode? val errorMessage: String? + + val language: Any? }