syntax: rework lazy parseables

GitOrigin-RevId: 2c535eeabeb0f9dcc4cfd49cd80c1eeb52ae2995

java/java-syntax/src/com/intellij/java/syntax/element/JavaCodeBlockParser.kt

@@ -2,65 +2,27 @@
 package com.intellij.java.syntax.element
 
 import com.intellij.java.syntax.parser.JavaParser
-import com.intellij.platform.syntax.CancellationProvider
 import com.intellij.platform.syntax.LazyParser
 import com.intellij.platform.syntax.LazyParsingContext
-import com.intellij.platform.syntax.lexer.TokenList
 import com.intellij.platform.syntax.parser.ProductionResult
 import com.intellij.platform.syntax.parser.prepareProduction
-import com.intellij.platform.syntax.tree.SyntaxNode
-import com.intellij.platform.syntax.util.cancellation.cancellationProvider
 import com.intellij.platform.syntax.util.parser.SyntaxBuilderUtil
-import com.intellij.pom.java.LanguageLevel
 
 internal class JavaCodeBlockParser : LazyParser {
   override fun parse(parsingContext: LazyParsingContext): ProductionResult {
-    return doParse(
-      node = parsingContext.node,
-      cachedLexemes = parsingContext.tokenList,
-      text = parsingContext.text,
-      level = getLanguageLevel(parsingContext),
-      cancellationProvider = cancellationProvider()
-    )
-  }
-
-  override fun tryReparse(parsingContext: LazyParsingContext): ProductionResult? {
-    val cancellationProvider = cancellationProvider()
-
     val level = getLanguageLevel(parsingContext)
-    val tokens = cachedOrLex(
-      cachedLexemes = parsingContext.tokenList,
-      text = parsingContext.text,
-      languageLevel = level,
-      cancellationProvider = cancellationProvider
-    )
-
-    val hasProperBraceBalance = SyntaxBuilderUtil.hasProperBraceBalance(
-      tokenList = tokens,
-      leftBrace = JavaSyntaxTokenType.LBRACE,
-      rightBrace = JavaSyntaxTokenType.RBRACE,
-      cancellationProvider = cancellationProvider
-    )
-    if (!hasProperBraceBalance) return null
-
-    return doParse(
-      node = parsingContext.node,
-      cachedLexemes = tokens,
-      text = parsingContext.text,
-      level = level,
-      cancellationProvider = cancellationProvider
-    )
-  }
-
-  private fun doParse(
-    node: SyntaxNode,
-    cachedLexemes: TokenList? = null,
-    text: CharSequence,
-    level: LanguageLevel,
-    cancellationProvider: CancellationProvider?,
-  ): ProductionResult {
-    val builder = createSyntaxBuilder(node, text, level, cachedLexemes, cancellationProvider)
-    JavaParser(level).statementParser.parseCodeBlockDeep(builder, true)
+    val javaParser = JavaParser(languageLevel = level)
+    val builder = parsingContext.syntaxTreeBuilder
+    javaParser.statementParser.parseCodeBlockDeep(builder, true)
     return prepareProduction(builder)
   }
+
+  override fun canBeReparsedIncrementally(parsingContext: LazyParsingContext): Boolean {
+    return SyntaxBuilderUtil.hasProperBraceBalance(
+      tokenList = parsingContext.tokenList,
+      leftBrace = JavaSyntaxTokenType.LBRACE,
+      rightBrace = JavaSyntaxTokenType.RBRACE,
+      cancellationProvider = parsingContext.cancellationProvider,
+    )
+  }
 }

java/java-syntax/src/com/intellij/java/syntax/element/JavaLanguageLevelProvider.kt

@@ -4,7 +4,7 @@ package com.intellij.java.syntax.element
 import com.intellij.platform.syntax.LazyParsingContext
 import com.intellij.platform.syntax.extensions.ExtensionPointKey
 import com.intellij.platform.syntax.extensions.ExtensionSupport
-import com.intellij.platform.syntax.tree.SyntaxTree
+import com.intellij.platform.syntax.tree.SyntaxNode
 import com.intellij.pom.java.LanguageLevel
 import org.jetbrains.annotations.ApiStatus
 
@@ -13,12 +13,12 @@ import org.jetbrains.annotations.ApiStatus
  */
 @ApiStatus.OverrideOnly
 interface JavaLanguageLevelProvider {
-  fun getLanguageLevel(syntaxTree: SyntaxTree): LanguageLevel
+  fun getLanguageLevel(node: SyntaxNode): LanguageLevel
 }
 
 internal fun getLanguageLevel(parsingContext: LazyParsingContext): LanguageLevel {
   val languageLevelProvider = ExtensionSupport().getExtensions(languageLevelExtensionPoint).firstOrNull()
-  val languageLevel = languageLevelProvider?.getLanguageLevel(parsingContext.tree) ?: LanguageLevel.HIGHEST
+  val languageLevel = languageLevelProvider?.getLanguageLevel(parsingContext.node) ?: LanguageLevel.HIGHEST
   return languageLevel
 }
 

java/java-syntax/src/com/intellij/java/syntax/element/JavaSyntaxUtil.kt

@@ -1,50 +0,0 @@
-// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
-@file:JvmName("JavaSyntaxUtil")
-
-package com.intellij.java.syntax.element
-
-import com.intellij.java.syntax.JavaSyntaxDefinition
-import com.intellij.java.syntax.JavaSyntaxLog
-import com.intellij.platform.syntax.CancellationProvider
-import com.intellij.platform.syntax.lexer.TokenList
-import com.intellij.platform.syntax.lexer.performLexing
-import com.intellij.platform.syntax.parser.SyntaxTreeBuilder
-import com.intellij.platform.syntax.parser.SyntaxTreeBuilderFactory
-import com.intellij.platform.syntax.tree.SyntaxNode
-import com.intellij.pom.java.LanguageLevel
-import kotlin.jvm.JvmName
-
-internal fun createSyntaxBuilder(
-  chameleon: SyntaxNode,
-  text: CharSequence,
-  languageLevel: LanguageLevel,
-  cachedLexemes: TokenList? = null,
-  cancellationProvider: CancellationProvider?,
-): SyntaxTreeBuilder {
-  val lexemes = cachedOrLex(cachedLexemes, text, languageLevel, cancellationProvider)
-  val builder = SyntaxTreeBuilderFactory.builder(
-    text = text,
-    whitespaces = JavaSyntaxDefinition.whitespaces,
-    comments = JavaSyntaxDefinition.commentSet,
-    tokenList = lexemes,
-  )
-    .withStartOffset(chameleon.startOffset)
-    .withLanguage(JavaSyntaxDefinition.language.id)
-    .withWhitespaceOrCommentBindingPolicy(JavaBindingPolicy)
-    .withCancellationProvider(cancellationProvider)
-    .withLogger(JavaSyntaxLog.log)
-
-  return builder.build()
-}
-
-internal fun cachedOrLex(
-  cachedLexemes: TokenList?,
-  text: CharSequence,
-  languageLevel: LanguageLevel,
-  cancellationProvider: CancellationProvider?,
-): TokenList {
-  return cachedLexemes ?: run {
-    val lexer = JavaSyntaxDefinition.createLexer(languageLevel)
-    performLexing(text, lexer, cancellationProvider, JavaSyntaxLog.log)
-  }
-}

platform/syntax/syntax-api/api-dump-experimental.txt

@@ -1,17 +1,23 @@
 *:com.intellij.platform.syntax.CancellationProvider
 - a:checkCancelled():V
-*:com.intellij.platform.syntax.LazyParser
-- a:parse(com.intellij.platform.syntax.LazyParsingContext):com.intellij.platform.syntax.parser.ProductionResult
-- tryReparse(com.intellij.platform.syntax.LazyParsingContext):com.intellij.platform.syntax.parser.ProductionResult
-*f:com.intellij.platform.syntax.LazyParserKt
-- *sf:parseLazyNode(com.intellij.platform.syntax.LazyParsingContext):com.intellij.platform.syntax.parser.ProductionResult
-- *sf:tryReparseLazyNode(com.intellij.platform.syntax.LazyParsingContext):com.intellij.platform.syntax.parser.ProductionResult
-*f:com.intellij.platform.syntax.LazyParsingContext
-- <init>(com.intellij.platform.syntax.tree.SyntaxTree,com.intellij.platform.syntax.tree.SyntaxNode,java.lang.CharSequence,com.intellij.platform.syntax.lexer.TokenList):V
+*f:com.intellij.platform.syntax.LazyLexingContext
+- <init>(com.intellij.platform.syntax.tree.SyntaxNode,com.intellij.platform.syntax.CancellationProvider):V
+- f:getCancellationProvider():com.intellij.platform.syntax.CancellationProvider
 - f:getNode():com.intellij.platform.syntax.tree.SyntaxNode
+*:com.intellij.platform.syntax.LazyParser
+- canBeReparsedIncrementally(com.intellij.platform.syntax.LazyParsingContext):Z
+- createLexer(com.intellij.platform.syntax.LazyLexingContext):com.intellij.platform.syntax.lexer.Lexer
+- a:parse(com.intellij.platform.syntax.LazyParsingContext):com.intellij.platform.syntax.parser.ProductionResult
+*f:com.intellij.platform.syntax.LazyParserKt
+- *sf:canLazyNodeBeReparsedIncrementally(com.intellij.platform.syntax.LazyParsingContext):Z
+- *sf:parseLazyNode(com.intellij.platform.syntax.LazyParsingContext):com.intellij.platform.syntax.parser.ProductionResult
+*f:com.intellij.platform.syntax.LazyParsingContext
+- <init>(com.intellij.platform.syntax.tree.SyntaxNode,com.intellij.platform.syntax.lexer.TokenList,com.intellij.platform.syntax.parser.SyntaxTreeBuilder,com.intellij.platform.syntax.CancellationProvider):V
+- f:getCancellationProvider():com.intellij.platform.syntax.CancellationProvider
+- f:getNode():com.intellij.platform.syntax.tree.SyntaxNode
+- f:getSyntaxTreeBuilder():com.intellij.platform.syntax.parser.SyntaxTreeBuilder
 - f:getText():java.lang.CharSequence
 - f:getTokenList():com.intellij.platform.syntax.lexer.TokenList
-- f:getTree():com.intellij.platform.syntax.tree.SyntaxTree
 *:com.intellij.platform.syntax.Logger
 - a:debug(java.lang.String,java.lang.Throwable):V
 - bs:debug$default(com.intellij.platform.syntax.Logger,java.lang.String,java.lang.Throwable,I,java.lang.Object):V
@@ -210,6 +216,7 @@
 - a:getEndOffset():I
 - a:getErrorMessage():java.lang.String
 - a:getFirstChild():com.intellij.platform.syntax.tree.SyntaxNode
+- a:getLanguage():java.lang.Object
 - a:getLastChild():com.intellij.platform.syntax.tree.SyntaxNode
 - a:getNextSibling():com.intellij.platform.syntax.tree.SyntaxNode
 - a:getParent():com.intellij.platform.syntax.tree.SyntaxNode
@@ -217,6 +224,3 @@
 - a:getStartOffset():I
 - a:getText():java.lang.CharSequence
 - a:getType():com.intellij.platform.syntax.SyntaxElementType
-*:com.intellij.platform.syntax.tree.SyntaxTree
-- a:getRoot():com.intellij.platform.syntax.tree.SyntaxNode
-- a:getText():java.lang.CharSequence

platform/syntax/syntax-api/src/com/intellij/platform/syntax/LazyParser.kt

@@ -3,24 +3,28 @@
 
 package com.intellij.platform.syntax
 
+import com.intellij.platform.syntax.lexer.Lexer
 import com.intellij.platform.syntax.lexer.TokenList
 import com.intellij.platform.syntax.parser.ProductionResult
+import com.intellij.platform.syntax.parser.SyntaxTreeBuilder
 import com.intellij.platform.syntax.tree.SyntaxNode
-import com.intellij.platform.syntax.tree.SyntaxTree
 import org.jetbrains.annotations.ApiStatus
 
 /**
- * A parser that is attached to so-called chameleon nodes which allows to parse them lazily on demand.
+ * A parser that is attached to chameleon (lazy parseable) nodes which allows to parse them lazily on demand.
+ * This parser also allows reparsing the node incrementally.
+ *
+ * It is guaranteed that the passed node has the element type corresponding to the lazy parser.
  *
  * ### Implementation note:
  *
- * Provided [SyntaxTree] and [SyntaxNode] might be backed by different tree implementations,
+ * Provided [SyntaxNode] might be backed by different tree implementations,
  * depending on the syntax-lib client. So please don't make any assumptions on the actual types of the passed instances.
  *
  * If you want to add platform-specific code, introduce an extension point, see [com.intellij.platform.syntax.extensions.ExtensionSupport].
  *
  * @see parseLazyNode
- * @see tryReparseLazyNode
+ * @see canBeReparsedIncrementally
  */
 @ApiStatus.Experimental
 @ApiStatus.OverrideOnly
@@ -33,12 +37,23 @@ interface LazyParser {
   fun parse(parsingContext: LazyParsingContext): ProductionResult
 
   /**
-   * Called when the node is requested to be reparsed.
+   * Called when the node is requested to be reparsed incrementally.
+   * The provided [parsingContext] contains a token list corresponding to the new node's text.
+   * The method should decide if the new token list can be used as a valid input for the parser.
    *
-   * @return the result of the parsing operation or `null` if reparsing is not possible
-   *         (e.g., when braces got unbalanced in the next)
+   * An example:
+   * This lazy parser corresponds to a code-block in Java. The method should check that the provided token list
+   * represents a valid brace structure, i.e. the braces are matched and there are no extra tokens before the opening brace and
+   * after the closing brace.
+   *
+   * @return true if the new token list can be used as a valid input for the parser, false otherwise.
    */
-  fun tryReparse(parsingContext: LazyParsingContext): ProductionResult? = null
+  fun canBeReparsedIncrementally(parsingContext: LazyParsingContext): Boolean = false
+
+  /**
+   * Creates a lexer for the given node.
+   */
+  fun createLexer(lexingContext: LazyLexingContext): Lexer? = null
 }
 
 /**
@@ -52,28 +67,46 @@ fun parseLazyNode(parsingContext: LazyParsingContext): ProductionResult {
 }
 
 /**
- * Tries to reparse the given node and returns [ProductionResult] if possible.
+ * Checks if the given node can be reparsed incrementally.
  *
- * @see LazyParser.tryReparse
+ * @see LazyParser.canBeReparsedIncrementally
  */
 @ApiStatus.Experimental
-fun tryReparseLazyNode(parsingContext: LazyParsingContext): ProductionResult? {
-  return parsingContext.lazyParser.tryReparse(parsingContext)
+fun canLazyNodeBeReparsedIncrementally(parsingContext: LazyParsingContext): Boolean {
+  return parsingContext.lazyParser.canBeReparsedIncrementally(parsingContext)
 }
 
 /**
- * @param tree the tree being parsed
  * @param node the node being parsed
- * @param text the text of the node being parsed
  * @param tokenList the token list being parsed. Might be missing if the parsing engine does not store this information.
+ * @param syntaxTreeBuilder a syntax tree builder for the node to be parsed
+ * @param cancellationProvider a cancellation provider for the parser
  */
 @ApiStatus.Experimental
 class LazyParsingContext(
-  val tree: SyntaxTree,
   val node: SyntaxNode,
-  val text: CharSequence,
-  val tokenList: TokenList?,
+  val tokenList: TokenList,
+  val syntaxTreeBuilder: SyntaxTreeBuilder,
+  val cancellationProvider: CancellationProvider,
 ) {
+  /**
+   * text of the node to be reparsed
+   */
+  val text: CharSequence get() = node.text
+
+  /**
+   * parser for the node to be reparsed
+   */
   internal val lazyParser: LazyParser
     get() = node.type.lazyParser ?: error("Node ${node} has non-lazy element type ${node.type}")
-}
+}
+
+/**
+ * @param node the node being lexed
+ * @param cancellationProvider a cancellation provider for the lexer
+ */
+@ApiStatus.Experimental
+class LazyLexingContext(
+  val node: SyntaxNode,
+  val cancellationProvider: CancellationProvider,
+)

platform/syntax/syntax-api/src/com/intellij/platform/syntax/SyntaxElementType.kt

@@ -25,7 +25,7 @@ class SyntaxElementType internal constructor(
 
   /**
    * Checks if this element type is lazy-parseable.
-   * For performing reparse, use [parseLazyNode] and [tryReparseLazyNode] functions.
+   * For performing reparse, use [parseLazyNode] and [canLazyNodeBeReparsedIncrementally] functions.
    *
    * @return `true` if this element type is lazy-parseable.
    */

platform/syntax/syntax-api/src/com/intellij/platform/syntax/tree/SyntaxTree.kt

@@ -4,17 +4,6 @@ package com.intellij.platform.syntax.tree
 import com.intellij.platform.syntax.SyntaxElementType
 import org.jetbrains.annotations.ApiStatus
 
-/**
- * API for a Syntax Tree.
- *
- * The tree is passed to [com.intellij.platform.syntax.LazyParser] as the context for parsing.
- */
-@ApiStatus.Experimental
-interface SyntaxTree {
-  val text: CharSequence
-  val root: SyntaxNode
-}
-
 /**
  * API for a Syntax Node.
  *
@@ -37,4 +26,6 @@ interface SyntaxNode {
   val lastChild: SyntaxNode?
 
   val errorMessage: String?
+
+  val language: Any?
 }