From b95ef424c1a4ef0636a1482146c6ed19f8bc6690 Mon Sep 17 00:00:00 2001
From: Max Medvedev <maxim.medvedev@jetbrains.com>
Date: Sat, 23 Aug 2025 17:23:04 +0200
Subject: [PATCH] don't add extra whitespace tokens on empty token sequence

GitOrigin-RevId: 40c2d896d320500187fc8c7e3b9305ffe9707c9b
---
 .../intellij/platform/syntax/tree/Parse.kt    | 30 ++-----------------
 1 file changed, 2 insertions(+), 28 deletions(-)

diff --git a/platform/syntax/syntax-tree/src/com/intellij/platform/syntax/tree/Parse.kt b/platform/syntax/syntax-tree/src/com/intellij/platform/syntax/tree/Parse.kt
index 816c01c17b83..1e7f1973075d 100644
--- a/platform/syntax/syntax-tree/src/com/intellij/platform/syntax/tree/Parse.kt
+++ b/platform/syntax/syntax-tree/src/com/intellij/platform/syntax/tree/Parse.kt
@@ -7,11 +7,9 @@ import com.intellij.platform.syntax.CancellationProvider
 import com.intellij.platform.syntax.LanguageSyntaxDefinition
 import com.intellij.platform.syntax.Logger
 import com.intellij.platform.syntax.SyntaxElementTypeSet
-import com.intellij.platform.syntax.element.SyntaxTokenTypes
 import com.intellij.platform.syntax.extensions.currentExtensionSupport
 import com.intellij.platform.syntax.lexer.Lexer
 import com.intellij.platform.syntax.lexer.TokenList
-import com.intellij.platform.syntax.lexer.buildTokenList
 import com.intellij.platform.syntax.lexer.performLexing
 import com.intellij.platform.syntax.parser.DefaultWhitespaceBindingPolicy
 import com.intellij.platform.syntax.parser.SyntaxTreeBuilder
@@ -64,32 +62,8 @@ fun parse(
     .withStartOffset(startLexemeOffset)
     .build()
 
-  fun performLexingImpl(
-    text: CharSequence,
-    lexer: Lexer,
-    cancellationProvider: CancellationProvider?,
-    logger: Logger?,
-  ): TokenList {
-    val result = performLexing(
-      text,
-      lexer,
-      cancellationProvider,
-      logger,
-    )
-    val isEmpty = result.tokenCount == 0
-    return if (isEmpty) buildTokenList {
-      token("", SyntaxTokenTypes.WHITE_SPACE)
-    }
-    else result
-  }
-
   val lexer = lexerFactory()
-  val tokens = performLexingImpl(
-    text,
-    lexer,
-    cancellationProvider,
-    logger,
-  )
+  val tokens = performLexing(text, lexer, cancellationProvider, logger)
   val builder = createBuilder(text, tokens)
   parser(builder)
   val markers = builder.toAstMarkers()
@@ -99,7 +73,7 @@ fun parse(
     tokens = builder.tokens,
     languageProvider = languageMapper,
     tokenizationPolicy = TokenizationPolicy { text, lexer, cancellation ->
-      performLexingImpl(text, lexer, cancellation, logger)
+      performLexing(text, lexer, cancellation, logger)
     },
     lexer = lexer,
     builderFactory = SyntaxBuilderFactory { text, tokens, startLexeme ->