don't add extra whitespace tokens on empty token sequence

GitOrigin-RevId: 40c2d896d320500187fc8c7e3b9305ffe9707c9b

platform/syntax/syntax-tree/src/com/intellij/platform/syntax/tree/Parse.kt

@@ -7,11 +7,9 @@ import com.intellij.platform.syntax.CancellationProvider
 import com.intellij.platform.syntax.LanguageSyntaxDefinition
 import com.intellij.platform.syntax.Logger
 import com.intellij.platform.syntax.SyntaxElementTypeSet
-import com.intellij.platform.syntax.element.SyntaxTokenTypes
 import com.intellij.platform.syntax.extensions.currentExtensionSupport
 import com.intellij.platform.syntax.lexer.Lexer
 import com.intellij.platform.syntax.lexer.TokenList
-import com.intellij.platform.syntax.lexer.buildTokenList
 import com.intellij.platform.syntax.lexer.performLexing
 import com.intellij.platform.syntax.parser.DefaultWhitespaceBindingPolicy
 import com.intellij.platform.syntax.parser.SyntaxTreeBuilder
@@ -64,32 +62,8 @@ fun parse(
     .withStartOffset(startLexemeOffset)
     .build()
 
-  fun performLexingImpl(
-    text: CharSequence,
-    lexer: Lexer,
-    cancellationProvider: CancellationProvider?,
-    logger: Logger?,
-  ): TokenList {
-    val result = performLexing(
-      text,
-      lexer,
-      cancellationProvider,
-      logger,
-    )
-    val isEmpty = result.tokenCount == 0
-    return if (isEmpty) buildTokenList {
-      token("", SyntaxTokenTypes.WHITE_SPACE)
-    }
-    else result
-  }
-
   val lexer = lexerFactory()
-  val tokens = performLexingImpl(
-    text,
-    lexer,
-    cancellationProvider,
-    logger,
-  )
+  val tokens = performLexing(text, lexer, cancellationProvider, logger)
   val builder = createBuilder(text, tokens)
   parser(builder)
   val markers = builder.toAstMarkers()
@@ -99,7 +73,7 @@ fun parse(
     tokens = builder.tokens,
     languageProvider = languageMapper,
     tokenizationPolicy = TokenizationPolicy { text, lexer, cancellation ->
-      performLexingImpl(text, lexer, cancellation, logger)
+      performLexing(text, lexer, cancellation, logger)
     },
     lexer = lexer,
     builderFactory = SyntaxBuilderFactory { text, tokens, startLexeme ->