diff --git a/platform/syntax/syntax-tree/src/com/intellij/platform/syntax/tree/Parse.kt b/platform/syntax/syntax-tree/src/com/intellij/platform/syntax/tree/Parse.kt index 816c01c17b83..1e7f1973075d 100644 --- a/platform/syntax/syntax-tree/src/com/intellij/platform/syntax/tree/Parse.kt +++ b/platform/syntax/syntax-tree/src/com/intellij/platform/syntax/tree/Parse.kt @@ -7,11 +7,9 @@ import com.intellij.platform.syntax.CancellationProvider import com.intellij.platform.syntax.LanguageSyntaxDefinition import com.intellij.platform.syntax.Logger import com.intellij.platform.syntax.SyntaxElementTypeSet -import com.intellij.platform.syntax.element.SyntaxTokenTypes import com.intellij.platform.syntax.extensions.currentExtensionSupport import com.intellij.platform.syntax.lexer.Lexer import com.intellij.platform.syntax.lexer.TokenList -import com.intellij.platform.syntax.lexer.buildTokenList import com.intellij.platform.syntax.lexer.performLexing import com.intellij.platform.syntax.parser.DefaultWhitespaceBindingPolicy import com.intellij.platform.syntax.parser.SyntaxTreeBuilder @@ -64,32 +62,8 @@ fun parse( .withStartOffset(startLexemeOffset) .build() - fun performLexingImpl( - text: CharSequence, - lexer: Lexer, - cancellationProvider: CancellationProvider?, - logger: Logger?, - ): TokenList { - val result = performLexing( - text, - lexer, - cancellationProvider, - logger, - ) - val isEmpty = result.tokenCount == 0 - return if (isEmpty) buildTokenList { - token("", SyntaxTokenTypes.WHITE_SPACE) - } - else result - } - val lexer = lexerFactory() - val tokens = performLexingImpl( - text, - lexer, - cancellationProvider, - logger, - ) + val tokens = performLexing(text, lexer, cancellationProvider, logger) val builder = createBuilder(text, tokens) parser(builder) val markers = builder.toAstMarkers() @@ -99,7 +73,7 @@ fun parse( tokens = builder.tokens, languageProvider = languageMapper, tokenizationPolicy = TokenizationPolicy { text, lexer, cancellation -> - performLexingImpl(text, lexer, cancellation, logger) + performLexing(text, lexer, cancellation, logger) }, lexer = lexer, builderFactory = SyntaxBuilderFactory { text, tokens, startLexeme ->