[markdown] IJPL-251345, IJPL-251944 Improve markdown performance

Merge-request: IJ-MR-217226
Merged-by: Ilia Permiashkin <ilia.permiashkin@jetbrains.com>
(cherry picked from commit 41fe8cc635017a3c5e653001a6b5e9fff2ca5d03)

Merge-request: IJ-MR-216956
Merged-by: Ilia Permiashkin <ilia.permiashkin@jetbrains.com>
(cherry picked from commit c4b52f21b200d56c716cd0f3bbf35b312bd8e809)

Merge-request: IJ-MR-219945
Merged-by: Ilia Permiashkin <ilia.permiashkin@jetbrains.com>

GitOrigin-RevId: 15c612b5b6f707f87ce537640a249b3c1eb90795

plugins/markdown/core/BUILD.bazel

@@ -79,6 +79,7 @@ jvm_library(
         "//platform/util/text-matching",
         "//platform/usageView-impl",
         "//platform/usageView",
+        "//libraries/fastutil",
     ],
 )
 
@@ -150,6 +151,7 @@ jvm_library(
         "//platform/util/text-matching:text-matching_test_lib",
         "//platform/usageView-impl:usageView-impl_test_lib",
         "//platform/usageView:usageView_test_lib",
+        "//libraries/fastutil:fastutil_test_lib",
     ],
 )
 ### auto-generated section `build intellij.markdown` end

plugins/markdown/core/intellij.markdown.iml

@@ -87,5 +87,6 @@
     <orderEntry type="module" module-name="intellij.platform.util.text.matching" />
     <orderEntry type="module" module-name="intellij.platform.usageView.impl" />
     <orderEntry type="module" module-name="intellij.platform.usageView" />
+    <orderEntry type="module" module-name="intellij.libraries.fastutil" />
   </component>
 </module>

plugins/markdown/core/src/org/intellij/plugins/markdown/lang/MarkdownFileElementType.kt

@@ -1,10 +1,49 @@
 package org.intellij.plugins.markdown.lang
 
+import com.intellij.lang.ASTNode
+import com.intellij.lang.PsiBuilder
+import com.intellij.lang.PsiBuilderFactory
+import com.intellij.lang.WhitespacesBinders
+import com.intellij.psi.ParsingDiagnostics
+import com.intellij.psi.PsiElement
 import com.intellij.psi.stubs.PsiFileStub
 import com.intellij.psi.tree.IStubFileElementType
+import org.intellij.plugins.markdown.lang.MarkdownLazyElementType.obtainFlavour
+import org.intellij.plugins.markdown.lang.lexer.MarkdownToplevelLexer
+import org.intellij.plugins.markdown.lang.parser.MarkdownParserManager
+import org.intellij.plugins.markdown.lang.parser.PsiBuilderFillingVisitor
 import org.intellij.plugins.markdown.lang.psi.impl.MarkdownFile
 
+/**
+ * The root element type for Markdown files.
+ *
+ * The Markdown parser produces its own AST, while IntelliJ PSI is built through [PsiBuilder]. The parsed Markdown AST is
+ * therefore created once here, then passed to [MarkdownToplevelLexer] to provide the builder with token boundaries and to
+ * [PsiBuilderFillingVisitor] to create the hierarchical IntelliJ AST. The lexer is backed by the already parsed tree and
+ * does not invoke the Markdown parser again.
+ *
+ * [PsiBuilder] is still required because it creates the IntelliJ AST/PSI nodes, handles whitespace binders, and preserves
+ * the platform's lazy-parse contract.
+ */
 open class MarkdownFileElementType: IStubFileElementType<PsiFileStub<MarkdownFile>>(
   "MarkdownFile",
   MarkdownLanguage.INSTANCE
-)
+) {
+  override fun doParseContents(chameleon: ASTNode, psi: PsiElement): ASTNode? {
+    val flavour = obtainFlavour(psi.containingFile)
+
+    val startTime = System.nanoTime()
+    val parsedTree = MarkdownParserManager.parseContent(chameleon.chars, flavour)
+    val lexer = MarkdownToplevelLexer(flavour, parsedTree)
+    val builder = PsiBuilderFactory.getInstance()
+      .createBuilder(psi.project, chameleon, lexer, MarkdownLanguage.INSTANCE, chameleon.chars)
+    ParsingDiagnostics.registerParse(builder, MarkdownLanguage.INSTANCE, System.nanoTime() - startTime)
+
+    val rootMarker = builder.mark()
+    rootMarker.setCustomEdgeTokenBinders(WhitespacesBinders.GREEDY_LEFT_BINDER, WhitespacesBinders.GREEDY_RIGHT_BINDER)
+    PsiBuilderFillingVisitor(builder, true).visitNode(parsedTree)
+    assert(builder.eof())
+    rootMarker.done(this)
+    return builder.treeBuilt.firstChildNode
+  }
+}

plugins/markdown/core/src/org/intellij/plugins/markdown/lang/MarkdownLazyElementType.java

@@ -18,6 +18,7 @@ import org.intellij.plugins.markdown.lang.parser.MarkdownFlavourUtil;
 import org.intellij.plugins.markdown.lang.parser.MarkdownParserManager;
 import org.intellij.plugins.markdown.lang.parser.PsiBuilderFillingVisitor;
 import org.intellij.plugins.markdown.lang.psi.impl.MarkdownFile;
+import org.jetbrains.annotations.ApiStatus;
 import org.jetbrains.annotations.NonNls;
 import org.jetbrains.annotations.NotNull;
 
@@ -67,7 +68,8 @@ public class MarkdownLazyElementType extends ILazyParseableElementType {
     return actualElement;
   }
 
-  private static @NotNull MarkdownFlavourDescriptor obtainFlavour(@NotNull PsiFile file) {
+  @ApiStatus.Internal
+  public static @NotNull MarkdownFlavourDescriptor obtainFlavour(@NotNull PsiFile file) {
     if (file instanceof MarkdownFile markdownFile) {
       return markdownFile.getFlavour();
     }

plugins/markdown/core/src/org/intellij/plugins/markdown/lang/lexer/MarkdownToplevelLexer.java

@@ -4,6 +4,8 @@ package org.intellij.plugins.markdown.lang.lexer;
 import com.intellij.lexer.LexerBase;
 import com.intellij.openapi.progress.ProgressManager;
 import com.intellij.psi.tree.IElementType;
+import it.unimi.dsi.fastutil.ints.IntArrayList;
+import it.unimi.dsi.fastutil.ints.IntList;
 import org.intellij.markdown.ast.ASTNode;
 import org.intellij.markdown.ast.ASTNodeKt;
 import org.intellij.markdown.ast.visitors.RecursiveVisitor;
@@ -21,46 +23,58 @@ public class MarkdownToplevelLexer extends LexerBase {
   private int myBufferStart;
   private int myBufferEnd;
 
-  private int lastBufferHash = 0;
+  private int myBufferHash = 0;
 
   private final List<IElementType> myLexemes = new ArrayList<>();
-  private final List<Integer> myStartOffsets = new ArrayList<>();
-  private final List<Integer> myEndOffsets = new ArrayList<>();
+  private final IntList myStartOffsets = new IntArrayList();
+  private final IntList myEndOffsets = new IntArrayList();
+  private final IntList myStates = new IntArrayList();
 
   private int myLexemeIndex;
 
   private final @NotNull MarkdownFlavourDescriptor flavour;
+  private final @Nullable ASTNode parsedTree;
 
   public MarkdownToplevelLexer() {
     this(MarkdownParserManager.FLAVOUR);
   }
 
   public MarkdownToplevelLexer(@NotNull MarkdownFlavourDescriptor flavour) {
+    this(flavour, null);
+  }
+
+  public MarkdownToplevelLexer(@NotNull MarkdownFlavourDescriptor flavour, @Nullable ASTNode parsedTree) {
     this.flavour = flavour;
+    this.parsedTree = parsedTree;
   }
 
   @Override
   public void start(@NotNull CharSequence buffer, int startOffset, int endOffset, int initialState) {
     final var bufferHash = buffer.hashCode();
-    myBufferStart = startOffset;
-    myBufferEnd = endOffset;
-    if (bufferHash == lastBufferHash && buffer.equals(myBuffer)) {
-      myLexemeIndex = initialState;
+    if (bufferHash == myBufferHash && startOffset == myBufferStart && buffer.equals(myBuffer)) {
+      myLexemeIndex = 0;
       return;
     }
-    lastBufferHash = bufferHash;
+    myBufferHash = bufferHash;
     myBuffer = buffer;
-    final var parsedTree = MarkdownParserManager.parseContent(buffer.subSequence(startOffset, endOffset), flavour);
+    myBufferStart = startOffset;
+    myBufferEnd = endOffset;
+    var parsedTree =
+      this.parsedTree != null ?
+      this.parsedTree : MarkdownParserManager.parseContent(buffer.subSequence(startOffset, endOffset), flavour);
     myLexemes.clear();
     myStartOffsets.clear();
     myEndOffsets.clear();
-    ASTNodeKt.accept(parsedTree, new LexerBuildingVisitor());
+    myStates.clear();
+    for (ASTNode child : parsedTree.getChildren()) {
+      ASTNodeKt.accept(child, new LexerBuildingVisitor());
+    }
     myLexemeIndex = 0;
   }
 
   @Override
   public int getState() {
-    return myLexemeIndex;
+    return myLexemeIndex < myStates.size() ? myStates.getInt(myLexemeIndex) : 1;
   }
 
   @Override
@@ -76,7 +90,7 @@ public class MarkdownToplevelLexer extends LexerBase {
     if (myLexemeIndex >= myLexemes.size()) {
       return myBufferEnd;
     }
-    return myBufferStart + myStartOffsets.get(myLexemeIndex);
+    return myBufferStart + myStartOffsets.getInt(myLexemeIndex);
   }
 
   @Override
@@ -84,7 +98,7 @@ public class MarkdownToplevelLexer extends LexerBase {
     if (myLexemeIndex >= myLexemes.size()) {
       return myBufferEnd;
     }
-    return myBufferStart + myEndOffsets.get(myLexemeIndex);
+    return myBufferStart + myEndOffsets.getInt(myLexemeIndex);
   }
 
   @Override
@@ -103,6 +117,7 @@ public class MarkdownToplevelLexer extends LexerBase {
   }
 
   private class LexerBuildingVisitor extends RecursiveVisitor {
+    private boolean myFirstLeaf = true;
 
     @Override
     public void visitNode(@NotNull ASTNode node) {
@@ -115,6 +130,8 @@ public class MarkdownToplevelLexer extends LexerBase {
         myLexemes.add(MarkdownElementType.platformType(node.getType()));
         myStartOffsets.add(node.getStartOffset());
         myEndOffsets.add(node.getEndOffset());
+        myStates.add(myFirstLeaf ? 0 : 1);
+        myFirstLeaf = false;
       }
       else {
         super.visitNode(node);

plugins/markdown/core/src/org/intellij/plugins/markdown/lang/parser/MarkdownParserManager.kt

@@ -26,22 +26,18 @@ class MarkdownParserManager: Disposable {
 
   @JvmOverloads
   fun parse(buffer: CharSequence, flavour: MarkdownFlavourDescriptor = FLAVOUR): ASTNode {
-    val wrappedBuffer = object: BombedCharSequence(buffer) {
-      override fun checkCanceled() {
-        ProgressManager.checkCanceled()
-      }
-    }
-    return performParsing(wrappedBuffer, flavour)
-  }
-
-  private fun performParsing(buffer: CharSequence, flavour: MarkdownFlavourDescriptor = FLAVOUR): ASTNode {
     val info = lastParsingResult.get()?.get()
     if (info != null && info.bufferHash == buffer.hashCode() && info.buffer == buffer) {
       return info.tree
     }
+    val stringBuffer = buffer as? String ?: buffer.toString()
     val parseResult = createMarkdownParser(flavour).parse(
       MarkdownElementTypes.MARKDOWN_FILE,
-      buffer,
+      object : BombedCharSequence(stringBuffer) {
+        override fun checkCanceled() {
+          ProgressManager.checkCanceled()
+        }
+      },
       parseInlines = false
     )
     lastParsingResult.set(SoftReference(ParsingResult(buffer, parseResult)))

plugins/markdown/core/src/org/intellij/plugins/markdown/lang/psi/impl/MarkdownHeader.kt

@@ -10,8 +10,8 @@ import com.intellij.openapi.util.registry.Registry
 import com.intellij.openapi.util.text.StringUtil
 import com.intellij.psi.PsiElement
 import com.intellij.psi.PsiElementVisitor
+import com.intellij.psi.PsiFile
 import com.intellij.psi.PsiReference
-import com.intellij.psi.SyntaxTraverser
 import com.intellij.psi.impl.source.resolve.reference.ReferenceProvidersRegistry
 import com.intellij.psi.impl.source.tree.LeafPsiElement
 import com.intellij.psi.stubs.StubBuildCachedValuesManager.StubBuildCachedValueProvider
@@ -150,13 +150,10 @@ class MarkdownHeader: MarkdownHeaderImpl {
     }
   }
 
-  private fun buildRawAnchorText(includeStartingHash: Boolean = false): String? {
+  private fun buildRawAnchorText(): String? {
     val contentHolder = findContentHolder() ?: return null
     val children = contentHolder.children().dropWhile { it.hasType(MarkdownTokenTypeSets.WHITE_SPACES) }
     val text = buildString {
-      if (includeStartingHash) {
-        append("#")
-      }
       var count = 0
       for (child in children) {
         if (child.hasType(MarkdownTokenTypeSets.WHITE_SPACES)) {
@@ -206,11 +203,11 @@ class MarkdownHeader: MarkdownHeaderImpl {
       return sameHeaders.takeWhile { it != header }.count()
     }
 
-    private val HEADERS_LIST_PROVIDER = StubBuildCachedValueProvider<Iterable<MarkdownHeader>, com.intellij.psi.PsiFile>(
+    private val HEADERS_LIST_PROVIDER = StubBuildCachedValueProvider<Iterable<MarkdownHeader>, PsiFile>(
       "markdown.header.headersList"
     ) { file ->
       CachedValueProvider.Result.create(
-        SyntaxTraverser.psiTraverser(file).filterIsInstance<MarkdownHeader>(),
+        file.children.filterIsInstance<MarkdownHeader>(),
         PsiModificationTracker.MODIFICATION_COUNT
       )
     }
@@ -241,7 +238,7 @@ class MarkdownHeader: MarkdownHeaderImpl {
     private val OBTAIN_RAW_ANCHOR_PROVIDER = StubBuildCachedValueProvider<String?, MarkdownHeader>(
       "markdown.header.rawAnchorText"
     ) { header ->
-      CachedValueProvider.Result.create(header.buildRawAnchorText(false), PsiModificationTracker.MODIFICATION_COUNT)
+      CachedValueProvider.Result.create(header.buildRawAnchorText(), PsiModificationTracker.MODIFICATION_COUNT)
     }
 
     private val ENTITY_REGEX = Regex("""&(?:([a-zA-Z0-9]+)|#([0-9]{1,8})|#[xX]([a-fA-F0-9]{1,8}));""")

plugins/markdown/test/src/org/intellij/plugins/markdown/highlighting/MarkdownHighlightingLexerTest.kt

@@ -0,0 +1,67 @@
+// Copyright 2000-2026 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package org.intellij.plugins.markdown.highlighting
+
+import com.intellij.lexer.Lexer
+import com.intellij.testFramework.fixtures.BasePlatformTestCase
+
+class MarkdownHighlightingLexerTest : BasePlatformTestCase() {
+  fun `test restart at Markdown block boundaries`() {
+    val text = """
+      # Header
+
+      Paragraph with *emphasis* and [a link](https://example.com).
+
+      > A quote
+      > continued
+
+      - first item
+      - second item
+
+      | column | value |
+      | --- | --- |
+      | one | two |
+
+      ```kotlin
+      val answer = 42
+      ```
+    """.trimIndent()
+    val allTokens = tokenize(text, 0, 0)
+    val lexer = createLexer()
+
+    lexer.start(text)
+    var index = 0
+    while (lexer.tokenType != null) {
+      if (lexer.state == 0) {
+        assertEquals(allTokens.subList(index, allTokens.size), tokenize(text, lexer.tokenStart, lexer.state))
+      }
+      index++
+      lexer.advance()
+    }
+
+    for (blockStart in listOf(
+      text.indexOf("# Header"),
+      text.indexOf("Paragraph with"),
+      text.indexOf("> A quote"),
+      text.indexOf("- first item"),
+      text.indexOf("| column"),
+      text.indexOf("```kotlin"),
+    )) {
+      assertTrue(allTokens.any { it.state == 0 && (it.start == blockStart || it.end == blockStart) })
+    }
+  }
+
+  private fun createLexer(): Lexer = MarkdownHighlightingLexer(null)
+
+  private fun tokenize(text: String, start: Int, state: Int): List<Token> {
+    val lexer = createLexer()
+    lexer.start(text, start, text.length, state)
+    val tokens = mutableListOf<Token>()
+    while (lexer.tokenType != null) {
+      tokens.add(Token(lexer.tokenStart, lexer.tokenEnd, lexer.state))
+      lexer.advance()
+    }
+    return tokens
+  }
+
+  private data class Token(val start: Int, val end: Int, val state: Int)
+}