diff --git a/plugins/markdown/core/BUILD.bazel b/plugins/markdown/core/BUILD.bazel index 7e59d5076d37..47596226274b 100644 --- a/plugins/markdown/core/BUILD.bazel +++ b/plugins/markdown/core/BUILD.bazel @@ -79,6 +79,7 @@ jvm_library( "//platform/util/text-matching", "//platform/usageView-impl", "//platform/usageView", + "//libraries/fastutil", ], ) @@ -150,6 +151,7 @@ jvm_library( "//platform/util/text-matching:text-matching_test_lib", "//platform/usageView-impl:usageView-impl_test_lib", "//platform/usageView:usageView_test_lib", + "//libraries/fastutil:fastutil_test_lib", ], ) ### auto-generated section `build intellij.markdown` end diff --git a/plugins/markdown/core/intellij.markdown.iml b/plugins/markdown/core/intellij.markdown.iml index 266ac963b17b..47b137848fb3 100644 --- a/plugins/markdown/core/intellij.markdown.iml +++ b/plugins/markdown/core/intellij.markdown.iml @@ -87,5 +87,6 @@ + \ No newline at end of file diff --git a/plugins/markdown/core/src/org/intellij/plugins/markdown/lang/MarkdownFileElementType.kt b/plugins/markdown/core/src/org/intellij/plugins/markdown/lang/MarkdownFileElementType.kt index feb2d9346157..be2fa1fc21ce 100644 --- a/plugins/markdown/core/src/org/intellij/plugins/markdown/lang/MarkdownFileElementType.kt +++ b/plugins/markdown/core/src/org/intellij/plugins/markdown/lang/MarkdownFileElementType.kt @@ -1,10 +1,49 @@ package org.intellij.plugins.markdown.lang +import com.intellij.lang.ASTNode +import com.intellij.lang.PsiBuilder +import com.intellij.lang.PsiBuilderFactory +import com.intellij.lang.WhitespacesBinders +import com.intellij.psi.ParsingDiagnostics +import com.intellij.psi.PsiElement import com.intellij.psi.stubs.PsiFileStub import com.intellij.psi.tree.IStubFileElementType +import org.intellij.plugins.markdown.lang.MarkdownLazyElementType.obtainFlavour +import org.intellij.plugins.markdown.lang.lexer.MarkdownToplevelLexer +import org.intellij.plugins.markdown.lang.parser.MarkdownParserManager +import org.intellij.plugins.markdown.lang.parser.PsiBuilderFillingVisitor import org.intellij.plugins.markdown.lang.psi.impl.MarkdownFile +/** + * The root element type for Markdown files. + * + * The Markdown parser produces its own AST, while IntelliJ PSI is built through [PsiBuilder]. The parsed Markdown AST is + * therefore created once here, then passed to [MarkdownToplevelLexer] to provide the builder with token boundaries and to + * [PsiBuilderFillingVisitor] to create the hierarchical IntelliJ AST. The lexer is backed by the already parsed tree and + * does not invoke the Markdown parser again. + * + * [PsiBuilder] is still required because it creates the IntelliJ AST/PSI nodes, handles whitespace binders, and preserves + * the platform's lazy-parse contract. + */ open class MarkdownFileElementType: IStubFileElementType>( "MarkdownFile", MarkdownLanguage.INSTANCE -) +) { + override fun doParseContents(chameleon: ASTNode, psi: PsiElement): ASTNode? { + val flavour = obtainFlavour(psi.containingFile) + + val startTime = System.nanoTime() + val parsedTree = MarkdownParserManager.parseContent(chameleon.chars, flavour) + val lexer = MarkdownToplevelLexer(flavour, parsedTree) + val builder = PsiBuilderFactory.getInstance() + .createBuilder(psi.project, chameleon, lexer, MarkdownLanguage.INSTANCE, chameleon.chars) + ParsingDiagnostics.registerParse(builder, MarkdownLanguage.INSTANCE, System.nanoTime() - startTime) + + val rootMarker = builder.mark() + rootMarker.setCustomEdgeTokenBinders(WhitespacesBinders.GREEDY_LEFT_BINDER, WhitespacesBinders.GREEDY_RIGHT_BINDER) + PsiBuilderFillingVisitor(builder, true).visitNode(parsedTree) + assert(builder.eof()) + rootMarker.done(this) + return builder.treeBuilt.firstChildNode + } +} diff --git a/plugins/markdown/core/src/org/intellij/plugins/markdown/lang/MarkdownLazyElementType.java b/plugins/markdown/core/src/org/intellij/plugins/markdown/lang/MarkdownLazyElementType.java index d80c1fea55f5..5fc5c419d321 100644 --- a/plugins/markdown/core/src/org/intellij/plugins/markdown/lang/MarkdownLazyElementType.java +++ b/plugins/markdown/core/src/org/intellij/plugins/markdown/lang/MarkdownLazyElementType.java @@ -18,6 +18,7 @@ import org.intellij.plugins.markdown.lang.parser.MarkdownFlavourUtil; import org.intellij.plugins.markdown.lang.parser.MarkdownParserManager; import org.intellij.plugins.markdown.lang.parser.PsiBuilderFillingVisitor; import org.intellij.plugins.markdown.lang.psi.impl.MarkdownFile; +import org.jetbrains.annotations.ApiStatus; import org.jetbrains.annotations.NonNls; import org.jetbrains.annotations.NotNull; @@ -67,7 +68,8 @@ public class MarkdownLazyElementType extends ILazyParseableElementType { return actualElement; } - private static @NotNull MarkdownFlavourDescriptor obtainFlavour(@NotNull PsiFile file) { + @ApiStatus.Internal + public static @NotNull MarkdownFlavourDescriptor obtainFlavour(@NotNull PsiFile file) { if (file instanceof MarkdownFile markdownFile) { return markdownFile.getFlavour(); } diff --git a/plugins/markdown/core/src/org/intellij/plugins/markdown/lang/lexer/MarkdownToplevelLexer.java b/plugins/markdown/core/src/org/intellij/plugins/markdown/lang/lexer/MarkdownToplevelLexer.java index 08abd4025d5f..ef2320b0200e 100644 --- a/plugins/markdown/core/src/org/intellij/plugins/markdown/lang/lexer/MarkdownToplevelLexer.java +++ b/plugins/markdown/core/src/org/intellij/plugins/markdown/lang/lexer/MarkdownToplevelLexer.java @@ -4,6 +4,8 @@ package org.intellij.plugins.markdown.lang.lexer; import com.intellij.lexer.LexerBase; import com.intellij.openapi.progress.ProgressManager; import com.intellij.psi.tree.IElementType; +import it.unimi.dsi.fastutil.ints.IntArrayList; +import it.unimi.dsi.fastutil.ints.IntList; import org.intellij.markdown.ast.ASTNode; import org.intellij.markdown.ast.ASTNodeKt; import org.intellij.markdown.ast.visitors.RecursiveVisitor; @@ -21,46 +23,58 @@ public class MarkdownToplevelLexer extends LexerBase { private int myBufferStart; private int myBufferEnd; - private int lastBufferHash = 0; + private int myBufferHash = 0; private final List myLexemes = new ArrayList<>(); - private final List myStartOffsets = new ArrayList<>(); - private final List myEndOffsets = new ArrayList<>(); + private final IntList myStartOffsets = new IntArrayList(); + private final IntList myEndOffsets = new IntArrayList(); + private final IntList myStates = new IntArrayList(); private int myLexemeIndex; private final @NotNull MarkdownFlavourDescriptor flavour; + private final @Nullable ASTNode parsedTree; public MarkdownToplevelLexer() { this(MarkdownParserManager.FLAVOUR); } public MarkdownToplevelLexer(@NotNull MarkdownFlavourDescriptor flavour) { + this(flavour, null); + } + + public MarkdownToplevelLexer(@NotNull MarkdownFlavourDescriptor flavour, @Nullable ASTNode parsedTree) { this.flavour = flavour; + this.parsedTree = parsedTree; } @Override public void start(@NotNull CharSequence buffer, int startOffset, int endOffset, int initialState) { final var bufferHash = buffer.hashCode(); - myBufferStart = startOffset; - myBufferEnd = endOffset; - if (bufferHash == lastBufferHash && buffer.equals(myBuffer)) { - myLexemeIndex = initialState; + if (bufferHash == myBufferHash && startOffset == myBufferStart && buffer.equals(myBuffer)) { + myLexemeIndex = 0; return; } - lastBufferHash = bufferHash; + myBufferHash = bufferHash; myBuffer = buffer; - final var parsedTree = MarkdownParserManager.parseContent(buffer.subSequence(startOffset, endOffset), flavour); + myBufferStart = startOffset; + myBufferEnd = endOffset; + var parsedTree = + this.parsedTree != null ? + this.parsedTree : MarkdownParserManager.parseContent(buffer.subSequence(startOffset, endOffset), flavour); myLexemes.clear(); myStartOffsets.clear(); myEndOffsets.clear(); - ASTNodeKt.accept(parsedTree, new LexerBuildingVisitor()); + myStates.clear(); + for (ASTNode child : parsedTree.getChildren()) { + ASTNodeKt.accept(child, new LexerBuildingVisitor()); + } myLexemeIndex = 0; } @Override public int getState() { - return myLexemeIndex; + return myLexemeIndex < myStates.size() ? myStates.getInt(myLexemeIndex) : 1; } @Override @@ -76,7 +90,7 @@ public class MarkdownToplevelLexer extends LexerBase { if (myLexemeIndex >= myLexemes.size()) { return myBufferEnd; } - return myBufferStart + myStartOffsets.get(myLexemeIndex); + return myBufferStart + myStartOffsets.getInt(myLexemeIndex); } @Override @@ -84,7 +98,7 @@ public class MarkdownToplevelLexer extends LexerBase { if (myLexemeIndex >= myLexemes.size()) { return myBufferEnd; } - return myBufferStart + myEndOffsets.get(myLexemeIndex); + return myBufferStart + myEndOffsets.getInt(myLexemeIndex); } @Override @@ -103,6 +117,7 @@ public class MarkdownToplevelLexer extends LexerBase { } private class LexerBuildingVisitor extends RecursiveVisitor { + private boolean myFirstLeaf = true; @Override public void visitNode(@NotNull ASTNode node) { @@ -115,6 +130,8 @@ public class MarkdownToplevelLexer extends LexerBase { myLexemes.add(MarkdownElementType.platformType(node.getType())); myStartOffsets.add(node.getStartOffset()); myEndOffsets.add(node.getEndOffset()); + myStates.add(myFirstLeaf ? 0 : 1); + myFirstLeaf = false; } else { super.visitNode(node); diff --git a/plugins/markdown/core/src/org/intellij/plugins/markdown/lang/parser/MarkdownParserManager.kt b/plugins/markdown/core/src/org/intellij/plugins/markdown/lang/parser/MarkdownParserManager.kt index 3503c95d9b33..b253a7796625 100644 --- a/plugins/markdown/core/src/org/intellij/plugins/markdown/lang/parser/MarkdownParserManager.kt +++ b/plugins/markdown/core/src/org/intellij/plugins/markdown/lang/parser/MarkdownParserManager.kt @@ -26,22 +26,18 @@ class MarkdownParserManager: Disposable { @JvmOverloads fun parse(buffer: CharSequence, flavour: MarkdownFlavourDescriptor = FLAVOUR): ASTNode { - val wrappedBuffer = object: BombedCharSequence(buffer) { - override fun checkCanceled() { - ProgressManager.checkCanceled() - } - } - return performParsing(wrappedBuffer, flavour) - } - - private fun performParsing(buffer: CharSequence, flavour: MarkdownFlavourDescriptor = FLAVOUR): ASTNode { val info = lastParsingResult.get()?.get() if (info != null && info.bufferHash == buffer.hashCode() && info.buffer == buffer) { return info.tree } + val stringBuffer = buffer as? String ?: buffer.toString() val parseResult = createMarkdownParser(flavour).parse( MarkdownElementTypes.MARKDOWN_FILE, - buffer, + object : BombedCharSequence(stringBuffer) { + override fun checkCanceled() { + ProgressManager.checkCanceled() + } + }, parseInlines = false ) lastParsingResult.set(SoftReference(ParsingResult(buffer, parseResult))) diff --git a/plugins/markdown/core/src/org/intellij/plugins/markdown/lang/psi/impl/MarkdownHeader.kt b/plugins/markdown/core/src/org/intellij/plugins/markdown/lang/psi/impl/MarkdownHeader.kt index 4c7693a048aa..c4c6e4297427 100644 --- a/plugins/markdown/core/src/org/intellij/plugins/markdown/lang/psi/impl/MarkdownHeader.kt +++ b/plugins/markdown/core/src/org/intellij/plugins/markdown/lang/psi/impl/MarkdownHeader.kt @@ -10,8 +10,8 @@ import com.intellij.openapi.util.registry.Registry import com.intellij.openapi.util.text.StringUtil import com.intellij.psi.PsiElement import com.intellij.psi.PsiElementVisitor +import com.intellij.psi.PsiFile import com.intellij.psi.PsiReference -import com.intellij.psi.SyntaxTraverser import com.intellij.psi.impl.source.resolve.reference.ReferenceProvidersRegistry import com.intellij.psi.impl.source.tree.LeafPsiElement import com.intellij.psi.stubs.StubBuildCachedValuesManager.StubBuildCachedValueProvider @@ -150,13 +150,10 @@ class MarkdownHeader: MarkdownHeaderImpl { } } - private fun buildRawAnchorText(includeStartingHash: Boolean = false): String? { + private fun buildRawAnchorText(): String? { val contentHolder = findContentHolder() ?: return null val children = contentHolder.children().dropWhile { it.hasType(MarkdownTokenTypeSets.WHITE_SPACES) } val text = buildString { - if (includeStartingHash) { - append("#") - } var count = 0 for (child in children) { if (child.hasType(MarkdownTokenTypeSets.WHITE_SPACES)) { @@ -206,11 +203,11 @@ class MarkdownHeader: MarkdownHeaderImpl { return sameHeaders.takeWhile { it != header }.count() } - private val HEADERS_LIST_PROVIDER = StubBuildCachedValueProvider, com.intellij.psi.PsiFile>( + private val HEADERS_LIST_PROVIDER = StubBuildCachedValueProvider, PsiFile>( "markdown.header.headersList" ) { file -> CachedValueProvider.Result.create( - SyntaxTraverser.psiTraverser(file).filterIsInstance(), + file.children.filterIsInstance(), PsiModificationTracker.MODIFICATION_COUNT ) } @@ -241,7 +238,7 @@ class MarkdownHeader: MarkdownHeaderImpl { private val OBTAIN_RAW_ANCHOR_PROVIDER = StubBuildCachedValueProvider( "markdown.header.rawAnchorText" ) { header -> - CachedValueProvider.Result.create(header.buildRawAnchorText(false), PsiModificationTracker.MODIFICATION_COUNT) + CachedValueProvider.Result.create(header.buildRawAnchorText(), PsiModificationTracker.MODIFICATION_COUNT) } private val ENTITY_REGEX = Regex("""&(?:([a-zA-Z0-9]+)|#([0-9]{1,8})|#[xX]([a-fA-F0-9]{1,8}));""") diff --git a/plugins/markdown/test/src/org/intellij/plugins/markdown/highlighting/MarkdownHighlightingLexerTest.kt b/plugins/markdown/test/src/org/intellij/plugins/markdown/highlighting/MarkdownHighlightingLexerTest.kt new file mode 100644 index 000000000000..6e1b909ebcdd --- /dev/null +++ b/plugins/markdown/test/src/org/intellij/plugins/markdown/highlighting/MarkdownHighlightingLexerTest.kt @@ -0,0 +1,67 @@ +// Copyright 2000-2026 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.intellij.plugins.markdown.highlighting + +import com.intellij.lexer.Lexer +import com.intellij.testFramework.fixtures.BasePlatformTestCase + +class MarkdownHighlightingLexerTest : BasePlatformTestCase() { + fun `test restart at Markdown block boundaries`() { + val text = """ + # Header + + Paragraph with *emphasis* and [a link](https://example.com). + + > A quote + > continued + + - first item + - second item + + | column | value | + | --- | --- | + | one | two | + + ```kotlin + val answer = 42 + ``` + """.trimIndent() + val allTokens = tokenize(text, 0, 0) + val lexer = createLexer() + + lexer.start(text) + var index = 0 + while (lexer.tokenType != null) { + if (lexer.state == 0) { + assertEquals(allTokens.subList(index, allTokens.size), tokenize(text, lexer.tokenStart, lexer.state)) + } + index++ + lexer.advance() + } + + for (blockStart in listOf( + text.indexOf("# Header"), + text.indexOf("Paragraph with"), + text.indexOf("> A quote"), + text.indexOf("- first item"), + text.indexOf("| column"), + text.indexOf("```kotlin"), + )) { + assertTrue(allTokens.any { it.state == 0 && (it.start == blockStart || it.end == blockStart) }) + } + } + + private fun createLexer(): Lexer = MarkdownHighlightingLexer(null) + + private fun tokenize(text: String, start: Int, state: Int): List { + val lexer = createLexer() + lexer.start(text, start, text.length, state) + val tokens = mutableListOf() + while (lexer.tokenType != null) { + tokens.add(Token(lexer.tokenStart, lexer.tokenEnd, lexer.state)) + lexer.advance() + } + return tokens + } + + private data class Token(val start: Int, val end: Int, val state: Int) +}