IDEA-359306 IDEA-383288 javadoc: properly parse Markdown reference links

The PSI tree now looks much closer to what it should have been originally.

Took me a few tries before settling on a solution that doesn't involve a huge code radius blast.

#IDEA-359306 Fixed
#IDEA-383288 Fixed

GitOrigin-RevId: 3c4f71bd7b8320933e170da28ddea46ca69cadff
This commit is contained in:
Mathias
2026-01-12 19:30:15 +00:00
committed by intellij-monorepo-bot
parent 364c540cf2
commit 8c359f76c7
26 changed files with 527 additions and 130 deletions
@@ -235,6 +235,7 @@ public abstract class AbstractBasicJavadocParsingTest extends AbstractBasicJavaP
public void testReferenceLinkMarkdown11() { doTest(true); }
public void testReferenceLinkMarkdown12() { doTest(true); }
public void testReferenceLinkMarkdown13() { doTest(true); }
public void testReferenceLinkMarkdown14() { doTest(true); }
public void testNestedTag0Markdown() { doTest(true); }
public void testNestedTag1Markdown() { doTest(true); }
@@ -262,4 +263,6 @@ public abstract class AbstractBasicJavadocParsingTest extends AbstractBasicJavaP
}
public void testNoValueElementTagsMarkdown() { doTest(true); }
public void testNoAsterisks() { doTest(true); }
}
@@ -4,6 +4,7 @@ package com.intellij.psi.impl.source.javadoc;
import com.intellij.lang.ASTNode;
import com.intellij.openapi.util.TextRange;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.openapi.util.text.Strings;
import com.intellij.psi.*;
import com.intellij.psi.filters.ElementFilter;
import com.intellij.psi.impl.PsiManagerEx;
@@ -34,6 +35,9 @@ import org.jetbrains.annotations.Nullable;
import java.util.*;
public class PsiDocMethodOrFieldRef extends CompositePsiElement implements PsiDocTagValue, Constants {
private static final List<String> SIGNATURE_TO_REPLACE = Arrays.asList("\\[", "\\]");
private static final List<String> SIGNATURE_REPLACEMENT = Arrays.asList("[", "]");
public PsiDocMethodOrFieldRef() {
super(DOC_METHOD_OR_FIELD_REF);
}
@@ -144,7 +148,8 @@ public class PsiDocMethodOrFieldRef extends CompositePsiElement implements PsiDo
List<String> types = new ArrayList<>();
for (PsiElement child = element.getFirstChild(); child != null; child = child.getNextSibling()) {
if (child.getNode().getElementType() == DOC_TYPE_HOLDER) {
types.add(child.getText());
// JEP-467: Markdown comments have escaped brackets for array types
types.add(Strings.replace(child.getText(), SIGNATURE_TO_REPLACE, SIGNATURE_REPLACEMENT));
}
}
@@ -322,13 +322,13 @@ companion object {
"\u0001\u000d\u0001\u0002\u0001\u000e\u0001\u000f\u0001\u0010\u0001\u0011\u0001\u0012\u0001\u0013"+
"\u0001\u0014\u0001\u0015\u0001\u0016\u0001\u0017\u0001\u0018\u0001\u0013\u0001\u0019\u0001\u0001"+
"\u0002\u001a\u0001\u001b\u0001\u001a\u0001\u001c\u0001\u001d\u0001\u0013\u0001\u001e\u0001\u001f"+
"\u0001\u001c\u0001\u0020\u0001\u0013\u0001\u000e\u0001\u0003\u0001\u0021\u0001\u0022\u0001\u0013"+
"\u0001\u0003\u0001\u0023\u0001\u000d\u0001\u0013\u0001\u0003\u0001\u000d\u0001\u0000\u0001\u0024"+
"\u0002\u0000\u0001\u0025\u0008\u0026\u0002\u0000\u0001\u0027\u0006\u0026\u0001\u0000\u0001\u0028"+
"\u0001\u0029\u0008\u0026\u0002\u002a\u0006\u0026\u0001\u002b\u0008\u0026\u0002\u002a\u000e\u0026"+
"\u0002\u002a\u0001\u002c\u0009\u0026\u0001\u002d\u0001\u002e\u0001\u0026\u0002\u002a\u0001\u0026"+
"\u0001\u002e\u0005\u0026\u0002\u002a\u0001\u0026\u0001\u002d\u0004\u0026\u0002\u002a\u0001\u002f"+
"\u0001\u0026\u0002\u002a\u0001\u0026\u0015\u002a"
"\u0001\u001c\u0001\u0020\u0001\u0013\u0001\u0021\u0001\u0003\u0001\u0022\u0001\u0023\u0001\u0013"+
"\u0001\u0003\u0001\u0024\u0001\u000d\u0001\u0013\u0001\u0003\u0001\u000d\u0001\u0000\u0001\u0025"+
"\u0002\u0000\u0001\u0026\u0008\u0027\u0002\u0000\u0001\u0028\u0006\u0027\u0001\u0000\u0001\u0029"+
"\u0001\u002a\u0008\u0027\u0002\u002b\u0006\u0027\u0001\u002c\u0008\u0027\u0002\u002b\u000e\u0027"+
"\u0002\u002b\u0001\u002d\u0009\u0027\u0001\u002e\u0001\u002f\u0001\u0027\u0002\u002b\u0001\u0027"+
"\u0001\u002f\u0005\u0027\u0002\u002b\u0001\u0027\u0001\u002e\u0004\u0027\u0002\u002b\u0001\u0030"+
"\u0001\u0027\u0002\u002b\u0001\u0027\u0015\u002b"
@JvmStatic
private fun zzUnpackAction(): IntArray {
@@ -665,6 +665,8 @@ companion object {
private var mySnippetBracesLevel = 0;
/* Enable markdown support for java 23 */
private var myMarkdownMode = false;
/** Whether comment data should take into account spaces, on used with [myMarkdownMode] */
private var commentDataWithSpaces = false;
constructor(isJdk15Enabled: Boolean) {
myJdk15Enabled = isJdk15Enabled;
@@ -673,6 +675,7 @@ companion object {
/** Should be called right after a reset */
public fun setMarkdownMode(isEnabled: Boolean) {
myMarkdownMode = isEnabled;
if (!myMarkdownMode) commentDataWithSpaces = false;
}
public fun checkAhead(c: Char): Boolean {
@@ -995,15 +998,15 @@ companion object {
1 -> {
return JavaDocSyntaxTokenType.DOC_COMMENT_BAD_CHARACTER;
}
48 -> { /* do nothing */ }
49 -> { /* do nothing */ }
2 -> {
yybegin(COMMENT_DATA); return JavaDocSyntaxTokenType.DOC_COMMENT_DATA;
}
49 -> { /* do nothing */ }
50 -> { /* do nothing */ }
3 -> {
return JavaDocSyntaxTokenType.DOC_SPACE;
}
50 -> { /* do nothing */ }
51 -> { /* do nothing */ }
4 -> {
yybegin(COMMENT_DATA);
if(myMarkdownMode) {
@@ -1011,7 +1014,7 @@ companion object {
}
return JavaDocSyntaxTokenType.DOC_COMMENT_DATA;
}
51 -> { /* do nothing */ }
52 -> { /* do nothing */ }
5 -> {
yybegin(COMMENT_DATA);
if(myMarkdownMode) {
@@ -1019,7 +1022,7 @@ companion object {
}
return JavaDocSyntaxTokenType.DOC_COMMENT_DATA;
}
52 -> { /* do nothing */ }
53 -> { /* do nothing */ }
6 -> {
yybegin(COMMENT_DATA);
if(myMarkdownMode) {
@@ -1027,7 +1030,7 @@ companion object {
}
return JavaDocSyntaxTokenType.DOC_COMMENT_DATA;
}
53 -> { /* do nothing */ }
54 -> { /* do nothing */ }
7 -> {
yybegin(COMMENT_DATA);
if(myMarkdownMode) {
@@ -1035,7 +1038,7 @@ companion object {
}
return JavaDocSyntaxTokenType.DOC_COMMENT_DATA;
}
54 -> { /* do nothing */ }
55 -> { /* do nothing */ }
8 -> {
yybegin(COMMENT_DATA);
if(myMarkdownMode) {
@@ -1043,23 +1046,25 @@ companion object {
}
return JavaDocSyntaxTokenType.DOC_COMMENT_DATA;
}
55 -> { /* do nothing */ }
56 -> { /* do nothing */ }
9 -> {
yybegin(COMMENT_DATA);
if(myMarkdownMode) {
commentDataWithSpaces = true;
return JavaDocSyntaxTokenType.DOC_LBRACKET;
}
return JavaDocSyntaxTokenType.DOC_COMMENT_DATA;
}
56 -> { /* do nothing */ }
57 -> { /* do nothing */ }
10 -> {
yybegin(COMMENT_DATA);
if(myMarkdownMode) {
commentDataWithSpaces = false;
return JavaDocSyntaxTokenType.DOC_RBRACKET;
}
return JavaDocSyntaxTokenType.DOC_COMMENT_DATA;
}
57 -> { /* do nothing */ }
58 -> { /* do nothing */ }
11 -> {
yybegin(COMMENT_DATA);
if(myMarkdownMode) {
@@ -1067,7 +1072,7 @@ companion object {
}
return JavaDocSyntaxTokenType.DOC_COMMENT_DATA;
}
58 -> { /* do nothing */ }
59 -> { /* do nothing */ }
12 -> {
if (checkAhead('@')) {
yybegin(INLINE_TAG_NAME);
@@ -1078,50 +1083,53 @@ companion object {
return JavaDocSyntaxTokenType.DOC_INLINE_TAG_START;
}
}
59 -> { /* do nothing */ }
60 -> { /* do nothing */ }
13 -> {
yybegin(COMMENT_DATA); return JavaDocSyntaxTokenType.DOC_INLINE_TAG_END;
}
60 -> { /* do nothing */ }
14 -> {
return JavaDocSyntaxTokenType.DOC_COMMENT_DATA;
}
61 -> { /* do nothing */ }
14 -> {
return when(commentDataWithSpaces) {
true -> JavaDocSyntaxTokenType.DOC_SPACE
false -> JavaDocSyntaxTokenType.DOC_COMMENT_DATA
}
}
62 -> { /* do nothing */ }
15 -> {
if (checkAhead('<') || checkAhead('\"')) yybegin(COMMENT_DATA);
else if (checkAhead('\u007b')) yybegin(COMMENT_DATA); // lbrace - there's a error in JLex when typing lbrace directly
else yybegin(DOC_TAG_VALUE);
return JavaDocSyntaxTokenType.DOC_SPACE;
}
62 -> { /* do nothing */ }
63 -> { /* do nothing */ }
16 -> {
yybegin(DOC_TAG_VALUE); return JavaDocSyntaxTokenType.DOC_SPACE;
}
63 -> { /* do nothing */ }
64 -> { /* do nothing */ }
17 -> {
yybegin(COMMENT_DATA); return JavaDocSyntaxTokenType.DOC_SPACE;
}
64 -> { /* do nothing */ }
65 -> { /* do nothing */ }
18 -> {
return JavaDocSyntaxTokenType.DOC_TAG_VALUE_SHARP_TOKEN;
}
65 -> { /* do nothing */ }
66 -> { /* do nothing */ }
19 -> {
return JavaDocSyntaxTokenType.DOC_TAG_VALUE_TOKEN;
}
66 -> { /* do nothing */ }
67 -> { /* do nothing */ }
20 -> {
yybegin(DOC_TAG_VALUE_IN_PAREN); return JavaDocSyntaxTokenType.DOC_TAG_VALUE_LPAREN;
}
67 -> { /* do nothing */ }
68 -> { /* do nothing */ }
21 -> {
return JavaDocSyntaxTokenType.DOC_TAG_VALUE_COMMA;
}
68 -> { /* do nothing */ }
69 -> { /* do nothing */ }
22 -> {
return JavaDocSyntaxTokenType.DOC_TAG_VALUE_SLASH;
}
69 -> { /* do nothing */ }
70 -> { /* do nothing */ }
23 -> {
if (myJdk15Enabled) {
yybegin(DOC_TAG_VALUE_IN_LTGT);
@@ -1132,51 +1140,55 @@ companion object {
return JavaDocSyntaxTokenType.DOC_COMMENT_DATA;
}
}
70 -> { /* do nothing */ }
71 -> { /* do nothing */ }
24 -> {
yybegin(DOC_TAG_VALUE); return JavaDocSyntaxTokenType.DOC_TAG_VALUE_RPAREN;
}
71 -> { /* do nothing */ }
72 -> { /* do nothing */ }
25 -> {
yybegin(COMMENT_DATA); return JavaDocSyntaxTokenType.DOC_TAG_VALUE_GT;
}
72 -> { /* do nothing */ }
73 -> { /* do nothing */ }
26 -> {
yybegin(CODE_TAG); return JavaDocSyntaxTokenType.DOC_COMMENT_DATA;
}
73 -> { /* do nothing */ }
74 -> { /* do nothing */ }
27 -> {
yybegin(CODE_TAG); return JavaDocSyntaxTokenType.DOC_SPACE;
}
74 -> { /* do nothing */ }
75 -> { /* do nothing */ }
28 -> {
yybegin(SNIPPET_TAG_COMMENT_DATA_UNTIL_COLON); return JavaDocSyntaxTokenType.DOC_COMMENT_DATA;
}
75 -> { /* do nothing */ }
76 -> { /* do nothing */ }
29 -> {
yybegin(SNIPPET_ATTRIBUTE_VALUE_DOUBLE_QUOTES); return JavaDocSyntaxTokenType.DOC_TAG_VALUE_QUOTE;
}
76 -> { /* do nothing */ }
77 -> { /* do nothing */ }
30 -> {
yybegin(SNIPPET_ATTRIBUTE_VALUE_SINGLE_QUOTES); return JavaDocSyntaxTokenType.DOC_TAG_VALUE_QUOTE;
}
77 -> { /* do nothing */ }
78 -> { /* do nothing */ }
31 -> {
if (myMarkdownMode) {
return JavaDocSyntaxTokenType.DOC_COMMENT_DATA;
}
return JavaDocSyntaxTokenType.DOC_COMMENT_LEADING_ASTERISKS;
}
78 -> { /* do nothing */ }
79 -> { /* do nothing */ }
32 -> {
yybegin(SNIPPET_TAG_BODY_DATA); return JavaDocSyntaxTokenType.DOC_TAG_VALUE_COLON;
}
79 -> { /* do nothing */ }
80 -> { /* do nothing */ }
33 -> {
return JavaDocSyntaxTokenType.DOC_COMMENT_DATA;
}
81 -> { /* do nothing */ }
34 -> {
mySnippetBracesLevel++; return JavaDocSyntaxTokenType.DOC_COMMENT_DATA;
}
80 -> { /* do nothing */ }
34 -> {
82 -> { /* do nothing */ }
35 -> {
if (mySnippetBracesLevel > 0) {
mySnippetBracesLevel--;
return JavaDocSyntaxTokenType.DOC_COMMENT_DATA;
@@ -1185,81 +1197,81 @@ companion object {
return JavaDocSyntaxTokenType.DOC_INLINE_TAG_END;
}
}
81 -> { /* do nothing */ }
35 -> {
83 -> { /* do nothing */ }
36 -> {
yybegin(SNIPPET_TAG_COMMENT_DATA_UNTIL_COLON); return JavaDocSyntaxTokenType.DOC_TAG_VALUE_QUOTE;
}
82 -> { /* do nothing */ }
36 -> {
84 -> { /* do nothing */ }
37 -> {
if(myMarkdownMode) {
return JavaDocSyntaxTokenType.DOC_COMMENT_DATA;
}
return JavaDocSyntaxTokenType.DOC_COMMENT_END;
}
83 -> { /* do nothing */ }
37 -> {
85 -> { /* do nothing */ }
38 -> {
yybegin(COMMENT_DATA);
if(myMarkdownMode) {
return JavaDocSyntaxTokenType.DOC_DOUBLE_SHARP;
}
return JavaDocSyntaxTokenType.DOC_COMMENT_DATA;
}
84 -> { /* do nothing */ }
38 -> {
86 -> { /* do nothing */ }
39 -> {
yybegin(TAG_DOC_SPACE); return JavaDocSyntaxTokenType.DOC_TAG_NAME;
}
85 -> { /* do nothing */ }
39 -> {
87 -> { /* do nothing */ }
40 -> {
return JavaDocSyntaxTokenType.DOC_TAG_VALUE_DOUBLE_SHARP_TOKEN;
}
86 -> { /* do nothing */ }
40 -> {
88 -> { /* do nothing */ }
41 -> {
if (myMarkdownMode) {
return JavaDocSyntaxTokenType.DOC_COMMENT_BAD_CHARACTER;
}
yybegin(COMMENT_DATA_START);
return JavaDocSyntaxTokenType.DOC_COMMENT_START;
}
87 -> { /* do nothing */ }
41 -> {
89 -> { /* do nothing */ }
42 -> {
if(myMarkdownMode) {
yybegin(COMMENT_DATA_START);
return JavaDocSyntaxTokenType.DOC_COMMENT_LEADING_ASTERISKS;
}
return JavaDocSyntaxTokenType.DOC_COMMENT_BAD_CHARACTER;
}
88 -> { /* do nothing */ }
42 -> {
90 -> { /* do nothing */ }
43 -> {
yybegin(COMMENT_DATA);
if(myMarkdownMode) {
return JavaDocSyntaxTokenType.DOC_CODE_FENCE;
}
return JavaDocSyntaxTokenType.DOC_COMMENT_DATA;
}
89 -> { /* do nothing */ }
43 -> {
91 -> { /* do nothing */ }
44 -> {
if (myMarkdownMode) {
return JavaDocSyntaxTokenType.DOC_COMMENT_LEADING_ASTERISKS;
}
return JavaDocSyntaxTokenType.DOC_COMMENT_DATA;
}
90 -> { /* do nothing */ }
44 -> {
92 -> { /* do nothing */ }
45 -> {
yybegin(CODE_TAG_SPACE); return JavaDocSyntaxTokenType.DOC_TAG_NAME;
}
91 -> { /* do nothing */ }
45 -> {
93 -> { /* do nothing */ }
46 -> {
yybegin(DOC_TAG_VALUE); return JavaDocSyntaxTokenType.DOC_TAG_NAME;
}
92 -> { /* do nothing */ }
46 -> {
94 -> { /* do nothing */ }
47 -> {
yybegin(PARAM_TAG_SPACE); return JavaDocSyntaxTokenType.DOC_TAG_NAME;
}
93 -> { /* do nothing */ }
47 -> {
95 -> { /* do nothing */ }
48 -> {
yybegin(SNIPPET_TAG_COMMENT_DATA_UNTIL_COLON); return JavaDocSyntaxTokenType.DOC_TAG_NAME;
}
94 -> { /* do nothing */ }
96 -> { /* do nothing */ }
else ->
zzScanError(ZZ_NO_MATCH)
}
@@ -14,6 +14,8 @@ import kotlin.jvm.JvmStatic
private var mySnippetBracesLevel = 0;
/* Enable markdown support for java 23 */
private var myMarkdownMode = false;
/** Whether comment data should take into account spaces, on used with [myMarkdownMode] */
private var commentDataWithSpaces = false;
constructor(isJdk15Enabled: Boolean) {
myJdk15Enabled = isJdk15Enabled;
@@ -22,6 +24,7 @@ import kotlin.jvm.JvmStatic
/** Should be called right after a reset */
public fun setMarkdownMode(isEnabled: Boolean) {
myMarkdownMode = isEnabled;
if (!myMarkdownMode) commentDataWithSpaces = false;
}
public fun checkAhead(c: Char): Boolean {
@@ -94,7 +97,11 @@ LEADING_TOKEN_MARKDOWN="///"
}
<COMMENT_DATA_START> {WHITE_DOC_SPACE_CHAR}+ { return JavaDocSyntaxTokenType.DOC_SPACE; }
<COMMENT_DATA> {WHITE_DOC_SPACE_NO_LR}+ { return JavaDocSyntaxTokenType.DOC_COMMENT_DATA; }
<COMMENT_DATA> {WHITE_DOC_SPACE_NO_LR}+ { return when(commentDataWithSpaces) {
true -> JavaDocSyntaxTokenType.DOC_SPACE
false -> JavaDocSyntaxTokenType.DOC_COMMENT_DATA
}
}
<COMMENT_DATA> [\n\r]+{WHITE_DOC_SPACE_CHAR}* { return JavaDocSyntaxTokenType.DOC_SPACE; }
<DOC_TAG_VALUE> {WHITE_DOC_SPACE_CHAR}+ { yybegin(COMMENT_DATA); return JavaDocSyntaxTokenType.DOC_SPACE; }
@@ -183,6 +190,7 @@ LEADING_TOKEN_MARKDOWN="///"
\[ {
yybegin(COMMENT_DATA);
if(myMarkdownMode) {
commentDataWithSpaces = true;
return JavaDocSyntaxTokenType.DOC_LBRACKET;
}
return JavaDocSyntaxTokenType.DOC_COMMENT_DATA;
@@ -190,6 +198,7 @@ LEADING_TOKEN_MARKDOWN="///"
\] {
yybegin(COMMENT_DATA);
if(myMarkdownMode) {
commentDataWithSpaces = false;
return JavaDocSyntaxTokenType.DOC_RBRACKET;
}
return JavaDocSyntaxTokenType.DOC_COMMENT_DATA;
@@ -9,9 +9,11 @@ import com.intellij.platform.syntax.SyntaxElementType
import com.intellij.platform.syntax.SyntaxElementTypeSet
import com.intellij.platform.syntax.element.SyntaxTokenTypes
import com.intellij.platform.syntax.parser.SyntaxTreeBuilder
import com.intellij.platform.syntax.parser.WhitespacesAndCommentsBinder
import com.intellij.platform.syntax.parser.WhitespacesBinders.greedyLeftBinder
import com.intellij.platform.syntax.parser.WhitespacesBinders.greedyRightBinder
import com.intellij.platform.syntax.syntaxElementTypeSetOf
import com.intellij.platform.syntax.util.parser.SyntaxBuilderUtil.rawTokenText
import com.intellij.pom.java.LanguageLevel
import org.jetbrains.annotations.Contract
@@ -19,6 +21,22 @@ class JavaDocParser(
val builder: SyntaxTreeBuilder,
val languageLevel: LanguageLevel,
) {
companion object {
/** Binder taking the next space if it isn't an EOL */
private val fakeCollapseRightBinder = object : WhitespacesAndCommentsBinder {
override fun getEdgePosition(
tokens: List<SyntaxElementType>,
atStreamEdge: Boolean,
getter: WhitespacesAndCommentsBinder.TokenTextGetter,
): Int {
if (tokens.isNotEmpty() && !isEolToken(tokens.first(), getter.get(0))) {
return 1
}
return 0
}
}
}
private var braceScope: Int = 0
private var closingStatusList: MutableList<Boolean> = mutableListOf() // true for all '{' opening an inline tag
@@ -153,7 +171,7 @@ class JavaDocParser(
else if (tokenType === JavaDocSyntaxTokenType.DOC_LBRACKET) {
parseMarkdownReferenceChecked()
}
else if (tokenType === JavaDocSyntaxTokenType.DOC_COMMENT_DATA) {
else if (tokenType === JavaDocSyntaxTokenType.DOC_COMMENT_DATA || (isWhiteSpace(tokenType) && !isEolToken(tokenType, builder.tokenText))) {
parseCommentData()
}
else {
@@ -164,44 +182,49 @@ class JavaDocParser(
private fun parseCommentData() {
val commentData = builder.mark()
val offset = builder.currentOffset
var aheadType: SyntaxElementType? = null
while (builder.rawLookup(1).also { aheadType = it } != null) {
val hasCommentTokenNext = COMMENT_DATA_TOKENS.contains(aheadType)
val hasWhiteSpaceTokenNext = isWhiteSpace(aheadType) && !isEolToken(aheadType, builder.rawTokenText(1))
while (COMMENT_DATA_TOKENS.contains(builder.rawLookup(1))) {
builder.advanceLexer()
if (hasCommentTokenNext || hasWhiteSpaceTokenNext) {
builder.rawAdvanceLexer(1)
} else {
break
}
}
if (builder.currentOffset != offset) {
builder.advanceLexer()
commentData.collapse(JavaDocSyntaxTokenType.DOC_COMMENT_DATA)
} else {
}
else {
commentData.drop()
builder.advanceLexer()
}
}
private fun parseInlineCodeBlock() {
var tag = builder.mark()
val blockMarker = builder.mark()
builder.rawAdvanceLexer(1)
val contentMarker = builder.mark()
val stopElementType = findInlineToken(JavaDocSyntaxTokenType.DOC_INLINE_CODE_FENCE)
val endOffset = builder.currentOffset
tag.rollbackTo()
if (stopElementType !== JavaDocSyntaxTokenType.DOC_INLINE_CODE_FENCE) {
// Bail out, no end
contentMarker.drop()
blockMarker.rollbackTo()
builder.advanceLexer()
return
}
tag = builder.mark()
builder.advanceLexer()
while (builder.currentOffset < endOffset && !builder.eof()) {
builder.remapCurrentToken(JavaDocSyntaxTokenType.DOC_COMMENT_DATA)
builder.advanceLexer()
}
fakeCollapse(contentMarker, JavaDocSyntaxTokenType.DOC_COMMENT_DATA)
if (!builder.eof()) {
builder.advanceLexer()
}
tag.done(JavaDocSyntaxElementType.DOC_MARKDOWN_CODE_BLOCK)
blockMarker.done(JavaDocSyntaxElementType.DOC_MARKDOWN_CODE_BLOCK)
}
private fun parseCodeBlock() {
@@ -294,19 +317,32 @@ class JavaDocParser(
if (hasLabel) {
builder.advanceLexer()
val label = builder.mark()
/* Collapses comment data within a label */
var commentDataMarker: SyntaxTreeBuilder.Marker? = null
// Label range already known, mark it as comment data
while (!builder.eof()) {
if (builder.tokenType === JavaDocSyntaxTokenType.DOC_INLINE_CODE_FENCE) {
commentDataMarker?.let {
fakeCollapse(it, JavaDocSyntaxTokenType.DOC_COMMENT_DATA)
commentDataMarker = null
}
parseInlineCodeBlock()
continue
}
if (builder.currentOffset < endLabelOffset) {
builder.remapCurrentToken(JavaDocSyntaxTokenType.DOC_COMMENT_DATA)
if(commentDataMarker == null) {
commentDataMarker = builder.mark()
}
builder.advanceLexer()
continue
}
break
}
commentDataMarker?.let {
fakeCollapse(it, JavaDocSyntaxTokenType.DOC_COMMENT_DATA)
}
label.done(JavaDocSyntaxElementType.DOC_MARKDOWN_REFERENCE_LABEL)
builder.advanceLexer()
}
@@ -347,19 +383,27 @@ class JavaDocParser(
builder.advanceLexer()
builder.remapCurrentToken(JavaDocSyntaxTokenType.DOC_TAG_VALUE_TOKEN)
// A method only has parenthesis and a few comment data, separated by commas
// A method only has parenthesis, comment data which may be the type, the optional argument name and commas
builder.advanceLexer()
if (builder.tokenType === JavaDocSyntaxTokenType.DOC_LPAREN) {
builder.advanceLexer()
val subValue = builder.mark()
// Only the first data element count as a type in links like [#foo(int arg1, int arg2)]
var dataSinceComma = false
while (!builder.eof()) {
val type = getTokenType()
if (type === JavaDocSyntaxTokenType.DOC_COMMENT_DATA) {
builder.remapCurrentToken(JavaDocSyntaxElementType.DOC_TYPE_HOLDER)
if(!dataSinceComma) {
dataSinceComma = true
builder.remapCurrentToken(JavaDocSyntaxElementType.DOC_TYPE_HOLDER)
}
}
else if (type !== JavaDocSyntaxTokenType.DOC_COMMA) {
break
} else {
dataSinceComma = false;
}
builder.advanceLexer()
}
@@ -402,6 +446,10 @@ class JavaDocParser(
} else {
refStart.drop()
}
// This method is guaranteed the existence of an end bracket
if (getTokenType() !== JavaDocSyntaxTokenType.DOC_RBRACKET) {
findInlineToken(JavaDocSyntaxTokenType.DOC_RBRACKET)
}
}
@@ -679,11 +727,48 @@ class JavaDocParser(
tagData.done(JavaDocSyntaxElementType.DOC_TAG_VALUE_ELEMENT)
}
private fun getTokenType(): SyntaxElementType? {
return getTokenType(true)
/**
* Behaves like-ish [SyntaxTreeBuilder.Marker.collapse] but skips over eol and leading asterisks.
* This means it cannot merge everything in a single node.
*
* Note that [marker] will be dropped.
*/
private fun fakeCollapse(marker: SyntaxTreeBuilder.Marker, tokenType: SyntaxElementType) {
/** Whether a token should be ignored during the fake collapse */
fun shouldIgnore(type: SyntaxElementType?) = (type != null &&
(type === JavaDocSyntaxTokenType.DOC_COMMENT_LEADING_ASTERISKS || isEolToken(type, builder.tokenText)))
val endOffset = builder.currentOffset
marker.rollbackTo()
val currentTokenType = getTokenType(false)
var fuseMarker: SyntaxTreeBuilder.Marker? = if (shouldIgnore(currentTokenType)) null else builder.mark()
while (!builder.eof() && builder.currentOffset < endOffset) {
if (shouldIgnore(currentTokenType)) {
fuseMarker?.collapse(tokenType)
fuseMarker?.setCustomEdgeTokenBinders(null, fakeCollapseRightBinder)
fuseMarker = null
builder.advanceLexer()
continue
}
if (fuseMarker == null) {
fuseMarker = builder.mark()
}
if (isWhiteSpace(builder.rawLookup(1)) && builder.rawLookup(2) == JavaDocSyntaxTokenType.DOC_COMMENT_LEADING_ASTERISKS) {
builder.advanceLexer()
fuseMarker.collapse(tokenType)
fuseMarker.setCustomEdgeTokenBinders(null, fakeCollapseRightBinder)
fuseMarker = null
} else {
builder.advanceLexer()
}
}
fuseMarker?.collapse(tokenType)
fuseMarker?.setCustomEdgeTokenBinders(null, fakeCollapseRightBinder)
}
private fun getTokenType(skipWhitespace: Boolean): SyntaxElementType? {
private fun getTokenType(skipWhitespace: Boolean = true): SyntaxElementType? {
var tokenType: SyntaxElementType?
while ((builder.tokenType.also { tokenType = it }) === JavaDocSyntaxTokenType.DOC_SPACE) {
builder.remapCurrentToken(SyntaxTokenTypes.WHITE_SPACE)
@@ -708,6 +793,16 @@ class JavaDocParser(
}
}
/** @return Whether the token is eol */
private fun isEolToken(type: SyntaxElementType?, text: CharSequence?): Boolean {
return isWhiteSpace(type) && text != null && text.contains('\n')
}
/** @return Whether the token is a whitespace */
private fun isWhiteSpace(type: SyntaxElementType?): Boolean {
return type != null && (type == JavaDocSyntaxTokenType.DOC_SPACE || type == SyntaxTokenTypes.WHITE_SPACE)
}
private val TAG_VALUES_SET: SyntaxElementTypeSet = syntaxElementTypeSetOf(
JavaDocSyntaxTokenType.DOC_TAG_VALUE_TOKEN, JavaDocSyntaxTokenType.DOC_TAG_VALUE_COMMA, JavaDocSyntaxTokenType.DOC_TAG_VALUE_DOT,
JavaDocSyntaxTokenType.DOC_TAG_VALUE_LPAREN, JavaDocSyntaxTokenType.DOC_TAG_VALUE_RPAREN,
@@ -6,9 +6,7 @@ PsiJavaFile:CodeBlockMarkdown05.java
PsiDocToken:DOC_INLINE_CODE_FENCE('`')
PsiWhiteSpace('\n')
PsiDocToken:DOC_COMMENT_LEADING_ASTERISKS('///')
PsiDocToken:DOC_COMMENT_DATA(' According to markdown rules')
PsiDocToken:DOC_COMMENT_DATA(',')
PsiDocToken:DOC_COMMENT_DATA(' this is inline')
PsiDocToken:DOC_COMMENT_DATA(' According to markdown rules, this is inline')
PsiWhiteSpace('\n')
PsiDocToken:DOC_COMMENT_LEADING_ASTERISKS('///')
PsiDocToken:DOC_COMMENT_DATA(' ')
@@ -7,8 +7,6 @@ java.FILE
WHITE_SPACE
DOC_COMMENT_LEADING_ASTERISKS
DOC_COMMENT_DATA
DOC_COMMENT_DATA
DOC_COMMENT_DATA
WHITE_SPACE
DOC_COMMENT_LEADING_ASTERISKS
DOC_COMMENT_DATA
@@ -0,0 +1,10 @@
class C {
/**
Hello
world
Beep Boop
*/
int foo() { return 1; }
}
@@ -0,0 +1,57 @@
PsiJavaFile:NoAsterisks.java
PsiImportList
<empty list>
PsiClass:C
PsiModifierList:
<empty list>
PsiKeyword:class('class')
PsiWhiteSpace(' ')
PsiIdentifier:C('C')
PsiTypeParameterList
<empty list>
PsiReferenceList
<empty list>
PsiReferenceList
<empty list>
PsiWhiteSpace(' ')
PsiJavaToken:LBRACE('{')
PsiWhiteSpace('\n ')
PsiMethod:foo
PsiDocComment
PsiDocToken:DOC_COMMENT_START('/**')
PsiWhiteSpace('\n ')
PsiDocToken:DOC_COMMENT_DATA('Hello')
PsiWhiteSpace('\n ')
PsiDocToken:DOC_COMMENT_DATA('world')
PsiWhiteSpace('\n\n\n ')
PsiDocToken:DOC_COMMENT_DATA('Beep Boop')
PsiWhiteSpace('\n ')
PsiDocToken:DOC_COMMENT_END('*/')
PsiWhiteSpace('\n ')
PsiModifierList:
<empty list>
PsiTypeParameterList
<empty list>
PsiTypeElement:int
PsiKeyword:int('int')
PsiWhiteSpace(' ')
PsiIdentifier:foo('foo')
PsiParameterList:()
PsiJavaToken:LPARENTH('(')
PsiJavaToken:RPARENTH(')')
PsiReferenceList
<empty list>
PsiWhiteSpace(' ')
PsiCodeBlock
PsiJavaToken:LBRACE('{')
PsiWhiteSpace(' ')
PsiReturnStatement
PsiKeyword:return('return')
PsiWhiteSpace(' ')
PsiLiteralExpression:1
PsiJavaToken:INTEGER_LITERAL('1')
PsiJavaToken:SEMICOLON(';')
PsiWhiteSpace(' ')
PsiJavaToken:RBRACE('}')
PsiWhiteSpace('\n')
PsiJavaToken:RBRACE('}')
@@ -0,0 +1,57 @@
java.FILE
IMPORT_LIST
<empty list>
CLASS
MODIFIER_LIST
<empty list>
CLASS_KEYWORD
WHITE_SPACE
IDENTIFIER
TYPE_PARAMETER_LIST
<empty list>
EXTENDS_LIST
<empty list>
IMPLEMENTS_LIST
<empty list>
WHITE_SPACE
LBRACE
WHITE_SPACE
METHOD
DOC_COMMENT
DOC_COMMENT_START
WHITE_SPACE
DOC_COMMENT_DATA
WHITE_SPACE
DOC_COMMENT_DATA
WHITE_SPACE
DOC_COMMENT_DATA
WHITE_SPACE
DOC_COMMENT_END
WHITE_SPACE
MODIFIER_LIST
<empty list>
TYPE_PARAMETER_LIST
<empty list>
TYPE
INT_KEYWORD
WHITE_SPACE
IDENTIFIER
PARAMETER_LIST
LPARENTH
RPARENTH
THROWS_LIST
<empty list>
WHITE_SPACE
CODE_BLOCK
LBRACE
WHITE_SPACE
RETURN_STATEMENT
RETURN_KEYWORD
WHITE_SPACE
LITERAL_EXPRESSION
INTEGER_LITERAL
SEMICOLON
WHITE_SPACE
RBRACE
WHITE_SPACE
RBRACE
@@ -8,9 +8,7 @@ PsiJavaFile:ReferenceLinkMarkdown03.java
PsiReferenceLink:
PsiDocToken:DOC_LBRACKET('[')
PsiReferenceLabel:
PsiDocToken:DOC_COMMENT_DATA('[')
PsiDocToken:DOC_COMMENT_DATA('label with balanced brackets')
PsiDocToken:DOC_COMMENT_DATA(']')
PsiDocToken:DOC_COMMENT_DATA('[label with balanced brackets]')
PsiDocToken:DOC_RBRACKET(']')
PsiDocToken:DOC_LBRACKET('[')
PsiElement(DOC_REFERENCE_HOLDER)
@@ -9,8 +9,6 @@ java.FILE
DOC_LBRACKET
DOC_REFERENCE_LABEL
DOC_COMMENT_DATA
DOC_COMMENT_DATA
DOC_COMMENT_DATA
DOC_RBRACKET
DOC_LBRACKET
DOC_REFERENCE_HOLDER
@@ -12,12 +12,12 @@ PsiJavaFile:ReferenceLinkMarkdown05.java
PsiIdentifier:label('label')
PsiReferenceParameterList
<empty list>
PsiWhiteSpace(' ')
PsiIdentifier:with('with')
PsiWhiteSpace(' ')
PsiIdentifier:unbalanced('unbalanced')
PsiWhiteSpace(' ')
PsiIdentifier:brackets('brackets')
PsiWhiteSpace(' ')
PsiDocToken:DOC_COMMENT_DATA('with')
PsiWhiteSpace(' ')
PsiDocToken:DOC_COMMENT_DATA('unbalanced')
PsiWhiteSpace(' ')
PsiDocToken:DOC_COMMENT_DATA('brackets')
PsiDocToken:DOC_RBRACKET(']')
PsiDocToken:DOC_RBRACKET(']')
PsiReferenceLink:
@@ -12,12 +12,12 @@ java.FILE
IDENTIFIER
REFERENCE_PARAMETER_LIST
<empty list>
WHITE_SPACE
IDENTIFIER
WHITE_SPACE
IDENTIFIER
WHITE_SPACE
IDENTIFIER
WHITE_SPACE
DOC_COMMENT_DATA
WHITE_SPACE
DOC_COMMENT_DATA
WHITE_SPACE
DOC_COMMENT_DATA
DOC_RBRACKET
DOC_RBRACKET
DOC_REFERENCE_LINK
@@ -26,8 +26,8 @@ PsiJavaFile:ReferenceLinkMarkdown07.java
PsiDocToken:DOC_TAG_VALUE_SHARP_TOKEN('#')
PsiDocToken:DOC_TAG_VALUE_TOKEN('EMPTY_')
PsiDocToken:DOC_SHARP('#')
PsiDocToken:DOC_COMMENT_DATA('LIST')
PsiDocToken:DOC_RBRACKET(']')
PsiDocToken:DOC_COMMENT_DATA('LIST')
PsiDocToken:DOC_RBRACKET(']')
PsiWhiteSpace('\n')
PsiModifierList:
<empty list>
@@ -26,8 +26,8 @@ java.FILE
DOC_TAG_VALUE_SHARP_TOKEN
DOC_TAG_VALUE_TOKEN
DOC_SHARP
DOC_COMMENT_DATA
DOC_RBRACKET
DOC_COMMENT_DATA
DOC_RBRACKET
WHITE_SPACE
MODIFIER_LIST
<empty list>
@@ -34,13 +34,13 @@ PsiJavaFile:ReferenceLinkMarkdown11.java
PsiJavaToken:LBRACKET('\[')
PsiJavaToken:RBRACKET('\]')
PsiDocToken:DOC_COMMA(',')
PsiWhiteSpace(' ')
PsiElement(DOC_TYPE_HOLDER)
PsiWhiteSpace(' ')
PsiTypeElement:int
PsiKeyword:int('int')
PsiDocToken:DOC_COMMA(',')
PsiWhiteSpace(' ')
PsiElement(DOC_TYPE_HOLDER)
PsiWhiteSpace(' ')
PsiTypeElement:int
PsiKeyword:int('int')
PsiDocToken:DOC_RPAREN(')')
@@ -34,13 +34,13 @@ java.FILE
LBRACKET
RBRACKET
DOC_COMMA
WHITE_SPACE
DOC_TYPE_HOLDER
WHITE_SPACE
TYPE
INT_KEYWORD
DOC_COMMA
WHITE_SPACE
DOC_TYPE_HOLDER
WHITE_SPACE
TYPE
INT_KEYWORD
DOC_RPAREN
@@ -0,0 +1,9 @@
/// [[unbalanced] Hey hey people
///
/// [ balanced with spaces ][java.lang.String]
///
/// [ Beep Boop all of you are forced to have
/// space
///
/// but not you ofc
class C {}
@@ -0,0 +1,76 @@
PsiJavaFile:ReferenceLinkMarkdown14.java
PsiImportList
<empty list>
PsiClass:C
PsiDocComment
PsiDocToken:DOC_COMMENT_LEADING_ASTERISKS('///')
PsiWhiteSpace(' ')
PsiDocToken:DOC_LBRACKET('[')
PsiReferenceLink:
PsiDocToken:DOC_LBRACKET('[')
PsiElement(DOC_REFERENCE_HOLDER)
PsiJavaCodeReferenceElement:unbalanced
PsiIdentifier:unbalanced('unbalanced')
PsiReferenceParameterList
<empty list>
PsiDocToken:DOC_RBRACKET(']')
PsiDocToken:DOC_COMMENT_DATA(' Hey hey people')
PsiWhiteSpace('\n')
PsiDocToken:DOC_COMMENT_LEADING_ASTERISKS('///')
PsiWhiteSpace(' \n')
PsiDocToken:DOC_COMMENT_LEADING_ASTERISKS('///')
PsiDocToken:DOC_COMMENT_DATA(' ')
PsiReferenceLink:
PsiDocToken:DOC_LBRACKET('[')
PsiWhiteSpace(' ')
PsiReferenceLabel:
PsiDocToken:DOC_COMMENT_DATA('balanced with spaces ')
PsiDocToken:DOC_RBRACKET(']')
PsiDocToken:DOC_LBRACKET('[')
PsiElement(DOC_REFERENCE_HOLDER)
PsiJavaCodeReferenceElement:java.lang.String
PsiJavaCodeReferenceElement:java.lang
PsiJavaCodeReferenceElement:java
PsiIdentifier:java('java')
PsiReferenceParameterList
<empty list>
PsiJavaToken:DOT('.')
PsiIdentifier:lang('lang')
PsiReferenceParameterList
<empty list>
PsiJavaToken:DOT('.')
PsiIdentifier:String('String')
PsiReferenceParameterList
<empty list>
PsiDocToken:DOC_RBRACKET(']')
PsiWhiteSpace('\n')
PsiDocToken:DOC_COMMENT_LEADING_ASTERISKS('///')
PsiWhiteSpace(' \n')
PsiDocToken:DOC_COMMENT_LEADING_ASTERISKS('///')
PsiDocToken:DOC_COMMENT_DATA(' ')
PsiDocToken:DOC_LBRACKET('[')
PsiWhiteSpace(' ')
PsiDocToken:DOC_COMMENT_DATA('Beep Boop all of you are forced to have')
PsiWhiteSpace(' \n')
PsiDocToken:DOC_COMMENT_LEADING_ASTERISKS('///')
PsiDocToken:DOC_COMMENT_DATA(' space')
PsiWhiteSpace('\n')
PsiDocToken:DOC_COMMENT_LEADING_ASTERISKS('///')
PsiWhiteSpace(' \n')
PsiDocToken:DOC_COMMENT_LEADING_ASTERISKS('///')
PsiDocToken:DOC_COMMENT_DATA(' but not you ofc')
PsiWhiteSpace('\n')
PsiModifierList:
<empty list>
PsiKeyword:class('class')
PsiWhiteSpace(' ')
PsiIdentifier:C('C')
PsiTypeParameterList
<empty list>
PsiReferenceList
<empty list>
PsiReferenceList
<empty list>
PsiWhiteSpace(' ')
PsiJavaToken:LBRACE('{')
PsiJavaToken:RBRACE('}')
@@ -0,0 +1,76 @@
java.FILE
IMPORT_LIST
<empty list>
CLASS
DOC_MARKDOWN_COMMENT
DOC_COMMENT_LEADING_ASTERISKS
WHITE_SPACE
DOC_LBRACKET
DOC_REFERENCE_LINK
DOC_LBRACKET
DOC_REFERENCE_HOLDER
JAVA_CODE_REFERENCE
IDENTIFIER
REFERENCE_PARAMETER_LIST
<empty list>
DOC_RBRACKET
DOC_COMMENT_DATA
WHITE_SPACE
DOC_COMMENT_LEADING_ASTERISKS
WHITE_SPACE
DOC_COMMENT_LEADING_ASTERISKS
DOC_COMMENT_DATA
DOC_REFERENCE_LINK
DOC_LBRACKET
WHITE_SPACE
DOC_REFERENCE_LABEL
DOC_COMMENT_DATA
DOC_RBRACKET
DOC_LBRACKET
DOC_REFERENCE_HOLDER
JAVA_CODE_REFERENCE
JAVA_CODE_REFERENCE
JAVA_CODE_REFERENCE
IDENTIFIER
REFERENCE_PARAMETER_LIST
<empty list>
DOT
IDENTIFIER
REFERENCE_PARAMETER_LIST
<empty list>
DOT
IDENTIFIER
REFERENCE_PARAMETER_LIST
<empty list>
DOC_RBRACKET
WHITE_SPACE
DOC_COMMENT_LEADING_ASTERISKS
WHITE_SPACE
DOC_COMMENT_LEADING_ASTERISKS
DOC_COMMENT_DATA
DOC_LBRACKET
WHITE_SPACE
DOC_COMMENT_DATA
WHITE_SPACE
DOC_COMMENT_LEADING_ASTERISKS
DOC_COMMENT_DATA
WHITE_SPACE
DOC_COMMENT_LEADING_ASTERISKS
WHITE_SPACE
DOC_COMMENT_LEADING_ASTERISKS
DOC_COMMENT_DATA
WHITE_SPACE
MODIFIER_LIST
<empty list>
CLASS_KEYWORD
WHITE_SPACE
IDENTIFIER
TYPE_PARAMETER_LIST
<empty list>
EXTENDS_LIST
<empty list>
IMPLEMENTS_LIST
<empty list>
WHITE_SPACE
LBRACE
RBRACE
@@ -14,9 +14,8 @@ PsiJavaFile:SnippetTag4.java
PsiDocToken:DOC_INLINE_TAG_START('{')
PsiDocToken:DOC_TAG_NAME('@snippet')
PsiWhiteSpace(' ')
PsiDocToken:DOC_COMMENT_DATA('###')
PsiDocToken:DOC_COMMENT_DATA('### ')
PsiSnippetDocTagValue
PsiWhiteSpace(' ')
PsiSnippetAttributeList
PsiSnippetAttribute:attrName
PsiDocToken:DOC_TAG_ATTRIBUTE_NAME('attrName')
@@ -14,9 +14,8 @@ PsiJavaFile:SnippetTag4Markdown.java
PsiDocToken:DOC_INLINE_TAG_START('{')
PsiDocToken:DOC_TAG_NAME('@snippet')
PsiWhiteSpace(' ')
PsiDocToken:DOC_COMMENT_DATA('###')
PsiDocToken:DOC_COMMENT_DATA('### ')
PsiSnippetDocTagValue
PsiWhiteSpace(' ')
PsiSnippetAttributeList
PsiSnippetAttribute:attrName
PsiDocToken:DOC_TAG_ATTRIBUTE_NAME('attrName')
@@ -16,7 +16,6 @@ java.FILE
WHITE_SPACE
DOC_COMMENT_DATA
DOC_SNIPPET_TAG_VALUE
WHITE_SPACE
DOC_SNIPPET_ATTRIBUTE_LIST
DOC_SNIPPET_ATTRIBUTE
DOC_TAG_ATTRIBUTE_NAME
@@ -16,7 +16,6 @@ java.FILE
WHITE_SPACE
DOC_COMMENT_DATA
DOC_SNIPPET_TAG_VALUE
WHITE_SPACE
DOC_SNIPPET_ATTRIBUTE_LIST
DOC_SNIPPET_ATTRIBUTE
DOC_TAG_ATTRIBUTE_NAME
@@ -2126,7 +2126,7 @@ public class Test {
fun testBracketsInReferenceLink(){
doTextTest("""
/// [String#copyValueOf(char \[ \], int, int)]
/// [String#copyValueOf(char\[\], int, int)]
public class Main {
void test(char[] foo) {}
}