diff --git a/java/java-frontback-psi-impl/src/com/intellij/lang/java/lexer/JavaTypeEscapeLexer.java b/java/java-frontback-psi-impl/src/com/intellij/lang/java/lexer/JavaTypeEscapeLexer.java new file mode 100644 index 000000000000..eb125a9d7d4d --- /dev/null +++ b/java/java-frontback-psi-impl/src/com/intellij/lang/java/lexer/JavaTypeEscapeLexer.java @@ -0,0 +1,44 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.lang.java.lexer; + +import com.intellij.lexer.Lexer; +import com.intellij.lexer.MergeFunction; +import com.intellij.lexer.MergingLexerAdapterBase; +import com.intellij.psi.TokenType; +import com.intellij.psi.tree.IElementType; + +/** + * Lexer to wrap around the Java Lexer when parsing Java types for the markdown JavaDoc (JEP-467) + * In reference links, array types are escaped: "char\[\]" which the JavaLexer doesn't like + *

+ * It does this by covering up a BAD_CHARACTER token if followed by [ or ] + */ +public class JavaTypeEscapeLexer extends MergingLexerAdapterBase { + private final MergeFunction myMergeFunction = new EscapeMarkdownFunction(); + + public JavaTypeEscapeLexer(BasicJavaLexer original) { + super(original); + } + + @Override + public MergeFunction getMergeFunction() { + return myMergeFunction; + } + + + private static class EscapeMarkdownFunction implements MergeFunction { + @Override + public IElementType merge(IElementType type, Lexer originalLexer) { + if(type != TokenType.BAD_CHARACTER) return type; + + CharSequence tokenText = originalLexer.getTokenSequence(); + if (tokenText.length() != 1 || (tokenText.charAt(0) != '[' && tokenText.charAt(0) != ']')) { + return type; + } + + type = originalLexer.getTokenType(); + originalLexer.advance(); + return type; + } + } +} diff --git a/java/java-frontback-psi-impl/src/com/intellij/lang/java/parser/BasicJavaParserUtil.java b/java/java-frontback-psi-impl/src/com/intellij/lang/java/parser/BasicJavaParserUtil.java index 3ccb89ea9227..7afb60d3f110 100644 --- a/java/java-frontback-psi-impl/src/com/intellij/lang/java/parser/BasicJavaParserUtil.java +++ b/java/java-frontback-psi-impl/src/com/intellij/lang/java/parser/BasicJavaParserUtil.java @@ -193,7 +193,7 @@ public final class BasicJavaParserUtil { final boolean eatAll, final LanguageLevel level, Function javaDocLexer, - Function javaLexer) { + Function javaLexer) { final PsiElement psi = chameleon.getTreeParent() != null ? chameleon.getTreeParent().getPsi() : chameleon.getPsi(); assert psi != null : chameleon; final Project project = psi.getProject(); diff --git a/java/java-frontback-psi-impl/src/com/intellij/psi/impl/source/BasicJavaDocElementType.java b/java/java-frontback-psi-impl/src/com/intellij/psi/impl/source/BasicJavaDocElementType.java index d9683d26aa56..67522d9dade9 100644 --- a/java/java-frontback-psi-impl/src/com/intellij/psi/impl/source/BasicJavaDocElementType.java +++ b/java/java-frontback-psi-impl/src/com/intellij/psi/impl/source/BasicJavaDocElementType.java @@ -5,6 +5,7 @@ import com.intellij.lang.Language; import com.intellij.lang.java.JavaLanguage; import com.intellij.lang.java.lexer.BasicJavaLexer; import com.intellij.lang.java.lexer.JavaDocLexer; +import com.intellij.lang.java.lexer.JavaTypeEscapeLexer; import com.intellij.lang.java.parser.BasicJavaDocParser; import com.intellij.lang.java.parser.BasicJavaParser; import com.intellij.lang.java.parser.BasicJavaParserUtil; @@ -125,11 +126,11 @@ public interface BasicJavaDocElementType { private final @NotNull Supplier myJavaThinParser; private final Function javaDocLexer; - private final Function javaLexer; + private final Function javaLexer; public DocTypeHolderElementType(@NotNull Supplier parser, @NotNull Function docLexerFunction, - @NotNull Function javaLexer) { + @NotNull Function javaLexer) { super("DOC_TYPE_HOLDER", BASIC_DOC_TYPE_HOLDER); this.myJavaThinParser = parser; this.javaDocLexer = docLexerFunction; diff --git a/java/java-frontback-psi-impl/src/com/intellij/psi/impl/source/BasicJavaElementType.java b/java/java-frontback-psi-impl/src/com/intellij/psi/impl/source/BasicJavaElementType.java index 40d30bec6ae3..4271df359cf0 100644 --- a/java/java-frontback-psi-impl/src/com/intellij/psi/impl/source/BasicJavaElementType.java +++ b/java/java-frontback-psi-impl/src/com/intellij/psi/impl/source/BasicJavaElementType.java @@ -251,10 +251,10 @@ public interface BasicJavaElementType { final class JavaDummyElementType extends ILazyParseableElementType implements ICompositeElementType, ParentProviderElementType { private static final Set PARENT_ELEMENT_TYPES = Collections.singleton(BASIC_DUMMY_ELEMENT); private final Function javaDocLexer; - private final Function javaLexer; + private final Function javaLexer; public JavaDummyElementType(@NotNull Function lexer, - @NotNull Function javaLexer) { + @NotNull Function javaLexer) { super("DUMMY_ELEMENT", JavaLanguage.INSTANCE); javaDocLexer = lexer; this.javaLexer = javaLexer; diff --git a/java/java-psi-impl/src/com/intellij/lang/java/JavaParserDefinition.java b/java/java-psi-impl/src/com/intellij/lang/java/JavaParserDefinition.java index 6a42b02016c8..a3a758eed7b3 100644 --- a/java/java-psi-impl/src/com/intellij/lang/java/JavaParserDefinition.java +++ b/java/java-psi-impl/src/com/intellij/lang/java/JavaParserDefinition.java @@ -7,6 +7,7 @@ import com.intellij.lang.ParserDefinition; import com.intellij.lang.PsiParser; import com.intellij.lang.java.lexer.JavaDocLexer; import com.intellij.lang.java.lexer.JavaLexer; +import com.intellij.lang.java.lexer.JavaTypeEscapeLexer; import com.intellij.lexer.Lexer; import com.intellij.openapi.project.Project; import com.intellij.openapi.roots.LanguageLevelProjectExtension; @@ -37,6 +38,11 @@ public class JavaParserDefinition implements ParserDefinition { return new JavaLexer(level); } + /** @return A lexer which handles JEP-467 bracket escapes when parsing Java types */ + public static @NotNull Lexer createLexerWithMarkdownEscape(@NotNull LanguageLevel level) { + return new JavaTypeEscapeLexer(new JavaLexer(level)); + } + public static @NotNull Lexer createDocLexer(@NotNull LanguageLevel level) { return new JavaDocLexer(level); } diff --git a/java/java-psi-impl/src/com/intellij/psi/impl/source/tree/JavaDocElementType.java b/java/java-psi-impl/src/com/intellij/psi/impl/source/tree/JavaDocElementType.java index a3ac5f532ac0..d9a93d98dda9 100644 --- a/java/java-psi-impl/src/com/intellij/psi/impl/source/tree/JavaDocElementType.java +++ b/java/java-psi-impl/src/com/intellij/psi/impl/source/tree/JavaDocElementType.java @@ -5,6 +5,7 @@ import com.intellij.lang.ASTNode; import com.intellij.lang.java.JavaParserDefinition; import com.intellij.lang.java.lexer.BasicJavaLexer; import com.intellij.lang.java.lexer.JavaDocLexer; +import com.intellij.lang.java.lexer.JavaTypeEscapeLexer; import com.intellij.lang.java.parser.JavaParser; import com.intellij.openapi.roots.LanguageLevelProjectExtension; import com.intellij.pom.java.LanguageLevel; @@ -68,7 +69,7 @@ public interface JavaDocElementType { ILazyParseableElementType DOC_TYPE_HOLDER = new BasicJavaDocElementType.DocTypeHolderElementType( () -> JavaParser.INSTANCE, (level) -> (JavaDocLexer)JavaParserDefinition.createDocLexer(level), - (level) -> (BasicJavaLexer)JavaParserDefinition.createLexer(level) + (level) -> (JavaTypeEscapeLexer)JavaParserDefinition.createLexerWithMarkdownEscape(level) ); ILazyParseableElementType DOC_COMMENT = new BasicJavaDocElementType.DocCommentElementType( diff --git a/java/java-psi-impl/src/com/intellij/psi/impl/source/tree/JavaElementType.java b/java/java-psi-impl/src/com/intellij/psi/impl/source/tree/JavaElementType.java index 61925d995e3c..fc9983cb6add 100644 --- a/java/java-psi-impl/src/com/intellij/psi/impl/source/tree/JavaElementType.java +++ b/java/java-psi-impl/src/com/intellij/psi/impl/source/tree/JavaElementType.java @@ -275,6 +275,6 @@ public interface JavaElementType { BASIC_TYPE_WITH_CONJUNCTIONS_TEXT); IElementType DUMMY_ELEMENT = new JavaDummyElementType(level -> (JavaDocLexer)JavaParserDefinition.createDocLexer(level), - level -> (BasicJavaLexer)JavaParserDefinition.createLexer(level) + level -> JavaParserDefinition.createLexerWithMarkdownEscape(level) ); } \ No newline at end of file diff --git a/java/java-tests/testData/psi/parser-full/javadocParsing/ReferenceLinkMarkdown11.java b/java/java-tests/testData/psi/parser-full/javadocParsing/ReferenceLinkMarkdown11.java index 7f8e6dcb259e..d023c07b8d00 100644 --- a/java/java-tests/testData/psi/parser-full/javadocParsing/ReferenceLinkMarkdown11.java +++ b/java/java-tests/testData/psi/parser-full/javadocParsing/ReferenceLinkMarkdown11.java @@ -1,2 +1,2 @@ -/// [java.util.HashMap#toString()] +/// [java.lang.String#copyValueOf(char\[\])] class C{} \ No newline at end of file diff --git a/java/java-tests/testData/psi/parser-full/javadocParsing/ReferenceLinkMarkdown11.txt b/java/java-tests/testData/psi/parser-full/javadocParsing/ReferenceLinkMarkdown11.txt index d3dec27f57fd..d50adb983518 100644 --- a/java/java-tests/testData/psi/parser-full/javadocParsing/ReferenceLinkMarkdown11.txt +++ b/java/java-tests/testData/psi/parser-full/javadocParsing/ReferenceLinkMarkdown11.txt @@ -9,25 +9,30 @@ PsiJavaFile:ReferenceLinkMarkdown11.java PsiDocToken:DOC_LBRACKET('[') PsiElement(DOC_METHOD_OR_FIELD_REF) PsiElement(DOC_REFERENCE_HOLDER) - PsiJavaCodeReferenceElement:java.util.HashMap - PsiJavaCodeReferenceElement:java.util + PsiJavaCodeReferenceElement:java.lang.String + PsiJavaCodeReferenceElement:java.lang PsiJavaCodeReferenceElement:java PsiIdentifier:java('java') PsiReferenceParameterList PsiJavaToken:DOT('.') - PsiIdentifier:util('util') + PsiIdentifier:lang('lang') PsiReferenceParameterList PsiJavaToken:DOT('.') - PsiIdentifier:HashMap('HashMap') + PsiIdentifier:String('String') PsiReferenceParameterList PsiDocToken:DOC_TAG_VALUE_SHARP_TOKEN('#') - PsiDocToken:DOC_TAG_VALUE_TOKEN('toString') + PsiDocToken:DOC_TAG_VALUE_TOKEN('copyValueOf') PsiDocToken:DOC_LPAREN('(') PsiElement(DOC_TAG_VALUE_ELEMENT) - + PsiElement(DOC_TYPE_HOLDER) + PsiTypeElement:char\[\] + PsiTypeElement:char + PsiKeyword:char('char') + PsiJavaToken:LBRACKET('\[') + PsiJavaToken:RBRACKET('\]') PsiDocToken:DOC_RPAREN(')') PsiDocToken:DOC_RBRACKET(']') PsiWhiteSpace('\n') diff --git a/java/java-tests/testData/psi/parser-full/javadocParsing/ReferenceLinkMarkdown11_node.txt b/java/java-tests/testData/psi/parser-full/javadocParsing/ReferenceLinkMarkdown11_node.txt index cd2526b26987..c76446f52662 100644 --- a/java/java-tests/testData/psi/parser-full/javadocParsing/ReferenceLinkMarkdown11_node.txt +++ b/java/java-tests/testData/psi/parser-full/javadocParsing/ReferenceLinkMarkdown11_node.txt @@ -27,7 +27,12 @@ java.FILE DOC_TAG_VALUE_TOKEN DOC_LPAREN DOC_TAG_VALUE_ELEMENT - + DOC_TYPE_HOLDER + TYPE + TYPE + CHAR_KEYWORD + LBRACKET + RBRACKET DOC_RPAREN DOC_RBRACKET WHITE_SPACE