diff --git a/plugins/ant/tests/src/com/intellij/lang/ant/AntParsingTest.java b/plugins/ant/tests/src/com/intellij/lang/ant/AntParsingTest.java index 0a48f7de237b..be806dd54e3c 100644 --- a/plugins/ant/tests/src/com/intellij/lang/ant/AntParsingTest.java +++ b/plugins/ant/tests/src/com/intellij/lang/ant/AntParsingTest.java @@ -6,10 +6,14 @@ import com.intellij.lang.LanguageASTFactory; import com.intellij.lang.xml.XMLLanguage; import com.intellij.lang.xml.XMLParserDefinition; import com.intellij.lang.xml.XmlASTFactory; +import com.intellij.lang.xml.XmlSyntaxDefinitionExtension; import com.intellij.openapi.application.PluginPathManager; import com.intellij.openapi.extensions.ExtensionPointName; import com.intellij.openapi.util.SystemInfo; +import com.intellij.platform.syntax.psi.ElementTypeConverters; +import com.intellij.platform.syntax.psi.LanguageSyntaxDefinitions; import com.intellij.psi.xml.StartTagEndTokenProvider; +import com.intellij.psi.xml.XmlElementTypeConverterExtension; import com.intellij.testFramework.ParsingTestCase; import static com.intellij.xml.testFramework.XmlElementTypeServiceHelper.registerXmlElementTypeServices; @@ -25,6 +29,8 @@ public class AntParsingTest extends ParsingTestCase { super.setUp(); registerXmlElementTypeServices(getApplication(), getTestRootDisposable()); addExplicitExtension(LanguageASTFactory.INSTANCE, XMLLanguage.INSTANCE, new XmlASTFactory()); + addExplicitExtension(LanguageSyntaxDefinitions.getINSTANCE(), XMLLanguage.INSTANCE, new XmlSyntaxDefinitionExtension()); + addExplicitExtension(ElementTypeConverters.getInstance(), XMLLanguage.INSTANCE, new XmlElementTypeConverterExtension()); registerExtensionPoint(new ExtensionPointName<>("com.intellij.xml.startTagEndToken"), StartTagEndTokenProvider.class); } diff --git a/xml/testFramework/BUILD.bazel b/xml/testFramework/BUILD.bazel index 00efe255f390..84b7de864f77 100644 --- a/xml/testFramework/BUILD.bazel +++ b/xml/testFramework/BUILD.bazel @@ -14,6 +14,7 @@ jvm_library( "//xml/xml-parser:parser", "//platform/core-impl", "//tools/intellij.tools.ide.metrics.benchmark:ide-metrics-benchmark", + "//platform/syntax/syntax-psi:psi", ] ) ### auto-generated section `build intellij.xml.testFramework` end \ No newline at end of file diff --git a/xml/testFramework/intellij.xml.testFramework.iml b/xml/testFramework/intellij.xml.testFramework.iml index 5f8c9e3beabb..24936b5400ae 100644 --- a/xml/testFramework/intellij.xml.testFramework.iml +++ b/xml/testFramework/intellij.xml.testFramework.iml @@ -14,6 +14,7 @@ + diff --git a/xml/testFramework/src/com/intellij/xml/testFramework/XmlParsingTestBase.java b/xml/testFramework/src/com/intellij/xml/testFramework/XmlParsingTestBase.java index 339b2a5bf50c..f79d1405d7b4 100644 --- a/xml/testFramework/src/com/intellij/xml/testFramework/XmlParsingTestBase.java +++ b/xml/testFramework/src/com/intellij/xml/testFramework/XmlParsingTestBase.java @@ -6,8 +6,11 @@ import com.intellij.lang.html.HTMLLanguage; import com.intellij.lang.html.HTMLParserDefinition; import com.intellij.lang.xml.XMLLanguage; import com.intellij.lang.xml.XmlASTFactory; +import com.intellij.lang.xml.XmlSyntaxDefinitionExtension; import com.intellij.lexer.EmbeddedTokenTypesProvider; import com.intellij.lexer.Lexer; +import com.intellij.platform.syntax.psi.ElementTypeConverters; +import com.intellij.platform.syntax.psi.LanguageSyntaxDefinitions; import com.intellij.psi.PsiFile; import com.intellij.psi.PsiManager; import com.intellij.psi.impl.source.tree.LeafElement; @@ -50,6 +53,8 @@ public abstract class XmlParsingTestBase extends ParsingTestCase { registerXmlElementTypeServices(getApplication(), getTestRootDisposable()); addExplicitExtension(LanguageASTFactory.INSTANCE, XMLLanguage.INSTANCE, new XmlASTFactory()); registerExtensionPoint(StartTagEndTokenProvider.EP_NAME, StartTagEndTokenProvider.class); + addExplicitExtension(LanguageSyntaxDefinitions.getINSTANCE(), XMLLanguage.INSTANCE, new XmlSyntaxDefinitionExtension()); + addExplicitExtension(ElementTypeConverters.getInstance(), XMLLanguage.INSTANCE, new XmlElementTypeConverterExtension()); } protected void doTestXml(@NonNls String text) throws Exception { diff --git a/xml/tests/intellij.xml.tests.iml b/xml/tests/intellij.xml.tests.iml index c488a0ecab06..8e0e9b058028 100644 --- a/xml/tests/intellij.xml.tests.iml +++ b/xml/tests/intellij.xml.tests.iml @@ -46,6 +46,8 @@ + + diff --git a/xml/tests/src/com/intellij/syntax/XmlSyntaxDefaultResourceTest.kt b/xml/tests/src/com/intellij/syntax/XmlSyntaxDefaultResourceTest.kt new file mode 100644 index 000000000000..9ae43a5ec3e7 --- /dev/null +++ b/xml/tests/src/com/intellij/syntax/XmlSyntaxDefaultResourceTest.kt @@ -0,0 +1,19 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.syntax + +import com.intellij.platform.syntax.scripts.assertPropertiesMatch +import com.intellij.xml.syntax.DefaultXmlSyntaxResourcesTestAccessor +import com.intellij.xml.syntax.XmlSyntaxBundle +import org.junit.jupiter.api.Test + +class XmlSyntaxDefaultResourceTest { + @Test + fun testResourcesMatch() { + assertPropertiesMatch( + propertiesFileName = XmlSyntaxBundle.BUNDLE, + defaultResourcesFileName = DefaultXmlSyntaxResourcesTestAccessor.defaultJavaSyntaxResourcesName, + classLoader = XmlSyntaxBundle.javaClass.classLoader, + actualMapping = DefaultXmlSyntaxResourcesTestAccessor.mappings, + ) + } +} \ No newline at end of file diff --git a/xml/xml-parser/src/com/intellij/psi/impl/source/parsing/xml/XmlParser.kt b/xml/xml-parser/src/com/intellij/psi/impl/source/parsing/xml/XmlParser.kt index b190859a2fb5..52e0bab327ee 100644 --- a/xml/xml-parser/src/com/intellij/psi/impl/source/parsing/xml/XmlParser.kt +++ b/xml/xml-parser/src/com/intellij/psi/impl/source/parsing/xml/XmlParser.kt @@ -13,7 +13,12 @@ import com.intellij.psi.xml.XmlTokenType import com.intellij.util.ThreeState import com.intellij.util.TripleFunction import com.intellij.util.diff.FlyweightCapableTreeStructure +import org.jetbrains.annotations.ApiStatus +/** + * Use [com.intellij.xml.syntax.XmlParser] instead + */ +@ApiStatus.Obsolete open class XmlParser : PsiParser, LightPsiParser { override fun parse( root: IElementType, @@ -36,7 +41,7 @@ open class XmlParser : PsiParser, LightPsiParser { } // tries to match an old and new XmlTag by name -private val REPARSE_XML_TAG_BY_NAME: TripleFunction, ThreeState> = +internal val REPARSE_XML_TAG_BY_NAME: TripleFunction, ThreeState> = TripleFunction(::reparseXmlTagByName) private fun reparseXmlTagByName( diff --git a/xml/xml-parser/src/com/intellij/psi/impl/source/parsing/xml/XmlParsing.kt b/xml/xml-parser/src/com/intellij/psi/impl/source/parsing/xml/XmlParsing.kt index 1242603b0be8..5078468eb77c 100644 --- a/xml/xml-parser/src/com/intellij/psi/impl/source/parsing/xml/XmlParsing.kt +++ b/xml/xml-parser/src/com/intellij/psi/impl/source/parsing/xml/XmlParsing.kt @@ -46,10 +46,14 @@ import com.intellij.psi.xml.XmlTokenType.XML_TAG_CHARACTERS import com.intellij.psi.xml.XmlTokenType.XML_TAG_END import com.intellij.util.containers.Stack import com.intellij.xml.parsing.XmlParserBundle.message +import org.jetbrains.annotations.ApiStatus -/* +/** + * Use [com.intellij.xml.syntax.XmlParsing] instead. + * * @author max */ +@ApiStatus.Obsolete open class XmlParsing( @JvmField protected val myBuilder: PsiBuilder, diff --git a/xml/xml-parser/src/com/intellij/psi/xml/XmlElementType.kt b/xml/xml-parser/src/com/intellij/psi/xml/XmlElementType.kt index 6124fd31ffd0..1ba3b141906b 100644 --- a/xml/xml-parser/src/com/intellij/psi/xml/XmlElementType.kt +++ b/xml/xml-parser/src/com/intellij/psi/xml/XmlElementType.kt @@ -117,7 +117,7 @@ object XmlElementType { @JvmField val XML_FILE: IFileElementType = - IFileElementType(XMLLanguage.INSTANCE) + XmlFileElementType() @JvmField val XHTML_FILE: IElementType = diff --git a/xml/xml-parser/src/com/intellij/psi/xml/XmlFileElementType.kt b/xml/xml-parser/src/com/intellij/psi/xml/XmlFileElementType.kt new file mode 100644 index 000000000000..aa8f0e2147c8 --- /dev/null +++ b/xml/xml-parser/src/com/intellij/psi/xml/XmlFileElementType.kt @@ -0,0 +1,32 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.psi.xml + +import com.intellij.lang.ASTNode +import com.intellij.lang.xml.XMLLanguage +import com.intellij.platform.syntax.emptySyntaxElementTypeSet +import com.intellij.platform.syntax.psi.PsiSyntaxBuilderFactory +import com.intellij.platform.syntax.psi.registerParse +import com.intellij.psi.PsiElement +import com.intellij.psi.impl.source.parsing.xml.REPARSE_XML_TAG_BY_NAME +import com.intellij.psi.tree.IFileElementType +import com.intellij.xml.syntax.XmlParser + +class XmlFileElementType : IFileElementType(XMLLanguage.INSTANCE) { + override fun doParseContents(chameleon: ASTNode, psi: PsiElement): ASTNode { + val factory = PsiSyntaxBuilderFactory.getInstance() + val builder = factory.createBuilder( + chameleon = chameleon, + lang = language, + text = chameleon.chars, + ) + val syntaxTreeBuilder = builder.getSyntaxTreeBuilder() + + val startTime = System.nanoTime() + syntaxTreeBuilder.enforceCommentTokens(emptySyntaxElementTypeSet()) + builder.setCustomComparator(REPARSE_XML_TAG_BY_NAME) + XmlParser().parse(syntaxTreeBuilder) + val node = builder.getTreeBuilt() + registerParse(builder, XMLLanguage.INSTANCE, System.nanoTime() - startTime) + return node.getFirstChildNode() + } +} \ No newline at end of file diff --git a/xml/xml-psi-impl/BUILD.bazel b/xml/xml-psi-impl/BUILD.bazel index 3e678b98e7cd..8fa435a1dbb2 100644 --- a/xml/xml-psi-impl/BUILD.bazel +++ b/xml/xml-psi-impl/BUILD.bazel @@ -42,6 +42,8 @@ jvm_library( "//platform/markdown-utils", "//platform/polySymbols", "//RegExpSupport:regexp", + "//platform/syntax/syntax-psi:psi", + "//xml/xml-syntax:syntax", ], runtime_deps = [ ":psi-impl_resources", diff --git a/xml/xml-psi-impl/intellij.xml.psi.impl.iml b/xml/xml-psi-impl/intellij.xml.psi.impl.iml index 04a6c6576fd5..b364045c06bd 100644 --- a/xml/xml-psi-impl/intellij.xml.psi.impl.iml +++ b/xml/xml-psi-impl/intellij.xml.psi.impl.iml @@ -34,5 +34,7 @@ + + \ No newline at end of file diff --git a/xml/xml-psi-impl/resources/intellij.xml.psi.impl.xml b/xml/xml-psi-impl/resources/intellij.xml.psi.impl.xml index 9b478cf69fb1..0586df933826 100644 --- a/xml/xml-psi-impl/resources/intellij.xml.psi.impl.xml +++ b/xml/xml-psi-impl/resources/intellij.xml.psi.impl.xml @@ -3,6 +3,7 @@ + @@ -31,6 +32,8 @@ + + diff --git a/xml/xml-psi-impl/src/com/intellij/lang/xml/XmlASTFactory.java b/xml/xml-psi-impl/src/com/intellij/lang/xml/XmlASTFactory.java index 5ffc4c7d4560..af0f2e39cbcc 100644 --- a/xml/xml-psi-impl/src/com/intellij/lang/xml/XmlASTFactory.java +++ b/xml/xml-psi-impl/src/com/intellij/lang/xml/XmlASTFactory.java @@ -7,6 +7,7 @@ package com.intellij.lang.xml; import com.intellij.lang.ASTFactory; import com.intellij.lang.impl.PsiBuilderImpl; +import com.intellij.platform.syntax.psi.ExtraWhitespaces; import com.intellij.psi.impl.source.html.HtmlDocumentImpl; import com.intellij.psi.impl.source.html.HtmlTagImpl; import com.intellij.psi.impl.source.tree.*; @@ -136,6 +137,17 @@ public class XmlASTFactory extends ASTFactory { } static { + /* + XML has a special whitespace kind `XmlTokenType.XML_REAL_WHITE_SPACE`. + On AST construction, a leaf with this token type gets replaced with a plain PsiWhiteSpaceImpl + (which has the plain WHITE_SPACE token type), see `XmlASTFactory.createLeaf`, so XML_REAL_WHITE_SPACE exists during parsing only. + + But when we want to reparse the file, we end up in a situation when a new not-yet-built tree is + compared with the existing tree. And here, we need to deal with inconsistency in token types between XML_REAL_WHITE_SPACE in the new tree and WHITE_SPACE in the old tree. + + To overcome this situation, this hack is introduced + */ PsiBuilderImpl.registerWhitespaceToken(XML_REAL_WHITE_SPACE); + ExtraWhitespaces.registerExtraWhitespace(XML_REAL_WHITE_SPACE); } } diff --git a/xml/xml-psi-impl/src/com/intellij/lang/xml/XmlSyntaxDefinitionExtension.kt b/xml/xml-psi-impl/src/com/intellij/lang/xml/XmlSyntaxDefinitionExtension.kt new file mode 100644 index 000000000000..f52298b04411 --- /dev/null +++ b/xml/xml-psi-impl/src/com/intellij/lang/xml/XmlSyntaxDefinitionExtension.kt @@ -0,0 +1,22 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.lang.xml + +import com.intellij.platform.syntax.LanguageSyntaxDefinition +import com.intellij.platform.syntax.SyntaxElementTypeSet +import com.intellij.platform.syntax.lexer.Lexer +import com.intellij.platform.syntax.parser.SyntaxTreeBuilder +import com.intellij.xml.syntax.XmlParser +import com.intellij.xml.syntax.XmlSyntaxDefinition + +class XmlSyntaxDefinitionExtension : LanguageSyntaxDefinition { + fun getWhitespaceTokens(): SyntaxElementTypeSet = XmlSyntaxDefinition.WHITESPACES + + override fun parse(builder: SyntaxTreeBuilder) { + XmlParser().parse(builder) + } + + override fun createLexer(): Lexer = XmlSyntaxDefinition.createLexer() + + override val comments: SyntaxElementTypeSet + get() = XmlSyntaxDefinition.COMMENTS +} \ No newline at end of file diff --git a/xml/xml-syntax/gen/com/intellij/xml/syntax/DefaultXmlSyntaxResources.kt b/xml/xml-syntax/gen/com/intellij/xml/syntax/DefaultXmlSyntaxResources.kt new file mode 100644 index 000000000000..2cb735a615a1 --- /dev/null +++ b/xml/xml-syntax/gen/com/intellij/xml/syntax/DefaultXmlSyntaxResources.kt @@ -0,0 +1,29 @@ +package com.intellij.xml.syntax + +// Generated by generateBundleMapping.kt +// To regenerate the file, please run `GenerateBundleMapping` run configuration with the following parameters: +// com.intellij.xml.syntax +internal object DefaultXmlSyntaxResources { + val mappings: Map = mapOf( + "xml.parsing.absent.root.tag" to "Valid XML document must have a root tag", + "xml.parsing.attribute.value.expected" to "Attribute value expected", + "xml.parsing.bad.character" to "Bad character", + "xml.parsing.closing.tag.is.not.done" to "Closing tag is not valid", + "xml.parsing.closing.tag.matches.nothing" to "Closing tag matches nothing", + "xml.parsing.closing.tag.name.missing" to "Closing tag name is missing", + "xml.parsing.expected.attribute.eq.sign" to "= expected", + "xml.parsing.multiple.root.tags" to "Multiple root tags", + "xml.parsing.named.element.is.not.closed" to "Element {0} is not closed", + "xml.parsing.processing.instruction.name.expected" to "Processing instruction name expected", + "xml.parsing.tag.name.expected" to "Tag name expected", + "xml.parsing.tag.start.is.not.closed" to "Tag start is not closed", + "xml.parsing.top.level.element.is.not.completed" to "Top level element is not completed", + "xml.parsing.unclosed.attribute.value" to "Attribute value is not closed", + "xml.parsing.unescaped.ampersand.or.nonterminated.character.entity.reference" to "Unescaped \\& or nonterminated character/entity reference", + "xml.parsing.unexpected.end.of.file" to "Unexpected end of file", + "xml.parsing.unexpected.token" to "Unexpected token", + "xml.parsing.unexpected.tokens" to "Unexpected tokens", + "xml.parsing.unterminated.processing.instruction" to "Processing instruction not terminated", + "xml.parsing.way.too.unbalanced" to "Way too unbalanced. Stopping attempt to balance tags properly at this point", + ) +} diff --git a/xml/xml-syntax/resources/messages/XmlSyntaxBundle.properties b/xml/xml-syntax/resources/messages/XmlSyntaxBundle.properties new file mode 100644 index 000000000000..b76a5c44d030 --- /dev/null +++ b/xml/xml-syntax/resources/messages/XmlSyntaxBundle.properties @@ -0,0 +1,20 @@ +xml.parsing.absent.root.tag=Valid XML document must have a root tag +xml.parsing.attribute.value.expected=Attribute value expected +xml.parsing.bad.character=Bad character +xml.parsing.closing.tag.is.not.done=Closing tag is not valid +xml.parsing.closing.tag.matches.nothing=Closing tag matches nothing +xml.parsing.closing.tag.name.missing=Closing tag name is missing +xml.parsing.expected.attribute.eq.sign== expected +xml.parsing.named.element.is.not.closed=Element {0} is not closed +xml.parsing.multiple.root.tags=Multiple root tags +xml.parsing.processing.instruction.name.expected=Processing instruction name expected +xml.parsing.tag.name.expected=Tag name expected +xml.parsing.tag.start.is.not.closed=Tag start is not closed +xml.parsing.top.level.element.is.not.completed=Top level element is not completed +xml.parsing.unclosed.attribute.value=Attribute value is not closed +xml.parsing.unescaped.ampersand.or.nonterminated.character.entity.reference=Unescaped \\& or nonterminated character/entity reference +xml.parsing.unexpected.end.of.file=Unexpected end of file +xml.parsing.unexpected.token=Unexpected token +xml.parsing.unexpected.tokens=Unexpected tokens +xml.parsing.unterminated.processing.instruction=Processing instruction not terminated +xml.parsing.way.too.unbalanced=Way too unbalanced. Stopping attempt to balance tags properly at this point diff --git a/xml/xml-syntax/src/com/intellij/xml/syntax/XmlParser.kt b/xml/xml-syntax/src/com/intellij/xml/syntax/XmlParser.kt new file mode 100644 index 000000000000..56930ab19e77 --- /dev/null +++ b/xml/xml-syntax/src/com/intellij/xml/syntax/XmlParser.kt @@ -0,0 +1,12 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.xml.syntax + +import com.intellij.platform.syntax.parser.SyntaxTreeBuilder + +class XmlParser { + fun parse(builder: SyntaxTreeBuilder) { + val file = builder.mark() + XmlParsing(builder).parseDocument() + file.done(XmlSyntaxElementType.XML_FILE) + } +} \ No newline at end of file diff --git a/xml/xml-syntax/src/com/intellij/xml/syntax/XmlParsing.kt b/xml/xml-syntax/src/com/intellij/xml/syntax/XmlParsing.kt new file mode 100644 index 000000000000..81dfa8ee532e --- /dev/null +++ b/xml/xml-syntax/src/com/intellij/xml/syntax/XmlParsing.kt @@ -0,0 +1,501 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.xml.syntax + +import com.intellij.openapi.util.NlsContexts +import com.intellij.platform.syntax.SyntaxElementType +import com.intellij.platform.syntax.element.SyntaxTokenTypes +import com.intellij.platform.syntax.parser.SyntaxTreeBuilder +import com.intellij.xml.syntax.XmlSyntaxElementType.XML_ATTRIBUTE +import com.intellij.xml.syntax.XmlSyntaxElementType.XML_ATTRIBUTE_VALUE +import com.intellij.xml.syntax.XmlSyntaxElementType.XML_CDATA +import com.intellij.xml.syntax.XmlSyntaxElementType.XML_COMMENT +import com.intellij.xml.syntax.XmlSyntaxElementType.XML_DOCTYPE +import com.intellij.xml.syntax.XmlSyntaxElementType.XML_DOCUMENT +import com.intellij.xml.syntax.XmlSyntaxElementType.XML_ENTITY_REF +import com.intellij.xml.syntax.XmlSyntaxElementType.XML_PROCESSING_INSTRUCTION +import com.intellij.xml.syntax.XmlSyntaxElementType.XML_PROLOG +import com.intellij.xml.syntax.XmlSyntaxElementType.XML_TAG +import com.intellij.xml.syntax.XmlSyntaxElementType.XML_TEXT +import com.intellij.xml.syntax.XmlSyntaxTokenType.XML_ATTRIBUTE_VALUE_END_DELIMITER +import com.intellij.xml.syntax.XmlSyntaxTokenType.XML_ATTRIBUTE_VALUE_START_DELIMITER +import com.intellij.xml.syntax.XmlSyntaxTokenType.XML_BAD_CHARACTER +import com.intellij.xml.syntax.XmlSyntaxTokenType.XML_CDATA_END +import com.intellij.xml.syntax.XmlSyntaxTokenType.XML_CDATA_START +import com.intellij.xml.syntax.XmlSyntaxTokenType.XML_CHAR_ENTITY_REF +import com.intellij.xml.syntax.XmlSyntaxTokenType.XML_COMMENT_CHARACTERS +import com.intellij.xml.syntax.XmlSyntaxTokenType.XML_COMMENT_END +import com.intellij.xml.syntax.XmlSyntaxTokenType.XML_COMMENT_START +import com.intellij.xml.syntax.XmlSyntaxTokenType.XML_CONDITIONAL_COMMENT_END +import com.intellij.xml.syntax.XmlSyntaxTokenType.XML_CONDITIONAL_COMMENT_END_START +import com.intellij.xml.syntax.XmlSyntaxTokenType.XML_CONDITIONAL_COMMENT_START +import com.intellij.xml.syntax.XmlSyntaxTokenType.XML_CONDITIONAL_COMMENT_START_END +import com.intellij.xml.syntax.XmlSyntaxTokenType.XML_DOCTYPE_END +import com.intellij.xml.syntax.XmlSyntaxTokenType.XML_DOCTYPE_START +import com.intellij.xml.syntax.XmlSyntaxTokenType.XML_EMPTY_ELEMENT_END +import com.intellij.xml.syntax.XmlSyntaxTokenType.XML_END_TAG_START +import com.intellij.xml.syntax.XmlSyntaxTokenType.XML_ENTITY_REF_TOKEN +import com.intellij.xml.syntax.XmlSyntaxTokenType.XML_EQ +import com.intellij.xml.syntax.XmlSyntaxTokenType.XML_NAME +import com.intellij.xml.syntax.XmlSyntaxTokenType.XML_PI_END +import com.intellij.xml.syntax.XmlSyntaxTokenType.XML_PI_START +import com.intellij.xml.syntax.XmlSyntaxTokenType.XML_REAL_WHITE_SPACE +import com.intellij.xml.syntax.XmlSyntaxTokenType.XML_START_TAG_START +import com.intellij.xml.syntax.XmlSyntaxTokenType.XML_TAG_CHARACTERS +import com.intellij.xml.syntax.XmlSyntaxTokenType.XML_TAG_END + +open class XmlParsing( + @JvmField protected val myBuilder: SyntaxTreeBuilder, +) { + private val tagNamesStack = ArrayDeque() + + fun parseDocument() { + val document = mark() + + while (isCommentToken(token())) { + parseComment() + } + + parseProlog() + + var rootTagCount = 0 + var error: SyntaxTreeBuilder.Marker? = null + + fun flushError() { + error?.let { + it.error(XmlSyntaxBundle.message("xml.parsing.unexpected.tokens")) + error = null + } + } + + while (!eof()) { + val tt = token() + when { + tt === XML_START_TAG_START -> { + flushError() + rootTagCount++ + parseTag(rootTagCount > 1) + } + isCommentToken(tt) -> { + flushError() + parseComment() + } + tt === XML_PI_START -> { + flushError() + parseProcessingInstruction() + } + tt === XML_REAL_WHITE_SPACE -> { + flushError() + advance() + } + else -> { + if (error == null) error = mark() + advance() + } + } + } + + error?.error(XmlSyntaxBundle.message("xml.parsing.top.level.element.is.not.completed")) + + if (rootTagCount == 0) { + val rootTag = mark() + mark().error(XmlSyntaxBundle.message("xml.parsing.absent.root.tag")) + rootTag.done(XML_TAG) + } + + document.done(XML_DOCUMENT) + } + + private fun parseDoctype() { + checkCurrentToken(XML_DOCTYPE_START) { "Doctype start expected" } + val doctype = mark() + advance() + + while (token() !== XML_DOCTYPE_END && !eof()) advance() + if (eof()) { + error(XmlSyntaxBundle.message("xml.parsing.unexpected.end.of.file")) + } + else { + advance() + } + + doctype.done(XML_DOCTYPE) + } + + protected fun parseTag(multipleRootTagError: Boolean) { + checkCurrentToken(XML_START_TAG_START) { "Tag start expected" } + val tag = mark() + + val tagName = parseTagHeader(multipleRootTagError, tag) ?: return + + val content = mark() + parseTagContent() + + if (token() === XML_END_TAG_START) { + val footer = mark() + advance() + + if (token() === XML_NAME) { + val endName = myBuilder.tokenText + if (tagName != endName && endName in tagNamesStack) { + footer.rollbackTo() + tagNamesStack.removeLast() + tag.doneBefore(XML_TAG, content, XmlSyntaxBundle.message("xml.parsing.named.element.is.not.closed", tagName)) + content.drop() + return + } + + advance() + } + else { + error(XmlSyntaxBundle.message("xml.parsing.closing.tag.name.missing")) + } + footer.drop() + + while (token() !== XML_TAG_END + && token() !== XML_START_TAG_START + && token() !== XML_END_TAG_START + && !eof() + ) { + error(XmlSyntaxBundle.message("xml.parsing.unexpected.token")) + advance() + } + + if (token() === XML_TAG_END) { + advance() + } + else { + error(XmlSyntaxBundle.message("xml.parsing.closing.tag.is.not.done")) + } + } + else { + error(XmlSyntaxBundle.message("xml.parsing.unexpected.end.of.file")) + } + + content.drop() + tagNamesStack.removeLast() + tag.done(XML_TAG) + } + + private fun parseTagHeader(multipleRootTagError: Boolean, tag: SyntaxTreeBuilder.Marker): String? { + if (multipleRootTagError) { + val error = mark() + advance() + error.error(XmlSyntaxBundle.message("xml.parsing.multiple.root.tags")) + } + else { + advance() + } + + val tagName: String? + if (token() !== XML_NAME || myBuilder.rawLookup(-1) === SyntaxTokenTypes.WHITE_SPACE) { + error(XmlSyntaxBundle.message("xml.parsing.tag.name.expected")) + tagName = "" + } + else { + tagName = myBuilder.tokenText + checkNotNull(tagName) + advance() + } + tagNamesStack.addLast(tagName) + + while (true) { + val tt = token() + when { + tt === XML_NAME -> parseAttribute() + tt === XML_CHAR_ENTITY_REF || tt === XML_ENTITY_REF_TOKEN -> parseReference() + else -> break + } + } + + if (token() === XML_EMPTY_ELEMENT_END) { + advance() + tagNamesStack.removeLast() + tag.done(XML_TAG) + return null + } + + if (token() === XML_TAG_END) { + advance() + } + else { + error(XmlSyntaxBundle.message("xml.parsing.tag.start.is.not.closed")) + tagNamesStack.removeLast() + tag.done(XML_TAG) + return null + } + + if (tagNamesStack.size > BALANCING_DEPTH_THRESHOLD) { + error(XmlSyntaxBundle.message("xml.parsing.way.too.unbalanced")) + tag.done(XML_TAG) + return null + } + + return tagName + } + + fun parseTagContent() { + var xmlText: SyntaxTreeBuilder.Marker? = null + + fun terminateText() { + xmlText?.let { + it.done(XML_TEXT) + xmlText = null + } + } + + fun startText() { + if (xmlText == null) xmlText = mark() + } + + while (true) { + val tt = token() + if (tt == null || tt === XML_END_TAG_START) { + break + } + + if (tt === XML_START_TAG_START) { + terminateText() + parseTag(false) + } + else if (tt === XML_PI_START) { + terminateText() + parseProcessingInstruction() + } + else if (tt === XML_ENTITY_REF_TOKEN) { + terminateText() + parseReference() + } + else if (tt === XML_CHAR_ENTITY_REF) { + startText() + parseReference() + } + else if (tt === XML_CDATA_START) { + startText() + parseCData() + } + else if (isCommentToken(tt)) { + terminateText() + parseComment() + } + else if (tt === XML_BAD_CHARACTER) { + startText() + val error = mark() + advance() + error.error(XmlSyntaxBundle.message("xml.parsing.unescaped.ampersand.or.nonterminated.character.entity.reference")) + } + //else if (tt is ICustomParsingType || tt is ILazyParseableElementType) { TODO figure out this!!! + // terminateText() + // advance() + //} + else { + startText() + advance() + } + } + + terminateText() + } + + protected open fun isCommentToken(tt: SyntaxElementType?): Boolean { + return tt === XML_COMMENT_START + } + + protected fun mark(): SyntaxTreeBuilder.Marker { + return myBuilder.mark() + } + + private fun parseCData() { + checkCurrentToken(XML_CDATA_START) + val cdata = mark() + while (token() !== XML_CDATA_END && !eof()) { + advance() + } + + if (!eof()) { + advance() + } + + cdata.done(XML_CDATA) + } + + protected open fun parseComment() { + val comment = mark() + advance() + while (true) { + val tt = token() + if (tt === XML_COMMENT_CHARACTERS + || tt === XML_CONDITIONAL_COMMENT_START + || tt === XML_CONDITIONAL_COMMENT_START_END + || tt === XML_CONDITIONAL_COMMENT_END_START + || tt === XML_CONDITIONAL_COMMENT_END + ) { + advance() + continue + } + else if (tt === XML_BAD_CHARACTER) { + val error = mark() + advance() + error.error(XmlSyntaxBundle.message("xml.parsing.bad.character")) + continue + } + if (tt === XML_COMMENT_END) { + advance() + } + break + } + comment.done(XML_COMMENT) + } + + private fun parseReference() { + val tt = token() + when { + tt === XML_CHAR_ENTITY_REF -> { + advance() + } + tt === XML_ENTITY_REF_TOKEN -> { + val ref = mark() + advance() + ref.done(XML_ENTITY_REF) + } + else -> { + kotlin.error("Unexpected token: $tt") + } + } + } + + private fun parseAttribute() { + checkCurrentToken(XML_NAME) + val att = mark() + advance() + if (token() === XML_EQ) { + advance() + parseAttributeValue() + } + else { + error(XmlSyntaxBundle.message("xml.parsing.expected.attribute.eq.sign")) + } + att.done(XML_ATTRIBUTE) + } + + private fun parseAttributeValue() { + val attValue = mark() + if (token() === XML_ATTRIBUTE_VALUE_START_DELIMITER) { + while (true) { + val tt = token() + if (tt == null + || tt === XML_ATTRIBUTE_VALUE_END_DELIMITER + || tt === XML_END_TAG_START + || tt === XML_EMPTY_ELEMENT_END + || tt === XML_START_TAG_START + ) { + break + } + + if (tt === XML_BAD_CHARACTER) { + val error = mark() + advance() + error.error(XmlSyntaxBundle.message("xml.parsing.unescaped.ampersand.or.nonterminated.character.entity.reference")) + } + else if (tt === XML_ENTITY_REF_TOKEN) { + parseReference() + } + else { + advance() + } + } + + if (token() === XML_ATTRIBUTE_VALUE_END_DELIMITER) { + advance() + } + else { + error(XmlSyntaxBundle.message("xml.parsing.unclosed.attribute.value")) + } + } + else { + error(XmlSyntaxBundle.message("xml.parsing.attribute.value.expected")) + } + + attValue.done(XML_ATTRIBUTE_VALUE) + } + + private fun parseProlog() { + val prolog = mark() + while (true) { + val tt = token() + when { + tt === XML_PI_START -> parseProcessingInstruction() + tt === XML_DOCTYPE_START -> parseDoctype() + isCommentToken(tt) -> parseComment() + tt === XML_REAL_WHITE_SPACE -> advance() + else -> break + } + } + prolog.done(XML_PROLOG) + } + + private fun parseProcessingInstruction() { + checkCurrentToken(XML_PI_START) + val pi = mark() + advance() + if (token() !== XML_NAME) { + error(XmlSyntaxBundle.message("xml.parsing.processing.instruction.name.expected")) + } + else { + advance() + } + + val tokenType = token() + if (tokenType === XML_TAG_CHARACTERS) { + while (token() === XML_TAG_CHARACTERS) { + advance() + } + } + else { + while (token() === XML_NAME) { + advance() + if (token() === XML_EQ) { + advance() + } + else { + error(XmlSyntaxBundle.message("xml.parsing.expected.attribute.eq.sign")) + } + parseAttributeValue() + } + } + + if (token() === XML_PI_END) { + advance() + } + else { + error(XmlSyntaxBundle.message("xml.parsing.unterminated.processing.instruction")) + } + + pi.done(XML_PROCESSING_INSTRUCTION) + } + + protected fun token(): SyntaxElementType? { + return myBuilder.tokenType + } + + protected fun eof(): Boolean { + return myBuilder.eof() + } + + protected fun advance() { + myBuilder.advanceLexer() + } + + private fun error(message: @NlsContexts.ParsingError String) { + myBuilder.error(message) + } + + private fun checkCurrentToken(type: SyntaxElementType) { + check(token() === type) { "Expected: $type, got: ${token()}" } + } + + private inline fun checkCurrentToken(type: SyntaxElementType, error: () -> String) { + check(token() === type, error) + } +} + +private const val BALANCING_DEPTH_THRESHOLD = 1000 diff --git a/xml/xml-syntax/src/com/intellij/xml/syntax/XmlSyntaxBundle.kt b/xml/xml-syntax/src/com/intellij/xml/syntax/XmlSyntaxBundle.kt new file mode 100644 index 000000000000..2c4105cdb71f --- /dev/null +++ b/xml/xml-syntax/src/com/intellij/xml/syntax/XmlSyntaxBundle.kt @@ -0,0 +1,40 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.xml.syntax + +import com.intellij.platform.syntax.i18n.ResourceBundle +import org.jetbrains.annotations.ApiStatus +import org.jetbrains.annotations.Nls +import org.jetbrains.annotations.NonNls +import org.jetbrains.annotations.PropertyKey +import org.jetbrains.annotations.VisibleForTesting + +object XmlSyntaxBundle { + const val BUNDLE: @NonNls String = "messages.XmlSyntaxBundle" + + val resourceBundle: ResourceBundle = run { + val defaultMapping by lazy { DefaultXmlSyntaxResources.mappings } + ResourceBundle( + bundleClass = "com.intellij.xml.syntax.XmlSyntaxBundle", + pathToBundle = BUNDLE, + self = this, + defaultMapping = defaultMapping + ) + } + + @JvmStatic + fun message(key: @PropertyKey(resourceBundle = BUNDLE) String, vararg params: Any): @Nls String { + return resourceBundle.message(key, *params) + } + + @JvmStatic + fun messagePointer(key: @PropertyKey(resourceBundle = BUNDLE) String, vararg params: Any): () -> @Nls String { + return resourceBundle.messagePointer(key, *params) + } +} + +@VisibleForTesting +@ApiStatus.Internal +object DefaultXmlSyntaxResourcesTestAccessor { + val mappings: Map get() = DefaultXmlSyntaxResources.mappings + val defaultJavaSyntaxResourcesName: String get() = DefaultXmlSyntaxResources::class.qualifiedName!! +} diff --git a/xml/xml-syntax/src/com/intellij/xml/syntax/XmlSyntaxDefinition.kt b/xml/xml-syntax/src/com/intellij/xml/syntax/XmlSyntaxDefinition.kt new file mode 100644 index 000000000000..66c164612c0e --- /dev/null +++ b/xml/xml-syntax/src/com/intellij/xml/syntax/XmlSyntaxDefinition.kt @@ -0,0 +1,21 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.xml.syntax + +import com.intellij.platform.syntax.SyntaxElementTypeSet +import com.intellij.platform.syntax.element.SyntaxTokenTypes +import com.intellij.platform.syntax.lexer.Lexer +import com.intellij.platform.syntax.syntaxElementTypeSetOf +import com.intellij.xml.syntax.lexer.XmlLexer + +object XmlSyntaxDefinition { + val WHITESPACES: SyntaxElementTypeSet = syntaxElementTypeSetOf(SyntaxTokenTypes.WHITE_SPACE) + + val COMMENTS: SyntaxElementTypeSet = syntaxElementTypeSetOf( + XmlSyntaxTokenType.XML_COMMENT_START, + XmlSyntaxTokenType.XML_COMMENT_CHARACTERS, + XmlSyntaxTokenType.XML_COMMENT_END, + ) + + @JvmStatic + fun createLexer(): Lexer = XmlLexer() +} \ No newline at end of file