From 752c180eb29ebdf5e44303c0b628004f1461c82b Mon Sep 17 00:00:00 2001 From: "Codrin.Ogreanu" Date: Wed, 23 Apr 2025 15:12:43 +0200 Subject: [PATCH] IJPL-163132 Bump to JFlex 1.10.14 Also added `offsetByCodePoints` function to `idea-flex-kotlin.skeleton`, and removed unnecessary `;` symbols from `idea-flex-kotlin.skeleton` GitOrigin-RevId: f721d1cd1e687873c3b9d47e233cfff89ddc09a8 --- .../java/syntax/lexer/_JavaDocLexer.kt | 343 +++++++------ .../intellij/java/syntax/lexer/_JavaLexer.kt | 470 ++++++++++-------- tools/lexer/idea-flex-kotlin.skeleton | 49 +- 3 files changed, 478 insertions(+), 384 deletions(-) diff --git a/java/java-syntax/gen/com/intellij/java/syntax/lexer/_JavaDocLexer.kt b/java/java-syntax/gen/com/intellij/java/syntax/lexer/_JavaDocLexer.kt index 837d6e4b666b..6764d773c971 100644 --- a/java/java-syntax/gen/com/intellij/java/syntax/lexer/_JavaDocLexer.kt +++ b/java/java-syntax/gen/com/intellij/java/syntax/lexer/_JavaDocLexer.kt @@ -1,6 +1,6 @@ // Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. -// Generated by JFlex 1.10.13 http://jflex.de/ (tweaked for IntelliJ platform) +// Generated by JFlex 1.10.14 http://jflex.de/ (tweaked for IntelliJ platform) // source: _JavaDocLexer.flex /* It's an automatically generated code. Do not modify it. */ @@ -15,7 +15,7 @@ import com.intellij.platform.syntax.util.lexer.FlexLexer class _JavaDocLexer : FlexLexer { companion object { /** This character denotes the end of file */ - private const val YYEOF = -1; + private const val YYEOF = -1 /** initial size of the lookahead buffer */ private const val ZZ_BUFFERSIZE: Int = 16384 @@ -72,29 +72,29 @@ companion object { "\u0001\u6200\u0001\u6300\u0001\u6400\u0009\u1f00\u0001\u6500\u0001\u6600\u0005\u1f00\u0001\u6700"+ "\u0001\u6800\u0002\u1f00\u0001\u6900\u0001\u1f00\u0001\u6a00\u0011\u1f00\u00a6\u0100\u0001\u6b00"+ "\u0010\u0100\u0001\u6c00\u0001\u6d00\u0015\u0100\u0001\u6e00\u001c\u0100\u0001\u6f00\u000c\u1f00"+ - "\u0002\u0100\u0001\u7000\u0005\u1f00\u0013\u0100\u0001\u7100\u0dec\u1f00"; + "\u0002\u0100\u0001\u7000\u0005\u1f00\u0013\u0100\u0001\u7100\u0dec\u1f00" @JvmStatic private fun zzUnpackcmap_top(): IntArray { - val result: IntArray = IntArray(4352); - var offset: Int = 0; - offset = zzUnpackcmap_top(ZZ_CMAP_TOP_PACKED_0, offset, result); - return result; + val result: IntArray = IntArray(4352) + var offset: Int = 0 + offset = zzUnpackcmap_top(ZZ_CMAP_TOP_PACKED_0, offset, result) + return result } @JvmStatic private fun zzUnpackcmap_top(packed: String , offset: Int, result: IntArray): Int { - var i: Int = 0; /* index in packed string */ - var j: Int = offset; /* index in unpacked array */ - val l: Int = packed.length; + var i: Int = 0 /* index in packed string */ + var j: Int = offset /* index in unpacked array */ + val l: Int = packed.length while (i < l) { - var count: Int = packed[i++].code; - var value: Int = packed[i++].code; + var count: Int = packed[i++].code + var value: Int = packed[i++].code do result[j++] = value while (--count > 0) } - return j; + return j } @JvmStatic @@ -272,29 +272,29 @@ companion object { "\u0001\u0000\u0004\u0008\u0001\u0000\u0004\u0008\u0001\u0000\u0001\u0008\u0001\u0000\u000a\u0008"+ "\u0001\u0000\u0011\u0008\u0005\u0000\u0003\u0008\u0001\u0000\u0005\u0008\u0001\u0000\u0011\u0008"+ "\u0044\u0000\u00de\u0008\u0022\u0000\u0035\u0008\u000b\u0000\u00de\u0008\u0002\u0000\u0182\u0008"+ - "\u000e\u0000\u0131\u0008\u001f\u0000\u001e\u0008\u00e2\u0000\u004b\u0008\u00b5\u0000"; + "\u000e\u0000\u0131\u0008\u001f\u0000\u001e\u0008\u00e2\u0000\u004b\u0008\u00b5\u0000" @JvmStatic private fun zzUnpackcmap_blocks(): IntArray { - val result: IntArray = IntArray(29184); - var offset: Int = 0; - offset = zzUnpackcmap_blocks(ZZ_CMAP_BLOCKS_PACKED_0, offset, result); - return result; + val result: IntArray = IntArray(29184) + var offset: Int = 0 + offset = zzUnpackcmap_blocks(ZZ_CMAP_BLOCKS_PACKED_0, offset, result) + return result } @JvmStatic private fun zzUnpackcmap_blocks(packed: String , offset: Int, result: IntArray): Int { - var i: Int = 0; /* index in packed string */ - var j: Int = offset; /* index in unpacked array */ - val l: Int = packed.length; + var i: Int = 0 /* index in packed string */ + var j: Int = offset /* index in unpacked array */ + val l: Int = packed.length while (i < l) { - var count: Int = packed[i++].code; - var value: Int = packed[i++].code; + var count: Int = packed[i++].code + var value: Int = packed[i++].code do result[j++] = value while (--count > 0) } - return j; + return j } @JvmStatic @@ -314,29 +314,29 @@ companion object { "\u0002\u0024\u0002\u0000\u0005\u0024\u0001\u0000\u0001\u0025\u0001\u0026\u0001\u0024\u0002\u0027"+ "\u0004\u0024\u0001\u0028\u0001\u0024\u0002\u0027\u0005\u0024\u0002\u0027\u0001\u0029\u0003\u0024"+ "\u0001\u002a\u0002\u0027\u0001\u0024\u0001\u002a\u0001\u0024\u0002\u0027\u0002\u0024\u0002\u0027"+ - "\u0001\u002b\u0017\u0027"; + "\u0001\u002b\u0017\u0027" @JvmStatic private fun zzUnpackAction(): IntArray { - val result: IntArray = IntArray(144); - var offset: Int = 0; - offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result); - return result; + val result: IntArray = IntArray(144) + var offset: Int = 0 + offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result) + return result } @JvmStatic private fun zzUnpackAction(packed: String , offset: Int, result: IntArray): Int { - var i: Int = 0; /* index in packed string */ - var j: Int = offset; /* index in unpacked array */ - val l: Int = packed.length; + var i: Int = 0 /* index in packed string */ + var j: Int = offset /* index in unpacked array */ + val l: Int = packed.length while (i < l) { - var count: Int = packed[i++].code; - var value: Int = packed[i++].code; + var count: Int = packed[i++].code + var value: Int = packed[i++].code do result[j++] = value while (--count > 0) } - return j; + return j } @JvmStatic @@ -364,26 +364,26 @@ companion object { "\u0000\u078f\u0000\u0c6e\u0000\u0c99\u0000\u0cc4\u0000\u0cef\u0000\u0d1a\u0000\u0d45\u0000\u0d70"+ "\u0000\u078f\u0000\u0d9b\u0000\u0dc6\u0000\u0df1\u0000\u0e1c\u0000\u0e47\u0000\u0e72\u0000\u0e9d"+ "\u0000\u0ec8\u0000\u0ef3\u0000\u0f1e\u0000\u0f49\u0000\u0f74\u0000\u0f9f\u0000\u0fca\u0000\u0ff5"+ - "\u0000\u1020\u0000\u104b\u0000\u1076\u0000\u10a1\u0000\u10cc\u0000\u10f7\u0000\u1122\u0000\u0285"; + "\u0000\u1020\u0000\u104b\u0000\u1076\u0000\u10a1\u0000\u10cc\u0000\u10f7\u0000\u1122\u0000\u0285" @JvmStatic private fun zzUnpackRowMap(): IntArray { - val result: IntArray = IntArray(144); - var offset: Int = 0; - offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result); - return result; + val result: IntArray = IntArray(144) + var offset: Int = 0 + offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result) + return result } @JvmStatic private fun zzUnpackRowMap(packed: String, offset: Int, result: IntArray): Int { - var i: Int = 0; /* index in packed string */ - var j: Int = offset; /* index in unpacked array */ - val l: Int = packed.length - 1; + var i: Int = 0 /* index in packed string */ + var j: Int = offset /* index in unpacked array */ + val l: Int = packed.length - 1 while (i < l) { - val high: Int = packed[i++].code shl 16; - result[j++] = high or packed[i++].code; + val high: Int = packed[i++].code shl 16 + result[j++] = high or packed[i++].code } - return j; + return j } @JvmStatic @@ -467,30 +467,30 @@ companion object { "\u003a\u0000\u0001\u0083\u001a\u0000\u0001\u0084\u003a\u0000\u0001\u0085\u001a\u0000\u0001\u0086"+ "\u003a\u0000\u0001\u0087\u001a\u0000\u0001\u0088\u003a\u0000\u0001\u0089\u001a\u0000\u0001\u008a"+ "\u003a\u0000\u0001\u008b\u001a\u0000\u0001\u008c\u003a\u0000\u0001\u008d\u001a\u0000\u0001\u008e"+ - "\u003a\u0000\u0001\u008f\u001a\u0000\u0001\u0090\u003a\u0000\u0001\u0090"; + "\u003a\u0000\u0001\u008f\u001a\u0000\u0001\u0090\u003a\u0000\u0001\u0090" @JvmStatic private fun zzUnpacktrans(): IntArray { - val result: IntArray = IntArray(4429); - var offset: Int = 0; - offset = zzUnpacktrans(ZZ_TRANS_PACKED_0, offset, result); - return result; + val result: IntArray = IntArray(4429) + var offset: Int = 0 + offset = zzUnpacktrans(ZZ_TRANS_PACKED_0, offset, result) + return result } @JvmStatic private fun zzUnpacktrans(packed: String , offset: Int, result: IntArray): Int { - var i: Int = 0; /* index in packed string */ - var j: Int = offset; /* index in unpacked array */ - val l: Int = packed.length; + var i: Int = 0 /* index in packed string */ + var j: Int = offset /* index in unpacked array */ + val l: Int = packed.length while (i < l) { - var count: Int = packed[i++].code; - var value: Int = packed[i++].code; - value--; + var count: Int = packed[i++].code + var value: Int = packed[i++].code + value-- do result[j++] = value while (--count > 0) } - return j; + return j } @JvmStatic @@ -519,29 +519,29 @@ companion object { "\u0001\u0001\u0005\u0009\u0001\u0001\u0001\u0009\u0001\u0001\u0001\u0009\u0003\u0001\u0002\u0009"+ "\u0001\u0001\u0001\u0009\u0002\u0001\u0002\u0009\u0002\u0001\u0002\u0009\u0002\u0001\u0001\u0009"+ "\u0004\u0001\u0001\u0000\u0001\u0009\u0002\u0000\u0002\u0001\u0002\u0000\u0005\u0001\u0001\u0000"+ - "\u0002\u0009\u0007\u0001\u0001\u0009\u0031\u0001\u0001\u0009"; + "\u0002\u0009\u0007\u0001\u0001\u0009\u0031\u0001\u0001\u0009" @JvmStatic private fun zzUnpackAttribute(): IntArray { - val result: IntArray = IntArray(144); - var offset: Int = 0; - offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result); - return result; + val result: IntArray = IntArray(144) + var offset: Int = 0 + offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result) + return result } @JvmStatic private fun zzUnpackAttribute(packed: String , offset: Int, result: IntArray): Int { - var i: Int = 0; /* index in packed string */ - var j: Int = offset; /* index in unpacked array */ - val l: Int = packed.length; + var i: Int = 0 /* index in packed string */ + var j: Int = offset /* index in unpacked array */ + val l: Int = packed.length while (i < l) { - var count: Int = packed[i++].code; - var value: Int = packed[i++].code; + var count: Int = packed[i++].code + var value: Int = packed[i++].code do result[j++] = value while (--count > 0) } - return j; + return j } @JvmStatic @@ -556,7 +556,7 @@ companion object { /** this buffer contains the current text to be matched and is the source of the yytext() string */ - private var zzBuffer: CharSequence = ""; + private var zzBuffer: CharSequence = "" /** the textposition at the last accepting state */ private var zzMarkedPos = 0 @@ -579,23 +579,23 @@ companion object { /** Number of newlines encountered up to the start of the matched text. */ @Suppress("unused") - private var yyline: Int = 0; + private var yyline: Int = 0 /** Number of characters from the last newline up to the start of the matched text. */ @Suppress("unused") - protected var yycolumn: Int = 0; + protected var yycolumn: Int = 0 /** Number of characters up to the start of the matched text. */ @Suppress("unused") - private var yychar: Long = 0; + private var yychar: Long = 0 /** Whether the scanner is currently at the beginning of a line. */ @Suppress("unused") - private var zzAtBOL: Boolean = false; + private var zzAtBOL: Boolean = false /** Whether the user-EOF-code has already been executed. */ @Suppress("unused") - private var zzEOFDone: Boolean = false; + private var zzEOFDone: Boolean = false /* user code: */ private val myJdk15Enabled: Boolean @@ -628,12 +628,12 @@ companion object { /** Returns the maximum size of the scanner buffer, which limits the size of tokens. */ private fun zzMaxBufferLen(): Int { - return Int.MAX_VALUE; + return Int.MAX_VALUE } /** Whether the scanner buffer can grow to accommodate a larger token. */ private fun zzCanGrow(): Boolean { - return true; + return true } @@ -704,7 +704,7 @@ companion object { * @return the character at position pos */ fun yycharat(position: Int): Char { - return zzBuffer[zzStartRead + position]; + return zzBuffer[zzStartRead + position] } @@ -712,7 +712,7 @@ companion object { * Returns the length of the matched text region. */ fun yylength(): Int { - return zzMarkedPos - zzStartRead; + return zzMarkedPos - zzStartRead } /** @@ -745,6 +745,37 @@ companion object { } return high.code } + + internal fun CharSequence.offsetByCodePoints(index: Int, codePointOffset: Int): Int { + val length = this.length + if (index < 0 || index > length) throw IndexOutOfBoundsException() + + var x = index + if (codePointOffset >= 0) { + var i: Int = 0 + + while (x < length && i < codePointOffset) { + if (this[x++].isHighSurrogate() && x < length && this[x].isLowSurrogate()) x++ + i++ + } + + if (i < codePointOffset) throw IndexOutOfBoundsException() + + } else { + var i: Int = codePointOffset + + while (x > 0 && i < 0) { + if (this[--x].isLowSurrogate() && x > 0 && this[x - 1].isHighSurrogate()) x-- + i++ + } + + if (i < 0) { + throw IndexOutOfBoundsException() + } + } + + return x + } internal fun Char.Companion.toCodePoint(high: Char, low: Char): Int = (((high - MIN_HIGH_SURROGATE) shl 10) or (low - MIN_LOW_SURROGATE)) + 0x10000 @@ -791,14 +822,14 @@ companion object { ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR] } - throw Error(message); + throw Error(message) } /** * Translates raw input code points to DFA table row */ private fun zzCMap(input: Int): Int { - val offset: Int = input and 255; - return if(offset == input) ZZ_CMAP_BLOCKS[offset] else ZZ_CMAP_BLOCKS[ZZ_CMAP_TOP[input shr 8] or offset]; + val offset: Int = input and 255 + return if(offset == input) ZZ_CMAP_BLOCKS[offset] else ZZ_CMAP_BLOCKS[ZZ_CMAP_TOP[input shr 8] or offset] } /** @@ -809,19 +840,19 @@ companion object { */ override fun advance(): SyntaxElementType? { - var zzInput: Int = 0; - var zzAction: Int = 0; + var zzInput: Int = 0 + var zzAction: Int = 0 - val zzTransL: IntArray = ZZ_TRANS; - val zzRowMapL: IntArray = ZZ_ROWMAP; - val zzAttrL: IntArray = ZZ_ATTRIBUTE; + val zzTransL: IntArray = ZZ_TRANS + val zzRowMapL: IntArray = ZZ_ROWMAP + val zzAttrL: IntArray = ZZ_ATTRIBUTE while (true) { // cached fields: var zzCurrentPosL = 0 - var zzMarkedPosL: Int = zzMarkedPos; - var zzEndReadL: Int = zzEndRead; - var zzBufferL: CharSequence = zzBuffer; + var zzMarkedPosL: Int = zzMarkedPos + var zzEndReadL: Int = zzEndRead + var zzBufferL: CharSequence = zzBuffer zzAction = -1 @@ -829,52 +860,52 @@ companion object { zzCurrentPos = zzMarkedPosL zzStartRead = zzMarkedPosL - zzState = ZZ_LEXSTATE[zzLexicalState]; + zzState = ZZ_LEXSTATE[zzLexicalState] // set up zzAction for empty match case: - var zzAttributes: Int = zzAttrL[zzState]; + var zzAttributes: Int = zzAttrL[zzState] if ( (zzAttributes and 1) == 1 ) { - zzAction = zzState; + zzAction = zzState } zzForAction@ while (true) { if (zzCurrentPosL < zzEndReadL) { - zzInput = zzBufferL.codePoint(zzCurrentPosL); + zzInput = zzBufferL.codePoint(zzCurrentPosL) zzCurrentPosL += charCount(zzInput) } else if (zzAtEOF) { - zzInput = YYEOF; - break@zzForAction; + zzInput = YYEOF + break@zzForAction } else { // store back cached positions - zzCurrentPos = zzCurrentPosL; - zzMarkedPos = zzMarkedPosL; - val eof: Boolean = zzRefill(); + zzCurrentPos = zzCurrentPosL + zzMarkedPos = zzMarkedPosL + val eof: Boolean = zzRefill() // get translated positions and possibly new buffer - zzCurrentPosL = zzCurrentPos; - zzMarkedPosL = zzMarkedPos; - zzBufferL = zzBuffer; - zzEndReadL = zzEndRead; + zzCurrentPosL = zzCurrentPos + zzMarkedPosL = zzMarkedPos + zzBufferL = zzBuffer + zzEndReadL = zzEndRead if (eof) { - zzInput = YYEOF; - break@zzForAction; + zzInput = YYEOF + break@zzForAction } else { - zzInput = zzBufferL.codePoint(zzCurrentPosL); + zzInput = zzBufferL.codePoint(zzCurrentPosL) zzCurrentPosL += charCount(zzInput) } } - val zzNext: Int = zzTransL[zzRowMapL[zzState] + zzCMap(zzInput)]; - if (zzNext == -1) break@zzForAction; - zzState = zzNext; + val zzNext: Int = zzTransL[zzRowMapL[zzState] + zzCMap(zzInput)] + if (zzNext == -1) break@zzForAction + zzState = zzNext - zzAttributes = zzAttrL[zzState]; + zzAttributes = zzAttrL[zzState] if ( (zzAttributes and 1) == 1 ) { zzAction = zzState zzMarkedPosL = zzCurrentPosL - if ( (zzAttributes and 8) == 8 ) break@zzForAction; + if ( (zzAttributes and 8) == 8 ) break@zzForAction } } @@ -883,7 +914,7 @@ companion object { if (zzInput == YYEOF && zzStartRead == zzCurrentPos) { zzAtEOF = true - return null; + return null } else { when (if (zzAction < 0) zzAction else ZZ_ACTION[zzAction]) { @@ -891,17 +922,17 @@ companion object { return JavaDocSyntaxTokenType.DOC_COMMENT_BAD_CHARACTER; } // fall through - 44 -> break; + 44 -> break 2 -> { yybegin(COMMENT_DATA); return JavaDocSyntaxTokenType.DOC_COMMENT_DATA; } // fall through - 45 -> break; + 45 -> break 3 -> { return JavaDocSyntaxTokenType.DOC_SPACE; } // fall through - 46 -> break; + 46 -> break 4 -> { yybegin(COMMENT_DATA); if(myMarkdownMode) { @@ -910,7 +941,7 @@ companion object { return JavaDocSyntaxTokenType.DOC_COMMENT_DATA; } // fall through - 47 -> break; + 47 -> break 5 -> { yybegin(COMMENT_DATA); if(myMarkdownMode) { @@ -919,7 +950,7 @@ companion object { return JavaDocSyntaxTokenType.DOC_COMMENT_DATA; } // fall through - 48 -> break; + 48 -> break 6 -> { yybegin(COMMENT_DATA); if(myMarkdownMode) { @@ -928,7 +959,7 @@ companion object { return JavaDocSyntaxTokenType.DOC_COMMENT_DATA; } // fall through - 49 -> break; + 49 -> break 7 -> { yybegin(COMMENT_DATA); if(myMarkdownMode) { @@ -937,7 +968,7 @@ companion object { return JavaDocSyntaxTokenType.DOC_COMMENT_DATA; } // fall through - 50 -> break; + 50 -> break 8 -> { yybegin(COMMENT_DATA); if(myMarkdownMode) { @@ -946,7 +977,7 @@ companion object { return JavaDocSyntaxTokenType.DOC_COMMENT_DATA; } // fall through - 51 -> break; + 51 -> break 9 -> { yybegin(COMMENT_DATA); if(myMarkdownMode) { @@ -955,7 +986,7 @@ companion object { return JavaDocSyntaxTokenType.DOC_COMMENT_DATA; } // fall through - 52 -> break; + 52 -> break 10 -> { yybegin(COMMENT_DATA); if(myMarkdownMode) { @@ -964,7 +995,7 @@ companion object { return JavaDocSyntaxTokenType.DOC_COMMENT_DATA; } // fall through - 53 -> break; + 53 -> break 11 -> { if (checkAhead('@')) { yybegin(INLINE_TAG_NAME); @@ -976,17 +1007,17 @@ companion object { } } // fall through - 54 -> break; + 54 -> break 12 -> { yybegin(COMMENT_DATA); return JavaDocSyntaxTokenType.DOC_INLINE_TAG_END; } // fall through - 55 -> break; + 55 -> break 13 -> { return JavaDocSyntaxTokenType.DOC_COMMENT_DATA; } // fall through - 56 -> break; + 56 -> break 14 -> { if (checkAhead('<') || checkAhead('\"')) yybegin(COMMENT_DATA); else if (checkAhead('\u007b')) yybegin(COMMENT_DATA); // lbrace - there's a error in JLex when typing lbrace directly @@ -994,42 +1025,42 @@ companion object { return JavaDocSyntaxTokenType.DOC_SPACE; } // fall through - 57 -> break; + 57 -> break 15 -> { yybegin(DOC_TAG_VALUE); return JavaDocSyntaxTokenType.DOC_SPACE; } // fall through - 58 -> break; + 58 -> break 16 -> { yybegin(COMMENT_DATA); return JavaDocSyntaxTokenType.DOC_SPACE; } // fall through - 59 -> break; + 59 -> break 17 -> { return JavaDocSyntaxTokenType.DOC_TAG_VALUE_SHARP_TOKEN; } // fall through - 60 -> break; + 60 -> break 18 -> { return JavaDocSyntaxTokenType.DOC_TAG_VALUE_TOKEN; } // fall through - 61 -> break; + 61 -> break 19 -> { yybegin(DOC_TAG_VALUE_IN_PAREN); return JavaDocSyntaxTokenType.DOC_TAG_VALUE_LPAREN; } // fall through - 62 -> break; + 62 -> break 20 -> { return JavaDocSyntaxTokenType.DOC_TAG_VALUE_COMMA; } // fall through - 63 -> break; + 63 -> break 21 -> { return JavaDocSyntaxTokenType.DOC_TAG_VALUE_DIV_TOKEN; } // fall through - 64 -> break; + 64 -> break 22 -> { if (myJdk15Enabled) { yybegin(DOC_TAG_VALUE_IN_LTGT); @@ -1041,42 +1072,42 @@ companion object { } } // fall through - 65 -> break; + 65 -> break 23 -> { yybegin(DOC_TAG_VALUE); return JavaDocSyntaxTokenType.DOC_TAG_VALUE_RPAREN; } // fall through - 66 -> break; + 66 -> break 24 -> { yybegin(COMMENT_DATA); return JavaDocSyntaxTokenType.DOC_TAG_VALUE_GT; } // fall through - 67 -> break; + 67 -> break 25 -> { yybegin(CODE_TAG); return JavaDocSyntaxTokenType.DOC_COMMENT_DATA; } // fall through - 68 -> break; + 68 -> break 26 -> { yybegin(CODE_TAG); return JavaDocSyntaxTokenType.DOC_SPACE; } // fall through - 69 -> break; + 69 -> break 27 -> { yybegin(SNIPPET_TAG_COMMENT_DATA_UNTIL_COLON); return JavaDocSyntaxTokenType.DOC_COMMENT_DATA; } // fall through - 70 -> break; + 70 -> break 28 -> { yybegin(SNIPPET_ATTRIBUTE_VALUE_DOUBLE_QUOTES); return JavaDocSyntaxTokenType.DOC_TAG_VALUE_QUOTE; } // fall through - 71 -> break; + 71 -> break 29 -> { yybegin(SNIPPET_ATTRIBUTE_VALUE_SINGLE_QUOTES); return JavaDocSyntaxTokenType.DOC_TAG_VALUE_QUOTE; } // fall through - 72 -> break; + 72 -> break 30 -> { if (myMarkdownMode) { return JavaDocSyntaxTokenType.DOC_COMMENT_DATA; @@ -1084,17 +1115,17 @@ companion object { return JavaDocSyntaxTokenType.DOC_COMMENT_LEADING_ASTERISKS; } // fall through - 73 -> break; + 73 -> break 31 -> { yybegin(SNIPPET_TAG_BODY_DATA); return JavaDocSyntaxTokenType.DOC_TAG_VALUE_COLON; } // fall through - 74 -> break; + 74 -> break 32 -> { mySnippetBracesLevel++; return JavaDocSyntaxTokenType.DOC_COMMENT_DATA; } // fall through - 75 -> break; + 75 -> break 33 -> { if (mySnippetBracesLevel > 0) { mySnippetBracesLevel--; @@ -1105,12 +1136,12 @@ companion object { } } // fall through - 76 -> break; + 76 -> break 34 -> { yybegin(SNIPPET_TAG_COMMENT_DATA_UNTIL_COLON); return JavaDocSyntaxTokenType.DOC_TAG_VALUE_QUOTE; } // fall through - 77 -> break; + 77 -> break 35 -> { if(myMarkdownMode) { return JavaDocSyntaxTokenType.DOC_COMMENT_DATA; @@ -1118,12 +1149,12 @@ companion object { return JavaDocSyntaxTokenType.DOC_COMMENT_END; } // fall through - 78 -> break; + 78 -> break 36 -> { yybegin(TAG_DOC_SPACE); return JavaDocSyntaxTokenType.DOC_TAG_NAME; } // fall through - 79 -> break; + 79 -> break 37 -> { if (myMarkdownMode) { return JavaDocSyntaxTokenType.DOC_COMMENT_BAD_CHARACTER; @@ -1132,7 +1163,7 @@ companion object { return JavaDocSyntaxTokenType.DOC_COMMENT_START; } // fall through - 80 -> break; + 80 -> break 38 -> { if(myMarkdownMode) { yybegin(COMMENT_DATA_START); @@ -1141,7 +1172,7 @@ companion object { return JavaDocSyntaxTokenType.DOC_COMMENT_BAD_CHARACTER; } // fall through - 81 -> break; + 81 -> break 39 -> { yybegin(COMMENT_DATA); if(myMarkdownMode) { @@ -1150,7 +1181,7 @@ companion object { return JavaDocSyntaxTokenType.DOC_COMMENT_DATA; } // fall through - 82 -> break; + 82 -> break 40 -> { if (myMarkdownMode) { return JavaDocSyntaxTokenType.DOC_COMMENT_LEADING_ASTERISKS; @@ -1158,24 +1189,24 @@ companion object { return JavaDocSyntaxTokenType.DOC_COMMENT_DATA; } // fall through - 83 -> break; + 83 -> break 41 -> { yybegin(CODE_TAG_SPACE); return JavaDocSyntaxTokenType.DOC_TAG_NAME; } // fall through - 84 -> break; + 84 -> break 42 -> { yybegin(PARAM_TAG_SPACE); return JavaDocSyntaxTokenType.DOC_TAG_NAME; } // fall through - 85 -> break; + 85 -> break 43 -> { yybegin(SNIPPET_TAG_COMMENT_DATA_UNTIL_COLON); return JavaDocSyntaxTokenType.DOC_TAG_NAME; } // fall through - 86 -> break; + 86 -> break else -> - zzScanError(ZZ_NO_MATCH); + zzScanError(ZZ_NO_MATCH) } } } diff --git a/java/java-syntax/gen/com/intellij/java/syntax/lexer/_JavaLexer.kt b/java/java-syntax/gen/com/intellij/java/syntax/lexer/_JavaLexer.kt index b290be60fab5..bbdcb3f74e29 100644 --- a/java/java-syntax/gen/com/intellij/java/syntax/lexer/_JavaLexer.kt +++ b/java/java-syntax/gen/com/intellij/java/syntax/lexer/_JavaLexer.kt @@ -1,8 +1,9 @@ // Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. -// Generated by JFlex 1.10.13 http://jflex.de/ (tweaked for IntelliJ platform) +// Generated by JFlex 1.10.14 http://jflex.de/ (tweaked for IntelliJ platform) // source: _JavaLexer.flex +/* It's an automatically generated code. Do not modify it. */ package com.intellij.java.syntax.lexer import com.intellij.java.syntax.element.JavaSyntaxTokenType @@ -16,7 +17,7 @@ import com.intellij.pom.java.LanguageLevel class _JavaLexer : FlexLexer { companion object { /** This character denotes the end of file */ - private const val YYEOF = -1; + private const val YYEOF = -1 /** initial size of the lookahead buffer */ private const val ZZ_BUFFERSIZE: Int = 16384 @@ -59,29 +60,29 @@ companion object { "\u0001\u6800\u0001\u6900\u0001\u6a00\u0005\u1f00\u0001\u6b00\u0001\u6c00\u0002\u1f00\u0001\u6d00"+ "\u0001\u1f00\u0001\u6e00\u000c\u1f00\u0001\u6f00\u0004\u1f00\u00a6\u0100\u0001\u7000\u0010\u0100"+ "\u0001\u7100\u0001\u7200\u0015\u0100\u0001\u7300\u001c\u0100\u0001\u7400\u000c\u1f00\u0002\u0100"+ - "\u0001\u7500\u0005\u1f00\u0013\u0100\u0001\u7600\u0aec\u1f00\u0001\u7700\u0001\u7800\u02fe\u1f00"; + "\u0001\u7500\u0005\u1f00\u0013\u0100\u0001\u7600\u0aec\u1f00\u0001\u7700\u0001\u7800\u02fe\u1f00" @JvmStatic private fun zzUnpackcmap_top(): IntArray { - val result: IntArray = IntArray(4352); - var offset: Int = 0; - offset = zzUnpackcmap_top(ZZ_CMAP_TOP_PACKED_0, offset, result); - return result; + val result: IntArray = IntArray(4352) + var offset: Int = 0 + offset = zzUnpackcmap_top(ZZ_CMAP_TOP_PACKED_0, offset, result) + return result } @JvmStatic private fun zzUnpackcmap_top(packed: String , offset: Int, result: IntArray): Int { - var i: Int = 0; /* index in packed string */ - var j: Int = offset; /* index in unpacked array */ - val l: Int = packed.length; + var i: Int = 0 /* index in packed string */ + var j: Int = offset /* index in unpacked array */ + val l: Int = packed.length while (i < l) { - var count: Int = packed[i++].code; - var value: Int = packed[i++].code; + var count: Int = packed[i++].code + var value: Int = packed[i++].code do result[j++] = value while (--count > 0) } - return j; + return j } @JvmStatic @@ -320,29 +321,29 @@ companion object { "\u0001\u0001\u0005\u0003\u0001\u0001\u0011\u0003\u0134\u0001\u000a\u0000\u0006\u0001\u00de\u0003"+ "\u0022\u0001\u0035\u0003\u000b\u0001\u00de\u0003\u0002\u0001\u0182\u0003\u000e\u0001\u0131\u0003"+ "\u001f\u0001\u001e\u0003\u00e2\u0001\u004b\u0003\u00b6\u0001\u0001\u0000\u001e\u0001\u0060\u0000"+ - "\u0080\u0001\u00f0\u0000\u0010\u0001"; + "\u0080\u0001\u00f0\u0000\u0010\u0001" @JvmStatic private fun zzUnpackcmap_blocks(): IntArray { - val result: IntArray = IntArray(30976); - var offset: Int = 0; - offset = zzUnpackcmap_blocks(ZZ_CMAP_BLOCKS_PACKED_0, offset, result); - return result; + val result: IntArray = IntArray(30976) + var offset: Int = 0 + offset = zzUnpackcmap_blocks(ZZ_CMAP_BLOCKS_PACKED_0, offset, result) + return result } @JvmStatic private fun zzUnpackcmap_blocks(packed: String , offset: Int, result: IntArray): Int { - var i: Int = 0; /* index in packed string */ - var j: Int = offset; /* index in unpacked array */ - val l: Int = packed.length; + var i: Int = 0 /* index in packed string */ + var j: Int = offset /* index in unpacked array */ + val l: Int = packed.length while (i < l) { - var count: Int = packed[i++].code; - var value: Int = packed[i++].code; + var count: Int = packed[i++].code + var value: Int = packed[i++].code do result[j++] = value while (--count > 0) } - return j; + return j } @JvmStatic @@ -370,29 +371,29 @@ companion object { "\u0001\u0054\u0001\u0055\u0001\u0056\u0001\u0003\u0001\u0057\u0001\u0003\u0001\u0058\u0003\u0003"+ "\u0001\u0059\u0001\u0003\u0001\u005a\u0001\u005b\u0001\u005c\u0003\u0003\u0001\u005d\u0001\u005e"+ "\u0005\u0003\u0001\u005f\u0001\u0060\u0004\u0003\u0001\u0061\u0002\u0003\u0001\u0062\u0002\u0003"+ - "\u0001\u0063\u0001\u0064\u0001\u0003\u0001\u0065\u0001\u0066\u0001\u0067\u0002\u0003\u0001\u0068"; + "\u0001\u0063\u0001\u0064\u0001\u0003\u0001\u0065\u0001\u0066\u0001\u0067\u0002\u0003\u0001\u0068" @JvmStatic private fun zzUnpackAction(): IntArray { - val result: IntArray = IntArray(304); - var offset: Int = 0; - offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result); - return result; + val result: IntArray = IntArray(304) + var offset: Int = 0 + offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result) + return result } @JvmStatic private fun zzUnpackAction(packed: String , offset: Int, result: IntArray): Int { - var i: Int = 0; /* index in packed string */ - var j: Int = offset; /* index in unpacked array */ - val l: Int = packed.length; + var i: Int = 0 /* index in packed string */ + var j: Int = offset /* index in unpacked array */ + val l: Int = packed.length while (i < l) { - var count: Int = packed[i++].code; - var value: Int = packed[i++].code; + var count: Int = packed[i++].code + var value: Int = packed[i++].code do result[j++] = value while (--count > 0) } - return j; + return j } @JvmStatic @@ -440,26 +441,26 @@ companion object { "\u0000\u00bd\u0000\u31b6\u0000\u31f5\u0000\u3234\u0000\u00bd\u0000\u00bd\u0000\u3273\u0000\u32b2"+ "\u0000\u32f1\u0000\u3330\u0000\u336f\u0000\u00bd\u0000\u00bd\u0000\u33ae\u0000\u33ed\u0000\u342c"+ "\u0000\u346b\u0000\u00bd\u0000\u34aa\u0000\u34e9\u0000\u00bd\u0000\u3528\u0000\u3567\u0000\u00bd"+ - "\u0000\u00bd\u0000\u35a6\u0000\u00bd\u0000\u00bd\u0000\u00bd\u0000\u35e5\u0000\u3624\u0000\u00bd"; + "\u0000\u00bd\u0000\u35a6\u0000\u00bd\u0000\u00bd\u0000\u00bd\u0000\u35e5\u0000\u3624\u0000\u00bd" @JvmStatic private fun zzUnpackRowMap(): IntArray { - val result: IntArray = IntArray(304); - var offset: Int = 0; - offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result); - return result; + val result: IntArray = IntArray(304) + var offset: Int = 0 + offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result) + return result } @JvmStatic private fun zzUnpackRowMap(packed: String, offset: Int, result: IntArray): Int { - var i: Int = 0; /* index in packed string */ - var j: Int = offset; /* index in unpacked array */ - val l: Int = packed.length - 1; + var i: Int = 0 /* index in packed string */ + var j: Int = offset /* index in unpacked array */ + val l: Int = packed.length - 1 while (i < l) { - val high: Int = packed[i++].code shl 16; - result[j++] = high or packed[i++].code; + val high: Int = packed[i++].code shl 16 + result[j++] = high or packed[i++].code } - return j; + return j } @JvmStatic @@ -677,7 +678,7 @@ companion object { "\u0004\u0000\u0001\u0004\u0002\u0000\u0001\u0004\u000a\u0000\u0002\u0004\u0007\u0000\u0008\u0004"+ "\u0003\u0000\u0010\u0004\u0001\u00e0\u0008\u0004\u0004\u0000\u0001\u0004\u0002\u0000\u0001\u0004"+ "\u000a\u0000\u0002\u0004\u0007\u0000\u0008\u0004\u0003\u0000\u0001\u0004\u0001\u00e1\u0017\u0004"+ - "\u0004\u0000\u0001\u0004\u0002\u0000"; + "\u0004\u0000\u0001\u0004\u0002\u0000" private const val ZZ_TRANS_PACKED_1: String = "\u0001\u0004\u000a\u0000\u0002\u0004\u0007\u0000\u0008\u0004\u0003\u0000\u0010\u0004\u0001\u00e2"+ "\u0008\u0004\u0004\u0000\u0001\u0004\u0002\u0000\u0001\u0004\u000a\u0000\u0002\u0004\u0007\u0000"+ @@ -797,31 +798,31 @@ companion object { "\u0004\u0000\u0001\u0004\u0002\u0000\u0001\u0004\u000a\u0000\u0002\u0004\u0007\u0000\u0008\u0004"+ "\u0003\u0000\u0005\u0004\u0001\u012f\u0013\u0004\u0004\u0000\u0001\u0004\u0002\u0000\u0001\u0004"+ "\u000a\u0000\u0002\u0004\u0007\u0000\u0008\u0004\u0003\u0000\u0004\u0004\u0001\u0130\u0014\u0004"+ - "\u0004\u0000"; + "\u0004\u0000" @JvmStatic private fun zzUnpacktrans(): IntArray { - val result: IntArray = IntArray(13923); - var offset: Int = 0; - offset = zzUnpacktrans(ZZ_TRANS_PACKED_0, offset, result); - offset = zzUnpacktrans(ZZ_TRANS_PACKED_1, offset, result); - return result; + val result: IntArray = IntArray(13923) + var offset: Int = 0 + offset = zzUnpacktrans(ZZ_TRANS_PACKED_0, offset, result) + offset = zzUnpacktrans(ZZ_TRANS_PACKED_1, offset, result) + return result } @JvmStatic private fun zzUnpacktrans(packed: String , offset: Int, result: IntArray): Int { - var i: Int = 0; /* index in packed string */ - var j: Int = offset; /* index in unpacked array */ - val l: Int = packed.length; + var i: Int = 0 /* index in packed string */ + var j: Int = offset /* index in unpacked array */ + val l: Int = packed.length while (i < l) { - var count: Int = packed[i++].code; - var value: Int = packed[i++].code; - value--; + var count: Int = packed[i++].code + var value: Int = packed[i++].code + value-- do result[j++] = value while (--count > 0) } - return j; + return j } @JvmStatic @@ -849,29 +850,29 @@ companion object { "\u0002\u0001\u0005\u0009\u0011\u0001\u0001\u0009\u0001\u0001\u000c\u0009\u0001\u0000\u0001\u0001"+ "\u0001\u0009\u0001\u0001\u0001\u0009\u0001\u0001\u0002\u0009\u0001\u0001\u0001\u0009\u0001\u0001"+ "\u0003\u0009\u0027\u0001\u0003\u0009\u0001\u0001\u0001\u0000\u0001\u0001\u0001\u0009\u002e\u0001"+ - "\u0001\u0000\u008d\u0001"; + "\u0001\u0000\u008d\u0001" @JvmStatic private fun zzUnpackAttribute(): IntArray { - val result: IntArray = IntArray(304); - var offset: Int = 0; - offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result); - return result; + val result: IntArray = IntArray(304) + var offset: Int = 0 + offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result) + return result } @JvmStatic private fun zzUnpackAttribute(packed: String , offset: Int, result: IntArray): Int { - var i: Int = 0; /* index in packed string */ - var j: Int = offset; /* index in unpacked array */ - val l: Int = packed.length; + var i: Int = 0 /* index in packed string */ + var j: Int = offset /* index in unpacked array */ + val l: Int = packed.length while (i < l) { - var count: Int = packed[i++].code; - var value: Int = packed[i++].code; + var count: Int = packed[i++].code + var value: Int = packed[i++].code do result[j++] = value while (--count > 0) } - return j; + return j } @JvmStatic @@ -886,7 +887,7 @@ companion object { /** this buffer contains the current text to be matched and is the source of the yytext() string */ - private var zzBuffer: CharSequence = ""; + private var zzBuffer: CharSequence = "" /** the textposition at the last accepting state */ private var zzMarkedPos = 0 @@ -909,23 +910,23 @@ companion object { /** Number of newlines encountered up to the start of the matched text. */ @Suppress("unused") - private var yyline: Int = 0; + private var yyline: Int = 0 /** Number of characters from the last newline up to the start of the matched text. */ @Suppress("unused") - protected var yycolumn: Int = 0; + protected var yycolumn: Int = 0 /** Number of characters up to the start of the matched text. */ @Suppress("unused") - private var yychar: Long = 0; + private var yychar: Long = 0 /** Whether the scanner is currently at the beginning of a line. */ @Suppress("unused") - private var zzAtBOL: Boolean = false; + private var zzAtBOL: Boolean = false /** Whether the user-EOF-code has already been executed. */ @Suppress("unused") - private var zzEOFDone: Boolean = false; + private var zzEOFDone: Boolean = false /* user code: */ private val myAssertKeyword: Boolean @@ -947,12 +948,12 @@ companion object { /** Returns the maximum size of the scanner buffer, which limits the size of tokens. */ private fun zzMaxBufferLen(): Int { - return Int.MAX_VALUE; + return Int.MAX_VALUE } /** Whether the scanner buffer can grow to accommodate a larger token. */ private fun zzCanGrow(): Boolean { - return true; + return true } @@ -1023,7 +1024,7 @@ companion object { * @return the character at position pos */ fun yycharat(position: Int): Char { - return zzBuffer[zzStartRead + position]; + return zzBuffer[zzStartRead + position] } @@ -1031,7 +1032,7 @@ companion object { * Returns the length of the matched text region. */ fun yylength(): Int { - return zzMarkedPos - zzStartRead; + return zzMarkedPos - zzStartRead } /** @@ -1064,6 +1065,37 @@ companion object { } return high.code } + + internal fun CharSequence.offsetByCodePoints(index: Int, codePointOffset: Int): Int { + val length = this.length + if (index < 0 || index > length) throw IndexOutOfBoundsException() + + var x = index + if (codePointOffset >= 0) { + var i: Int = 0 + + while (x < length && i < codePointOffset) { + if (this[x++].isHighSurrogate() && x < length && this[x].isLowSurrogate()) x++ + i++ + } + + if (i < codePointOffset) throw IndexOutOfBoundsException() + + } else { + var i: Int = codePointOffset + + while (x > 0 && i < 0) { + if (this[--x].isLowSurrogate() && x > 0 && this[x - 1].isHighSurrogate()) x-- + i++ + } + + if (i < 0) { + throw IndexOutOfBoundsException() + } + } + + return x + } internal fun Char.Companion.toCodePoint(high: Char, low: Char): Int = (((high - MIN_HIGH_SURROGATE) shl 10) or (low - MIN_LOW_SURROGATE)) + 0x10000 @@ -1110,14 +1142,14 @@ companion object { ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR] } - throw Error(message); + throw Error(message) } /** * Translates raw input code points to DFA table row */ private fun zzCMap(input: Int): Int { - val offset: Int = input and 255; - return if(offset == input) ZZ_CMAP_BLOCKS[offset] else ZZ_CMAP_BLOCKS[ZZ_CMAP_TOP[input shr 8] or offset]; + val offset: Int = input and 255 + return if(offset == input) ZZ_CMAP_BLOCKS[offset] else ZZ_CMAP_BLOCKS[ZZ_CMAP_TOP[input shr 8] or offset] } /** @@ -1128,19 +1160,19 @@ companion object { */ override fun advance(): SyntaxElementType? { - var zzInput: Int = 0; - var zzAction: Int = 0; + var zzInput: Int = 0 + var zzAction: Int = 0 - val zzTransL: IntArray = ZZ_TRANS; - val zzRowMapL: IntArray = ZZ_ROWMAP; - val zzAttrL: IntArray = ZZ_ATTRIBUTE; + val zzTransL: IntArray = ZZ_TRANS + val zzRowMapL: IntArray = ZZ_ROWMAP + val zzAttrL: IntArray = ZZ_ATTRIBUTE while (true) { // cached fields: var zzCurrentPosL = 0 - var zzMarkedPosL: Int = zzMarkedPos; - var zzEndReadL: Int = zzEndRead; - var zzBufferL: CharSequence = zzBuffer; + var zzMarkedPosL: Int = zzMarkedPos + var zzEndReadL: Int = zzEndRead + var zzBufferL: CharSequence = zzBuffer zzAction = -1 @@ -1148,52 +1180,52 @@ companion object { zzCurrentPos = zzMarkedPosL zzStartRead = zzMarkedPosL - zzState = ZZ_LEXSTATE[zzLexicalState]; + zzState = ZZ_LEXSTATE[zzLexicalState] // set up zzAction for empty match case: - var zzAttributes: Int = zzAttrL[zzState]; + var zzAttributes: Int = zzAttrL[zzState] if ( (zzAttributes and 1) == 1 ) { - zzAction = zzState; + zzAction = zzState } zzForAction@ while (true) { if (zzCurrentPosL < zzEndReadL) { - zzInput = zzBufferL.codePoint(zzCurrentPosL); + zzInput = zzBufferL.codePoint(zzCurrentPosL) zzCurrentPosL += charCount(zzInput) } else if (zzAtEOF) { - zzInput = YYEOF; - break@zzForAction; + zzInput = YYEOF + break@zzForAction } else { // store back cached positions - zzCurrentPos = zzCurrentPosL; - zzMarkedPos = zzMarkedPosL; - val eof: Boolean = zzRefill(); + zzCurrentPos = zzCurrentPosL + zzMarkedPos = zzMarkedPosL + val eof: Boolean = zzRefill() // get translated positions and possibly new buffer - zzCurrentPosL = zzCurrentPos; - zzMarkedPosL = zzMarkedPos; - zzBufferL = zzBuffer; - zzEndReadL = zzEndRead; + zzCurrentPosL = zzCurrentPos + zzMarkedPosL = zzMarkedPos + zzBufferL = zzBuffer + zzEndReadL = zzEndRead if (eof) { - zzInput = YYEOF; - break@zzForAction; + zzInput = YYEOF + break@zzForAction } else { - zzInput = zzBufferL.codePoint(zzCurrentPosL); + zzInput = zzBufferL.codePoint(zzCurrentPosL) zzCurrentPosL += charCount(zzInput) } } - val zzNext: Int = zzTransL[zzRowMapL[zzState] + zzCMap(zzInput)]; - if (zzNext == -1) break@zzForAction; - zzState = zzNext; + val zzNext: Int = zzTransL[zzRowMapL[zzState] + zzCMap(zzInput)] + if (zzNext == -1) break@zzForAction + zzState = zzNext - zzAttributes = zzAttrL[zzState]; + zzAttributes = zzAttrL[zzState] if ( (zzAttributes and 1) == 1 ) { zzAction = zzState zzMarkedPosL = zzCurrentPosL - if ( (zzAttributes and 8) == 8 ) break@zzForAction; + if ( (zzAttributes and 8) == 8 ) break@zzForAction } } @@ -1202,7 +1234,7 @@ companion object { if (zzInput == YYEOF && zzStartRead == zzCurrentPos) { zzAtEOF = true - return null; + return null } else { when (if (zzAction < 0) zzAction else ZZ_ACTION[zzAction]) { @@ -1210,524 +1242,524 @@ companion object { return SyntaxTokenTypes.BAD_CHARACTER; } // fall through - 105 -> break; + 105 -> break 2 -> { return JavaSyntaxTokenType.EXCL; } // fall through - 106 -> break; + 106 -> break 3 -> { return JavaSyntaxTokenType.IDENTIFIER; } // fall through - 107 -> break; + 107 -> break 4 -> { return JavaSyntaxTokenType.PERC; } // fall through - 108 -> break; + 108 -> break 5 -> { return JavaSyntaxTokenType.AND; } // fall through - 109 -> break; + 109 -> break 6 -> { return JavaSyntaxTokenType.LPARENTH; } // fall through - 110 -> break; + 110 -> break 7 -> { return JavaSyntaxTokenType.RPARENTH; } // fall through - 111 -> break; + 111 -> break 8 -> { return JavaSyntaxTokenType.ASTERISK; } // fall through - 112 -> break; + 112 -> break 9 -> { return JavaSyntaxTokenType.PLUS; } // fall through - 113 -> break; + 113 -> break 10 -> { return JavaSyntaxTokenType.COMMA; } // fall through - 114 -> break; + 114 -> break 11 -> { return JavaSyntaxTokenType.MINUS; } // fall through - 115 -> break; + 115 -> break 12 -> { return JavaSyntaxTokenType.DOT; } // fall through - 116 -> break; + 116 -> break 13 -> { return JavaSyntaxTokenType.DIV; } // fall through - 117 -> break; + 117 -> break 14 -> { return JavaSyntaxTokenType.INTEGER_LITERAL; } // fall through - 118 -> break; + 118 -> break 15 -> { return JavaSyntaxTokenType.COLON; } // fall through - 119 -> break; + 119 -> break 16 -> { return JavaSyntaxTokenType.SEMICOLON; } // fall through - 120 -> break; + 120 -> break 17 -> { return JavaSyntaxTokenType.LT; } // fall through - 121 -> break; + 121 -> break 18 -> { return JavaSyntaxTokenType.EQ; } // fall through - 122 -> break; + 122 -> break 19 -> { return JavaSyntaxTokenType.GT; } // fall through - 123 -> break; + 123 -> break 20 -> { return JavaSyntaxTokenType.QUEST; } // fall through - 124 -> break; + 124 -> break 21 -> { return JavaSyntaxTokenType.AT; } // fall through - 125 -> break; + 125 -> break 22 -> { return JavaSyntaxTokenType.LBRACKET; } // fall through - 126 -> break; + 126 -> break 23 -> { return JavaSyntaxTokenType.RBRACKET; } // fall through - 127 -> break; + 127 -> break 24 -> { return JavaSyntaxTokenType.XOR; } // fall through - 128 -> break; + 128 -> break 25 -> { return JavaSyntaxTokenType.LBRACE; } // fall through - 129 -> break; + 129 -> break 26 -> { return JavaSyntaxTokenType.OR; } // fall through - 130 -> break; + 130 -> break 27 -> { return JavaSyntaxTokenType.RBRACE; } // fall through - 131 -> break; + 131 -> break 28 -> { return JavaSyntaxTokenType.TILDE; } // fall through - 132 -> break; + 132 -> break 29 -> { return JavaSyntaxTokenType.NE; } // fall through - 133 -> break; + 133 -> break 30 -> { return JavaSyntaxTokenType.PERCEQ; } // fall through - 134 -> break; + 134 -> break 31 -> { return JavaSyntaxTokenType.ANDAND; } // fall through - 135 -> break; + 135 -> break 32 -> { return JavaSyntaxTokenType.ANDEQ; } // fall through - 136 -> break; + 136 -> break 33 -> { return JavaSyntaxTokenType.ASTERISKEQ; } // fall through - 137 -> break; + 137 -> break 34 -> { return JavaSyntaxTokenType.PLUSPLUS; } // fall through - 138 -> break; + 138 -> break 35 -> { return JavaSyntaxTokenType.PLUSEQ; } // fall through - 139 -> break; + 139 -> break 36 -> { return JavaSyntaxTokenType.MINUSMINUS; } // fall through - 140 -> break; + 140 -> break 37 -> { return JavaSyntaxTokenType.MINUSEQ; } // fall through - 141 -> break; + 141 -> break 38 -> { return JavaSyntaxTokenType.ARROW; } // fall through - 142 -> break; + 142 -> break 39 -> { return JavaSyntaxTokenType.DOUBLE_LITERAL; } // fall through - 143 -> break; + 143 -> break 40 -> { return JavaSyntaxTokenType.DIVEQ; } // fall through - 144 -> break; + 144 -> break 41 -> { return JavaSyntaxTokenType.FLOAT_LITERAL; } // fall through - 145 -> break; + 145 -> break 42 -> { return JavaSyntaxTokenType.LONG_LITERAL; } // fall through - 146 -> break; + 146 -> break 43 -> { return JavaSyntaxTokenType.DOUBLE_COLON; } // fall through - 147 -> break; + 147 -> break 44 -> { return JavaSyntaxTokenType.LTLT; } // fall through - 148 -> break; + 148 -> break 45 -> { return JavaSyntaxTokenType.LE; } // fall through - 149 -> break; + 149 -> break 46 -> { return JavaSyntaxTokenType.EQEQ; } // fall through - 150 -> break; + 150 -> break 47 -> { return JavaSyntaxTokenType.XOREQ; } // fall through - 151 -> break; + 151 -> break 48 -> { return JavaSyntaxTokenType.DO_KEYWORD; } // fall through - 152 -> break; + 152 -> break 49 -> { return JavaSyntaxTokenType.IF_KEYWORD; } // fall through - 153 -> break; + 153 -> break 50 -> { return JavaSyntaxTokenType.OREQ; } // fall through - 154 -> break; + 154 -> break 51 -> { return JavaSyntaxTokenType.OROR; } // fall through - 155 -> break; + 155 -> break 52 -> { return JavaSyntaxTokenType.ELLIPSIS; } // fall through - 156 -> break; + 156 -> break 53 -> { return JavaSyntaxTokenType.LTLTEQ; } // fall through - 157 -> break; + 157 -> break 54 -> { return JavaSyntaxTokenType.FOR_KEYWORD; } // fall through - 158 -> break; + 158 -> break 55 -> { return JavaSyntaxTokenType.INT_KEYWORD; } // fall through - 159 -> break; + 159 -> break 56 -> { return JavaSyntaxTokenType.NEW_KEYWORD; } // fall through - 160 -> break; + 160 -> break 57 -> { return JavaSyntaxTokenType.TRY_KEYWORD; } // fall through - 161 -> break; + 161 -> break 58 -> { return JavaSyntaxTokenType.BYTE_KEYWORD; } // fall through - 162 -> break; + 162 -> break 59 -> { return JavaSyntaxTokenType.CASE_KEYWORD; } // fall through - 163 -> break; + 163 -> break 60 -> { return JavaSyntaxTokenType.CHAR_KEYWORD; } // fall through - 164 -> break; + 164 -> break 61 -> { return JavaSyntaxTokenType.ELSE_KEYWORD; } // fall through - 165 -> break; + 165 -> break 62 -> { return if (myEnumKeyword) JavaSyntaxTokenType.ENUM_KEYWORD else JavaSyntaxTokenType.IDENTIFIER; } // fall through - 166 -> break; + 166 -> break 63 -> { return JavaSyntaxTokenType.GOTO_KEYWORD; } // fall through - 167 -> break; + 167 -> break 64 -> { return JavaSyntaxTokenType.LONG_KEYWORD; } // fall through - 168 -> break; + 168 -> break 65 -> { return JavaSyntaxTokenType.NULL_KEYWORD; } // fall through - 169 -> break; + 169 -> break 66 -> { return JavaSyntaxTokenType.THIS_KEYWORD; } // fall through - 170 -> break; + 170 -> break 67 -> { return JavaSyntaxTokenType.TRUE_KEYWORD; } // fall through - 171 -> break; + 171 -> break 68 -> { return JavaSyntaxTokenType.VOID_KEYWORD; } // fall through - 172 -> break; + 172 -> break 69 -> { return JavaSyntaxTokenType.BREAK_KEYWORD; } // fall through - 173 -> break; + 173 -> break 70 -> { return JavaSyntaxTokenType.CATCH_KEYWORD; } // fall through - 174 -> break; + 174 -> break 71 -> { return JavaSyntaxTokenType.CLASS_KEYWORD; } // fall through - 175 -> break; + 175 -> break 72 -> { return JavaSyntaxTokenType.CONST_KEYWORD; } // fall through - 176 -> break; + 176 -> break 73 -> { return JavaSyntaxTokenType.FALSE_KEYWORD; } // fall through - 177 -> break; + 177 -> break 74 -> { return JavaSyntaxTokenType.FINAL_KEYWORD; } // fall through - 178 -> break; + 178 -> break 75 -> { return JavaSyntaxTokenType.FLOAT_KEYWORD; } // fall through - 179 -> break; + 179 -> break 76 -> { return JavaSyntaxTokenType.SHORT_KEYWORD; } // fall through - 180 -> break; + 180 -> break 77 -> { return JavaSyntaxTokenType.SUPER_KEYWORD; } // fall through - 181 -> break; + 181 -> break 78 -> { return JavaSyntaxTokenType.THROW_KEYWORD; } // fall through - 182 -> break; + 182 -> break 79 -> { return JavaSyntaxTokenType.WHILE_KEYWORD; } // fall through - 183 -> break; + 183 -> break 80 -> { return if (myAssertKeyword) JavaSyntaxTokenType.ASSERT_KEYWORD else JavaSyntaxTokenType.IDENTIFIER; } // fall through - 184 -> break; + 184 -> break 81 -> { return JavaSyntaxTokenType.DOUBLE_KEYWORD; } // fall through - 185 -> break; + 185 -> break 82 -> { return JavaSyntaxTokenType.IMPORT_KEYWORD; } // fall through - 186 -> break; + 186 -> break 83 -> { return JavaSyntaxTokenType.NATIVE_KEYWORD; } // fall through - 187 -> break; + 187 -> break 84 -> { return JavaSyntaxTokenType.PUBLIC_KEYWORD; } // fall through - 188 -> break; + 188 -> break 85 -> { return JavaSyntaxTokenType.RETURN_KEYWORD; } // fall through - 189 -> break; + 189 -> break 86 -> { return JavaSyntaxTokenType.STATIC_KEYWORD; } // fall through - 190 -> break; + 190 -> break 87 -> { return JavaSyntaxTokenType.SWITCH_KEYWORD; } // fall through - 191 -> break; + 191 -> break 88 -> { return JavaSyntaxTokenType.THROWS_KEYWORD; } // fall through - 192 -> break; + 192 -> break 89 -> { return JavaSyntaxTokenType.BOOLEAN_KEYWORD; } // fall through - 193 -> break; + 193 -> break 90 -> { return JavaSyntaxTokenType.DEFAULT_KEYWORD; } // fall through - 194 -> break; + 194 -> break 91 -> { return JavaSyntaxTokenType.EXTENDS_KEYWORD; } // fall through - 195 -> break; + 195 -> break 92 -> { return JavaSyntaxTokenType.FINALLY_KEYWORD; } // fall through - 196 -> break; + 196 -> break 93 -> { return JavaSyntaxTokenType.PACKAGE_KEYWORD; } // fall through - 197 -> break; + 197 -> break 94 -> { return JavaSyntaxTokenType.PRIVATE_KEYWORD; } // fall through - 198 -> break; + 198 -> break 95 -> { return JavaSyntaxTokenType.ABSTRACT_KEYWORD; } // fall through - 199 -> break; + 199 -> break 96 -> { return JavaSyntaxTokenType.CONTINUE_KEYWORD; } // fall through - 200 -> break; + 200 -> break 97 -> { return JavaSyntaxTokenType.STRICTFP_KEYWORD; } // fall through - 201 -> break; + 201 -> break 98 -> { return JavaSyntaxTokenType.VOLATILE_KEYWORD; } // fall through - 202 -> break; + 202 -> break 99 -> { return JavaSyntaxTokenType.INTERFACE_KEYWORD; } // fall through - 203 -> break; + 203 -> break 100 -> { return JavaSyntaxTokenType.PROTECTED_KEYWORD; } // fall through - 204 -> break; + 204 -> break 101 -> { return JavaSyntaxTokenType.TRANSIENT_KEYWORD; } // fall through - 205 -> break; + 205 -> break 102 -> { return JavaSyntaxTokenType.IMPLEMENTS_KEYWORD; } // fall through - 206 -> break; + 206 -> break 103 -> { return JavaSyntaxTokenType.INSTANCEOF_KEYWORD; } // fall through - 207 -> break; + 207 -> break 104 -> { return JavaSyntaxTokenType.SYNCHRONIZED_KEYWORD; } // fall through - 208 -> break; + 208 -> break else -> - zzScanError(ZZ_NO_MATCH); + zzScanError(ZZ_NO_MATCH) } } } diff --git a/tools/lexer/idea-flex-kotlin.skeleton b/tools/lexer/idea-flex-kotlin.skeleton index 7cb46dd3c6df..0854d9eca621 100644 --- a/tools/lexer/idea-flex-kotlin.skeleton +++ b/tools/lexer/idea-flex-kotlin.skeleton @@ -1,6 +1,6 @@ companion object { /** This character denotes the end of file */ - private const val YYEOF = -1; + private const val YYEOF = -1 /** initial size of the lookahead buffer */ --- private static final int ZZ_BUFFERSIZE = ...; L1 @@ -31,7 +31,7 @@ companion object { /** this buffer contains the current text to be matched and is the source of the yytext() string */ - private var zzBuffer: CharSequence = ""; + private var zzBuffer: CharSequence = "" /** the textposition at the last accepting state */ private var zzMarkedPos = 0 @@ -123,7 +123,7 @@ companion object { * @return the character at position pos */ fun yycharat(position: Int): Char { - return zzBuffer[zzStartRead + position]; + return zzBuffer[zzStartRead + position] } @@ -131,7 +131,7 @@ companion object { * Returns the length of the matched text region. */ fun yylength(): Int { - return zzMarkedPos - zzStartRead; + return zzMarkedPos - zzStartRead } /** @@ -164,6 +164,37 @@ companion object { } return high.code } + + internal fun CharSequence.offsetByCodePoints(index: Int, codePointOffset: Int): Int { + val length = this.length + if (index < 0 || index > length) throw IndexOutOfBoundsException() + + var x = index + if (codePointOffset >= 0) { + var i: Int = 0 + + while (x < length && i < codePointOffset) { + if (this[x++].isHighSurrogate() && x < length && this[x].isLowSurrogate()) x++ + i++ + } + + if (i < codePointOffset) throw IndexOutOfBoundsException() + + } else { + var i: Int = codePointOffset + + while (x > 0 && i < 0) { + if (this[--x].isLowSurrogate() && x > 0 && this[x - 1].isHighSurrogate()) x-- + i++ + } + + if (i < 0) { + throw IndexOutOfBoundsException() + } + } + + return x + } internal fun Char.Companion.toCodePoint(high: Char, low: Char): Int = (((high - MIN_HIGH_SURROGATE) shl 10) or (low - MIN_LOW_SURROGATE)) + 0x10000 @@ -219,17 +250,17 @@ companion object { * @return the next token */ --- yylex declaration L10 - var zzInput: Int = 0; - var zzAction: Int = 0; + var zzInput: Int = 0 + var zzAction: Int = 0 --- local declarations L11 while (true) { // cached fields: var zzCurrentPosL = 0 - var zzMarkedPosL: Int = zzMarkedPos; - var zzEndReadL: Int = zzEndRead; - var zzBufferL: CharSequence = zzBuffer; + var zzMarkedPosL: Int = zzMarkedPos + var zzEndReadL: Int = zzEndRead + var zzBufferL: CharSequence = zzBuffer --- start admin (line, char, col count) L12 zzAction = -1