mirror of
https://gitflic.ru/project/openide/openide.git
synced 2025-12-16 22:51:17 +07:00
[python] Remove Python- and docstring type hint lexer duplicates
Lexers should be generated in modules where the corresponding grammar files belong. Namely Python's own lexer in intellij.python.parser, and the legacy type hint lexer in intellij.python.psi.impl. GitOrigin-RevId: 602387221c84e92811608aac6730eeb5a7bf2f90
This commit is contained in:
Mikhail Golubev
committed by
intellij-monorepo-bot
intellij-monorepo-bot
parent
c2de41bc55
commit
ecb85823a1
@@ -1,565 +0,0 @@
|
||||
// Generated by JFlex 1.9.1 http://jflex.de/ (tweaked for IntelliJ platform)
|
||||
// source: PyType.flex
|
||||
|
||||
package com.jetbrains.python.psi.types;
|
||||
|
||||
import com.intellij.lexer.FlexLexer;
|
||||
import com.jetbrains.python.psi.PyElementType;
|
||||
import static com.jetbrains.python.psi.types.PyTypeTokenTypes.*;
|
||||
|
||||
|
||||
|
||||
public class _PyTypeLexer implements FlexLexer {
|
||||
|
||||
/** This character denotes the end of file */
|
||||
public static final int YYEOF = -1;
|
||||
|
||||
/** initial size of the lookahead buffer */
|
||||
private static final int ZZ_BUFFERSIZE = 16384;
|
||||
|
||||
/** lexical states */
|
||||
public static final int YYINITIAL = 0;
|
||||
|
||||
/**
|
||||
* ZZ_LEXSTATE[l] is the state in the DFA for the lexical state l
|
||||
* ZZ_LEXSTATE[l+1] is the state in the DFA for the lexical state l
|
||||
* at the beginning of a line
|
||||
* l is of the form l = 2*k, k a non negative integer
|
||||
*/
|
||||
private static final int ZZ_LEXSTATE[] = {
|
||||
0, 0
|
||||
};
|
||||
|
||||
/**
|
||||
* Top-level table for translating characters to character classes
|
||||
*/
|
||||
private static final int [] ZZ_CMAP_TOP = zzUnpackcmap_top();
|
||||
|
||||
private static final String ZZ_CMAP_TOP_PACKED_0 =
|
||||
"\1\0\u10ff\u0100";
|
||||
|
||||
private static int [] zzUnpackcmap_top() {
|
||||
int [] result = new int[4352];
|
||||
int offset = 0;
|
||||
offset = zzUnpackcmap_top(ZZ_CMAP_TOP_PACKED_0, offset, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
private static int zzUnpackcmap_top(String packed, int offset, int [] result) {
|
||||
int i = 0; /* index in packed string */
|
||||
int j = offset; /* index in unpacked array */
|
||||
int l = packed.length();
|
||||
while (i < l) {
|
||||
int count = packed.charAt(i++);
|
||||
int value = packed.charAt(i++);
|
||||
do result[j++] = value; while (--count > 0);
|
||||
}
|
||||
return j;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Second-level tables for translating characters to character classes
|
||||
*/
|
||||
private static final int [] ZZ_CMAP_BLOCKS = zzUnpackcmap_blocks();
|
||||
|
||||
private static final String ZZ_CMAP_BLOCKS_PACKED_0 =
|
||||
"\11\0\1\1\1\2\2\0\1\2\22\0\1\1\1\3"+
|
||||
"\6\0\3\4\1\0\1\4\1\5\1\6\1\0\12\7"+
|
||||
"\1\10\1\0\1\11\1\12\1\13\2\0\23\14\7\15"+
|
||||
"\1\4\1\0\1\4\1\0\1\16\1\17\1\20\1\16"+
|
||||
"\1\21\2\16\1\22\5\16\1\23\1\24\1\16\1\25"+
|
||||
"\1\26\1\16\1\27\1\30\1\31\4\16\1\32\1\16"+
|
||||
"\1\33\1\4\1\34\1\3\u0181\0";
|
||||
|
||||
private static int [] zzUnpackcmap_blocks() {
|
||||
int [] result = new int[512];
|
||||
int offset = 0;
|
||||
offset = zzUnpackcmap_blocks(ZZ_CMAP_BLOCKS_PACKED_0, offset, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
private static int zzUnpackcmap_blocks(String packed, int offset, int [] result) {
|
||||
int i = 0; /* index in packed string */
|
||||
int j = offset; /* index in unpacked array */
|
||||
int l = packed.length();
|
||||
while (i < l) {
|
||||
int count = packed.charAt(i++);
|
||||
int value = packed.charAt(i++);
|
||||
do result[j++] = value; while (--count > 0);
|
||||
}
|
||||
return j;
|
||||
}
|
||||
|
||||
/**
|
||||
* Translates DFA states to action switch labels.
|
||||
*/
|
||||
private static final int [] ZZ_ACTION = zzUnpackAction();
|
||||
|
||||
private static final String ZZ_ACTION_PACKED_0 =
|
||||
"\1\0\1\1\1\2\1\3\1\0\1\3\2\0\1\4"+
|
||||
"\1\5\1\4\1\6\3\4\3\0\1\4\1\3\2\0"+
|
||||
"\1\4\5\0\1\6";
|
||||
|
||||
private static int [] zzUnpackAction() {
|
||||
int [] result = new int[29];
|
||||
int offset = 0;
|
||||
offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
private static int zzUnpackAction(String packed, int offset, int [] result) {
|
||||
int i = 0; /* index in packed string */
|
||||
int j = offset; /* index in unpacked array */
|
||||
int l = packed.length();
|
||||
while (i < l) {
|
||||
int count = packed.charAt(i++);
|
||||
int value = packed.charAt(i++);
|
||||
do result[j++] = value; while (--count > 0);
|
||||
}
|
||||
return j;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Translates a state to a row index in the transition table
|
||||
*/
|
||||
private static final int [] ZZ_ROWMAP = zzUnpackRowMap();
|
||||
|
||||
private static final String ZZ_ROWMAP_PACKED_0 =
|
||||
"\0\0\0\35\0\72\0\35\0\127\0\164\0\221\0\256"+
|
||||
"\0\313\0\313\0\350\0\35\0\u0105\0\u0122\0\u013f\0\u015c"+
|
||||
"\0\u0179\0\u0196\0\u01b3\0\350\0\u01d0\0\u01ed\0\u020a\0\u0227"+
|
||||
"\0\u0244\0\u0261\0\u027e\0\u029b\0\u02b8";
|
||||
|
||||
private static int [] zzUnpackRowMap() {
|
||||
int [] result = new int[29];
|
||||
int offset = 0;
|
||||
offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
private static int zzUnpackRowMap(String packed, int offset, int [] result) {
|
||||
int i = 0; /* index in packed string */
|
||||
int j = offset; /* index in unpacked array */
|
||||
int l = packed.length() - 1;
|
||||
while (i < l) {
|
||||
int high = packed.charAt(i++) << 16;
|
||||
result[j++] = high | packed.charAt(i++);
|
||||
}
|
||||
return j;
|
||||
}
|
||||
|
||||
/**
|
||||
* The transition table of the DFA
|
||||
*/
|
||||
private static final int [] ZZ_TRANS = zzUnpacktrans();
|
||||
|
||||
private static final String ZZ_TRANS_PACKED_0 =
|
||||
"\1\0\1\2\1\3\1\0\1\4\1\5\1\6\1\0"+
|
||||
"\1\7\1\10\2\0\1\11\1\12\1\13\1\14\2\13"+
|
||||
"\1\15\2\13\1\16\3\13\1\17\1\13\1\0\1\14"+
|
||||
"\37\0\1\3\45\0\1\4\27\0\1\20\47\0\1\21"+
|
||||
"\4\0\1\22\20\0\1\4\31\0\1\13\4\0\3\13"+
|
||||
"\1\0\13\13\1\14\10\0\1\13\4\0\3\13\1\0"+
|
||||
"\13\13\11\0\1\13\4\0\3\13\1\0\7\13\1\23"+
|
||||
"\3\13\11\0\1\13\4\0\3\13\1\0\2\13\1\24"+
|
||||
"\4\13\1\24\3\13\11\0\1\13\4\0\3\13\1\0"+
|
||||
"\5\13\1\24\5\13\10\0\1\4\51\0\1\25\43\0"+
|
||||
"\1\26\11\0\1\13\4\0\3\13\1\0\5\13\1\27"+
|
||||
"\5\13\22\0\1\30\24\0\1\31\33\0\1\13\4\0"+
|
||||
"\3\13\1\0\4\13\1\24\6\13\32\0\1\32\25\0"+
|
||||
"\1\21\43\0\1\33\14\0\1\34\43\0\1\35\20\0"+
|
||||
"\1\14\31\0";
|
||||
|
||||
private static int [] zzUnpacktrans() {
|
||||
int [] result = new int[725];
|
||||
int offset = 0;
|
||||
offset = zzUnpacktrans(ZZ_TRANS_PACKED_0, offset, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
private static int zzUnpacktrans(String packed, int offset, int [] result) {
|
||||
int i = 0; /* index in packed string */
|
||||
int j = offset; /* index in unpacked array */
|
||||
int l = packed.length();
|
||||
while (i < l) {
|
||||
int count = packed.charAt(i++);
|
||||
int value = packed.charAt(i++);
|
||||
value--;
|
||||
do result[j++] = value; while (--count > 0);
|
||||
}
|
||||
return j;
|
||||
}
|
||||
|
||||
|
||||
/* error codes */
|
||||
private static final int ZZ_UNKNOWN_ERROR = 0;
|
||||
private static final int ZZ_NO_MATCH = 1;
|
||||
private static final int ZZ_PUSHBACK_2BIG = 2;
|
||||
|
||||
/* error messages for the codes above */
|
||||
private static final String[] ZZ_ERROR_MSG = {
|
||||
"Unknown internal scanner error",
|
||||
"Error: could not match input",
|
||||
"Error: pushback value was too large"
|
||||
};
|
||||
|
||||
/**
|
||||
* ZZ_ATTRIBUTE[aState] contains the attributes of state {@code aState}
|
||||
*/
|
||||
private static final int [] ZZ_ATTRIBUTE = zzUnpackAttribute();
|
||||
|
||||
private static final String ZZ_ATTRIBUTE_PACKED_0 =
|
||||
"\1\0\1\11\1\1\1\11\1\0\1\1\2\0\3\1"+
|
||||
"\1\11\3\1\3\0\2\1\2\0\1\1\5\0\1\1";
|
||||
|
||||
private static int [] zzUnpackAttribute() {
|
||||
int [] result = new int[29];
|
||||
int offset = 0;
|
||||
offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
private static int zzUnpackAttribute(String packed, int offset, int [] result) {
|
||||
int i = 0; /* index in packed string */
|
||||
int j = offset; /* index in unpacked array */
|
||||
int l = packed.length();
|
||||
while (i < l) {
|
||||
int count = packed.charAt(i++);
|
||||
int value = packed.charAt(i++);
|
||||
do result[j++] = value; while (--count > 0);
|
||||
}
|
||||
return j;
|
||||
}
|
||||
|
||||
/** the input device */
|
||||
private java.io.Reader zzReader;
|
||||
|
||||
/** the current state of the DFA */
|
||||
private int zzState;
|
||||
|
||||
/** the current lexical state */
|
||||
private int zzLexicalState = YYINITIAL;
|
||||
|
||||
/** this buffer contains the current text to be matched and is
|
||||
the source of the yytext() string */
|
||||
private CharSequence zzBuffer = "";
|
||||
|
||||
/** the textposition at the last accepting state */
|
||||
private int zzMarkedPos;
|
||||
|
||||
/** the current text position in the buffer */
|
||||
private int zzCurrentPos;
|
||||
|
||||
/** startRead marks the beginning of the yytext() string in the buffer */
|
||||
private int zzStartRead;
|
||||
|
||||
/** endRead marks the last character in the buffer, that has been read
|
||||
from input */
|
||||
private int zzEndRead;
|
||||
|
||||
/** zzAtEOF == true <=> the scanner is at the EOF */
|
||||
private boolean zzAtEOF;
|
||||
|
||||
/** Number of newlines encountered up to the start of the matched text. */
|
||||
@SuppressWarnings("unused")
|
||||
private int yyline;
|
||||
|
||||
/** Number of characters from the last newline up to the start of the matched text. */
|
||||
@SuppressWarnings("unused")
|
||||
protected int yycolumn;
|
||||
|
||||
/** Number of characters up to the start of the matched text. */
|
||||
@SuppressWarnings("unused")
|
||||
private long yychar;
|
||||
|
||||
/** Whether the scanner is currently at the beginning of a line. */
|
||||
@SuppressWarnings("unused")
|
||||
private boolean zzAtBOL = true;
|
||||
|
||||
/** Whether the user-EOF-code has already been executed. */
|
||||
@SuppressWarnings("unused")
|
||||
private boolean zzEOFDone;
|
||||
|
||||
|
||||
/**
|
||||
* Creates a new scanner
|
||||
*
|
||||
* @param in the java.io.Reader to read input from.
|
||||
*/
|
||||
public _PyTypeLexer(java.io.Reader in) {
|
||||
this.zzReader = in;
|
||||
}
|
||||
|
||||
|
||||
/** Returns the maximum size of the scanner buffer, which limits the size of tokens. */
|
||||
private int zzMaxBufferLen() {
|
||||
return Integer.MAX_VALUE;
|
||||
}
|
||||
|
||||
/** Whether the scanner buffer can grow to accommodate a larger token. */
|
||||
private boolean zzCanGrow() {
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Translates raw input code points to DFA table row
|
||||
*/
|
||||
private static int zzCMap(int input) {
|
||||
int offset = input & 255;
|
||||
return offset == input ? ZZ_CMAP_BLOCKS[offset] : ZZ_CMAP_BLOCKS[ZZ_CMAP_TOP[input >> 8] | offset];
|
||||
}
|
||||
|
||||
public final int getTokenStart() {
|
||||
return zzStartRead;
|
||||
}
|
||||
|
||||
public final int getTokenEnd() {
|
||||
return getTokenStart() + yylength();
|
||||
}
|
||||
|
||||
public void reset(CharSequence buffer, int start, int end, int initialState) {
|
||||
zzBuffer = buffer;
|
||||
zzCurrentPos = zzMarkedPos = zzStartRead = start;
|
||||
zzAtEOF = false;
|
||||
zzAtBOL = true;
|
||||
zzEndRead = end;
|
||||
yybegin(initialState);
|
||||
}
|
||||
|
||||
/**
|
||||
* Refills the input buffer.
|
||||
*
|
||||
* @return {@code false}, iff there was new input.
|
||||
*
|
||||
* @exception java.io.IOException if any I/O-Error occurs
|
||||
*/
|
||||
private boolean zzRefill() throws java.io.IOException {
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns the current lexical state.
|
||||
*/
|
||||
public final int yystate() {
|
||||
return zzLexicalState;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Enters a new lexical state
|
||||
*
|
||||
* @param newState the new lexical state
|
||||
*/
|
||||
public final void yybegin(int newState) {
|
||||
zzLexicalState = newState;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns the text matched by the current regular expression.
|
||||
*/
|
||||
public final CharSequence yytext() {
|
||||
return zzBuffer.subSequence(zzStartRead, zzMarkedPos);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns the character at position {@code pos} from the
|
||||
* matched text.
|
||||
*
|
||||
* It is equivalent to yytext().charAt(pos), but faster
|
||||
*
|
||||
* @param pos the position of the character to fetch.
|
||||
* A value from 0 to yylength()-1.
|
||||
*
|
||||
* @return the character at position pos
|
||||
*/
|
||||
public final char yycharat(int pos) {
|
||||
return zzBuffer.charAt(zzStartRead+pos);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns the length of the matched text region.
|
||||
*/
|
||||
public final int yylength() {
|
||||
return zzMarkedPos-zzStartRead;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Reports an error that occurred while scanning.
|
||||
*
|
||||
* In a wellformed scanner (no or only correct usage of
|
||||
* yypushback(int) and a match-all fallback rule) this method
|
||||
* will only be called with things that "Can't Possibly Happen".
|
||||
* If this method is called, something is seriously wrong
|
||||
* (e.g. a JFlex bug producing a faulty scanner etc.).
|
||||
*
|
||||
* Usual syntax/scanner level error handling should be done
|
||||
* in error fallback rules.
|
||||
*
|
||||
* @param errorCode the code of the errormessage to display
|
||||
*/
|
||||
private void zzScanError(int errorCode) {
|
||||
String message;
|
||||
try {
|
||||
message = ZZ_ERROR_MSG[errorCode];
|
||||
}
|
||||
catch (ArrayIndexOutOfBoundsException e) {
|
||||
message = ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR];
|
||||
}
|
||||
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Pushes the specified amount of characters back into the input stream.
|
||||
*
|
||||
* They will be read again by then next call of the scanning method
|
||||
*
|
||||
* @param number the number of characters to be read again.
|
||||
* This number must not be greater than yylength()!
|
||||
*/
|
||||
public void yypushback(int number) {
|
||||
if ( number > yylength() )
|
||||
zzScanError(ZZ_PUSHBACK_2BIG);
|
||||
|
||||
zzMarkedPos -= number;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Resumes scanning until the next regular expression is matched,
|
||||
* the end of input is encountered or an I/O-Error occurs.
|
||||
*
|
||||
* @return the next token
|
||||
* @exception java.io.IOException if any I/O-Error occurs
|
||||
*/
|
||||
public PyElementType advance() throws java.io.IOException
|
||||
{
|
||||
int zzInput;
|
||||
int zzAction;
|
||||
|
||||
// cached fields:
|
||||
int zzCurrentPosL;
|
||||
int zzMarkedPosL;
|
||||
int zzEndReadL = zzEndRead;
|
||||
CharSequence zzBufferL = zzBuffer;
|
||||
|
||||
int [] zzTransL = ZZ_TRANS;
|
||||
int [] zzRowMapL = ZZ_ROWMAP;
|
||||
int [] zzAttrL = ZZ_ATTRIBUTE;
|
||||
|
||||
while (true) {
|
||||
zzMarkedPosL = zzMarkedPos;
|
||||
|
||||
zzAction = -1;
|
||||
|
||||
zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;
|
||||
|
||||
zzState = ZZ_LEXSTATE[zzLexicalState];
|
||||
|
||||
// set up zzAction for empty match case:
|
||||
int zzAttributes = zzAttrL[zzState];
|
||||
if ( (zzAttributes & 1) == 1 ) {
|
||||
zzAction = zzState;
|
||||
}
|
||||
|
||||
|
||||
zzForAction: {
|
||||
while (true) {
|
||||
|
||||
if (zzCurrentPosL < zzEndReadL) {
|
||||
zzInput = Character.codePointAt(zzBufferL, zzCurrentPosL);
|
||||
zzCurrentPosL += Character.charCount(zzInput);
|
||||
}
|
||||
else if (zzAtEOF) {
|
||||
zzInput = YYEOF;
|
||||
break zzForAction;
|
||||
}
|
||||
else {
|
||||
// store back cached positions
|
||||
zzCurrentPos = zzCurrentPosL;
|
||||
zzMarkedPos = zzMarkedPosL;
|
||||
boolean eof = zzRefill();
|
||||
// get translated positions and possibly new buffer
|
||||
zzCurrentPosL = zzCurrentPos;
|
||||
zzMarkedPosL = zzMarkedPos;
|
||||
zzBufferL = zzBuffer;
|
||||
zzEndReadL = zzEndRead;
|
||||
if (eof) {
|
||||
zzInput = YYEOF;
|
||||
break zzForAction;
|
||||
}
|
||||
else {
|
||||
zzInput = Character.codePointAt(zzBufferL, zzCurrentPosL);
|
||||
zzCurrentPosL += Character.charCount(zzInput);
|
||||
}
|
||||
}
|
||||
int zzNext = zzTransL[ zzRowMapL[zzState] + zzCMap(zzInput) ];
|
||||
if (zzNext == -1) break zzForAction;
|
||||
zzState = zzNext;
|
||||
|
||||
zzAttributes = zzAttrL[zzState];
|
||||
if ( (zzAttributes & 1) == 1 ) {
|
||||
zzAction = zzState;
|
||||
zzMarkedPosL = zzCurrentPosL;
|
||||
if ( (zzAttributes & 8) == 8 ) break zzForAction;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// store back cached position
|
||||
zzMarkedPos = zzMarkedPosL;
|
||||
|
||||
if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
|
||||
zzAtEOF = true;
|
||||
return null;
|
||||
}
|
||||
else {
|
||||
switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) {
|
||||
case 1:
|
||||
{ return SPACE;
|
||||
}
|
||||
// fall through
|
||||
case 7: break;
|
||||
case 2:
|
||||
{ return NL;
|
||||
}
|
||||
// fall through
|
||||
case 8: break;
|
||||
case 3:
|
||||
{ return OP;
|
||||
}
|
||||
// fall through
|
||||
case 9: break;
|
||||
case 4:
|
||||
{ return IDENTIFIER;
|
||||
}
|
||||
// fall through
|
||||
case 10: break;
|
||||
case 5:
|
||||
{ return PARAMETER;
|
||||
}
|
||||
// fall through
|
||||
case 11: break;
|
||||
case 6:
|
||||
{ return MARKUP;
|
||||
}
|
||||
// fall through
|
||||
case 12: break;
|
||||
default:
|
||||
zzScanError(ZZ_NO_MATCH);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user