MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

made the triple quote string lexer a bit uglier but a bit more correct (PY-1777)

Browse Source
This commit is contained in:
Dmitry Jemerov
2011-02-22 19:17:34 +01:00
parent 803b545070
commit 51aefc768f
3 changed files with 58 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/src/com/jetbrains/python/lexer/Python.flex
+3 -3
View File
@@ -46,19 +46,19 @@ IMAGNUMBER=(({FLOATNUMBER})|({INTPART}))[Jj]
STRING_LITERAL=[UuBb]?({RAW_STRING}|{QUOTED_STRING})
RAW_STRING=[Rr]{QUOTED_STRING}
QUOTED_STRING=({TRIPLE_APOS_LITERAL})|({QUOTED_LITERAL})|({DOUBLE_QUOTED_LITERAL})|({TRIPLE_QUOTED_LITERAL})
QUOTED_LITERAL="'"([^\\\'\r\n]|{ESCAPE_SEQUENCE}|(\\[\r\n]))*("'"|\\)?
QUOTED_LITERAL="'" ([^\\\'\r\n] | {ESCAPE_SEQUENCE} | (\\[\r\n]))* ("'"|\\)?
DOUBLE_QUOTED_LITERAL=\"([^\\\"\r\n]|{ESCAPE_SEQUENCE}|(\\[\r\n]))*?(\"|\\)?
ESCAPE_SEQUENCE=\\[^\r\n]
ANY_ESCAPE_SEQUENCE = \\[^]
THREE_QUO = (\"\"\")
ONE_TWO_QUO = (\"[^\"]) | (\"\"[^\"])
ONE_TWO_QUO = (\"[^\"]) | (\"\\[^]) | (\"\"[^\"]) | (\"\"\\[^])
QUO_STRING_CHAR = [^\\\"] | {ANY_ESCAPE_SEQUENCE} | {ONE_TWO_QUO}
TRIPLE_QUOTED_LITERAL = {THREE_QUO} {QUO_STRING_CHAR}* {THREE_QUO}?
THREE_APOS = (\'\'\')
ONE_TWO_APOS = ('[^']) | (''[^'])
ONE_TWO_APOS = ('[^']) | ('\\[^]) | (''[^']) | (''\\[^])
APOS_STRING_CHAR = [^\\'] | {ANY_ESCAPE_SEQUENCE} | {ONE_TWO_APOS}
TRIPLE_APOS_LITERAL = {THREE_APOS} {APOS_STRING_CHAR}* {THREE_APOS}?
python/src/com/jetbrains/python/lexer/_PythonLexer.java
+47 -41
View File
@@ -1,4 +1,4 @@
/* The following code was generated by JFlex 1.4.3 on 08.09.10 17:19 */
/* The following code was generated by JFlex 1.4.3 on 22.02.11 16:31 */
/* It's an automatically generated code. Do not modify it. */
package com.jetbrains.python.lexer;
@@ -11,7 +11,7 @@ import com.jetbrains.python.PyTokenTypes;
/**
* This class is a scanner generated by
* <a href="http://www.jflex.de/">JFlex</a> 1.4.3
* on 08.09.10 17:19 from the specification file
* on 22.02.11 16:31 from the specification file
* <tt>C:/JetBrains/IDEA/tools/lexer/../../python/src/com/jetbrains/python/lexer/Python.flex</tt>
*/
class _PythonLexer implements FlexLexer {
@@ -226,13 +226,13 @@ class _PythonLexer implements FlexLexer {
"\1\70\1\71\3\2\1\43\1\0\1\72\11\3\3\7"+
"\1\73\1\3\1\74\1\75\1\76\1\77\7\3\1\100"+
"\1\101\1\102\1\103\1\104\4\3\1\105\1\106\2\3"+
"\4\0\4\3\1\107\4\3\1\110\3\3\1\111\2\0"+
"\2\3\1\112\1\3\1\113\1\3\1\114\1\3\1\115"+
"\1\116\1\117\1\120\1\3\1\121\1\122\1\123\1\3"+
"\1\124";
"\4\0\4\3\1\107\4\3\1\110\3\3\1\111\1\0"+
"\2\7\1\0\2\3\1\112\1\3\1\113\1\3\1\114"+
"\1\3\1\115\1\116\1\117\2\7\1\120\1\3\1\121"+
"\1\122\1\123\2\0\1\3\2\7\1\124";
private static int [] zzUnpackAction() {
int [] result = new int[195];
int [] result = new int[203];
int offset = 0;
offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result);
return result;
@@ -279,12 +279,13 @@ class _PythonLexer implements FlexLexer {
"\0\u1d40\0\u0168\0\u0168\0\u1d88\0\u1dd0\0\u1e18\0\u1e60\0\u1ea8"+
"\0\u1ef0\0\u1f38\0\u1f80\0\u1fc8\0\u2010\0\u0168\0\u2058\0\u20a0"+
"\0\u20e8\0\u2130\0\u0168\0\u2178\0\u21c0\0\u2208\0\u0168\0\u2250"+
"\0\u2298\0\u22e0\0\u2328\0\u0168\0\u2370\0\u0168\0\u23b8\0\u0168"+
"\0\u2400\0\u0168\0\u0168\0\u0168\0\u0168\0\u2448\0\u0168\0\u0168"+
"\0\u0168\0\u2490\0\u0168";
"\0\u2298\0\u22e0\0\u2328\0\u2370\0\u23b8\0\u0168\0\u2400\0\u0168"+
"\0\u2448\0\u0168\0\u2490\0\u0168\0\u0168\0\u0168\0\u24d8\0\u2520"+
"\0\u0168\0\u2568\0\u0168\0\u0168\0\u0168\0\u25b0\0\u25f8\0\u2640"+
"\0\u2250\0\u2328\0\u0168";
private static int [] zzUnpackRowMap() {
int [] result = new int[195];
int [] result = new int[203];
int offset = 0;
offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result);
return result;
@@ -441,37 +442,41 @@ class _PythonLexer implements FlexLexer {
"\30\0\11\6\2\0\7\6\1\0\2\6\1\0\4\6"+
"\1\256\7\0\16\6\30\0\11\6\2\0\7\6\1\0"+
"\1\257\1\6\1\0\5\6\7\0\16\6\27\0\34\202"+
"\1\260\163\202\146\203\1\261\51\203\1\0\11\6\2\0"+
"\7\6\1\0\2\6\1\0\4\6\1\262\7\0\16\6"+
"\1\260\1\261\162\202\145\203\1\262\1\263\51\203\1\0"+
"\11\6\2\0\7\6\1\0\2\6\1\0\4\6\1\264"+
"\7\0\16\6\30\0\11\6\2\0\7\6\1\0\2\6"+
"\1\0\5\6\7\0\7\6\1\265\6\6\30\0\11\6"+
"\2\0\7\6\1\0\2\6\1\0\5\6\7\0\3\6"+
"\1\266\12\6\30\0\11\6\2\0\7\6\1\0\2\6"+
"\1\0\4\6\1\267\7\0\16\6\30\0\11\6\2\0"+
"\7\6\1\0\2\6\1\0\5\6\7\0\2\6\1\270"+
"\13\6\30\0\11\6\2\0\7\6\1\0\2\6\1\0"+
"\5\6\7\0\1\271\15\6\30\0\11\6\2\0\7\6"+
"\1\0\1\272\1\6\1\0\5\6\7\0\16\6\30\0"+
"\10\6\1\273\2\0\7\6\1\0\2\6\1\0\5\6"+
"\7\0\16\6\30\0\11\6\2\0\7\6\1\0\2\6"+
"\1\0\5\6\7\0\1\274\15\6\30\0\11\6\2\0"+
"\7\6\1\0\2\6\1\0\5\6\7\0\4\6\1\275"+
"\11\6\30\0\11\6\2\0\7\6\1\0\2\6\1\0"+
"\5\6\7\0\1\6\1\276\14\6\27\0\34\202\1\201"+
"\1\261\106\202\1\277\1\261\52\202\35\203\1\262\1\300"+
"\106\203\1\262\1\201\51\203\1\0\11\6\2\0\7\6"+
"\1\0\2\6\1\0\5\6\7\0\4\6\1\301\11\6"+
"\30\0\11\6\2\0\7\6\1\0\2\6\1\0\5\6"+
"\7\0\7\6\1\263\6\6\30\0\11\6\2\0\7\6"+
"\1\0\2\6\1\0\5\6\7\0\3\6\1\264\12\6"+
"\30\0\11\6\2\0\7\6\1\0\2\6\1\0\4\6"+
"\1\265\7\0\16\6\30\0\11\6\2\0\7\6\1\0"+
"\2\6\1\0\5\6\7\0\2\6\1\266\13\6\30\0"+
"\11\6\2\0\7\6\1\0\2\6\1\0\5\6\7\0"+
"\1\267\15\6\30\0\11\6\2\0\7\6\1\0\1\270"+
"\1\6\1\0\5\6\7\0\16\6\30\0\10\6\1\271"+
"\2\0\7\6\1\0\2\6\1\0\5\6\7\0\16\6"+
"\30\0\11\6\2\0\7\6\1\0\2\6\1\0\5\6"+
"\7\0\1\272\15\6\30\0\11\6\2\0\7\6\1\0"+
"\2\6\1\0\5\6\7\0\4\6\1\273\11\6\30\0"+
"\11\6\2\0\7\6\1\0\2\6\1\0\5\6\7\0"+
"\1\6\1\274\14\6\27\0\34\202\1\201\53\202\36\203"+
"\1\201\51\203\1\0\11\6\2\0\7\6\1\0\2\6"+
"\1\0\5\6\7\0\4\6\1\275\11\6\30\0\11\6"+
"\2\0\7\6\1\0\2\6\1\0\5\6\7\0\1\6"+
"\1\276\14\6\30\0\11\6\2\0\7\6\1\0\2\6"+
"\1\0\5\6\7\0\4\6\1\277\11\6\30\0\10\6"+
"\1\300\2\0\7\6\1\0\2\6\1\0\5\6\7\0"+
"\16\6\30\0\11\6\2\0\7\6\1\0\2\6\1\0"+
"\5\6\7\0\11\6\1\301\4\6\30\0\11\6\2\0"+
"\7\6\1\0\2\6\1\0\2\6\1\302\2\6\7\0"+
"\16\6\30\0\11\6\2\0\7\6\1\0\1\303\1\6"+
"\1\0\5\6\7\0\16\6\27\0";
"\7\0\1\6\1\302\14\6\30\0\11\6\2\0\7\6"+
"\1\0\2\6\1\0\5\6\7\0\4\6\1\303\11\6"+
"\30\0\10\6\1\304\2\0\7\6\1\0\2\6\1\0"+
"\5\6\7\0\16\6\30\0\11\6\2\0\7\6\1\0"+
"\2\6\1\0\5\6\7\0\11\6\1\305\4\6\27\0"+
"\34\202\1\306\1\261\52\202\35\203\1\262\1\307\51\203"+
"\1\0\11\6\2\0\7\6\1\0\2\6\1\0\2\6"+
"\1\310\2\6\7\0\16\6\27\0\34\202\1\311\1\261"+
"\52\202\35\203\1\262\1\312\51\203\1\0\11\6\2\0"+
"\7\6\1\0\1\313\1\6\1\0\5\6\7\0\16\6"+
"\27\0";
private static int [] zzUnpackTrans() {
int [] result = new int[9432];
int [] result = new int[9864];
int offset = 0;
offset = zzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result);
return result;
@@ -516,10 +521,11 @@ class _PythonLexer implements FlexLexer {
"\1\1\4\0\1\11\1\1\1\0\1\11\10\1\1\11"+
"\27\1\3\11\1\1\1\11\1\1\5\11\1\1\2\11"+
"\1\1\1\11\4\1\1\0\12\1\1\11\17\1\4\11"+
"\11\1\4\0\16\1\2\0\22\1";
"\11\1\4\0\16\1\1\0\2\1\1\0\22\1\2\0"+
"\4\1";
private static int [] zzUnpackAttribute() {
int [] result = new int[195];
int [] result = new int[203];
int offset = 0;
offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result);
return result;
python/testSrc/com/jetbrains/python/PythonLexerTest.java
+8
View File
@@ -158,6 +158,14 @@ public class PythonLexerTest extends PyLexerTestCase {
doTest("'''abc\nd", "Py:STRING_LITERAL");
}
public void testEscapedClosingTripleApos() { // PY-1777
doTest("''' foo '\\''' bar '''", "Py:STRING_LITERAL");
}
public void testEscapedClosingTripleQuote() { // PY-1777
doTest(THREE_QUOTES + " foo \"\\" + THREE_QUOTES + " bar " + THREE_QUOTES, "Py:STRING_LITERAL");
}
public void testDedentBeforeComment() { // PY-2209 & friends
doTest("class UserProfile:\n" +
" pass\n" +