From 51aefc768f20a2c462c875c0ecd47a3a6d58767a Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Tue, 22 Feb 2011 16:33:06 +0100 Subject: [PATCH] made the triple quote string lexer a bit uglier but a bit more correct (PY-1777) --- .../com/jetbrains/python/lexer/Python.flex | 6 +- .../jetbrains/python/lexer/_PythonLexer.java | 88 ++++++++++--------- .../com/jetbrains/python/PythonLexerTest.java | 8 ++ 3 files changed, 58 insertions(+), 44 deletions(-) diff --git a/python/src/com/jetbrains/python/lexer/Python.flex b/python/src/com/jetbrains/python/lexer/Python.flex index d71019ba098e..0c1786c630ab 100644 --- a/python/src/com/jetbrains/python/lexer/Python.flex +++ b/python/src/com/jetbrains/python/lexer/Python.flex @@ -46,19 +46,19 @@ IMAGNUMBER=(({FLOATNUMBER})|({INTPART}))[Jj] STRING_LITERAL=[UuBb]?({RAW_STRING}|{QUOTED_STRING}) RAW_STRING=[Rr]{QUOTED_STRING} QUOTED_STRING=({TRIPLE_APOS_LITERAL})|({QUOTED_LITERAL})|({DOUBLE_QUOTED_LITERAL})|({TRIPLE_QUOTED_LITERAL}) -QUOTED_LITERAL="'"([^\\\'\r\n]|{ESCAPE_SEQUENCE}|(\\[\r\n]))*("'"|\\)? +QUOTED_LITERAL="'" ([^\\\'\r\n] | {ESCAPE_SEQUENCE} | (\\[\r\n]))* ("'"|\\)? DOUBLE_QUOTED_LITERAL=\"([^\\\"\r\n]|{ESCAPE_SEQUENCE}|(\\[\r\n]))*?(\"|\\)? ESCAPE_SEQUENCE=\\[^\r\n] ANY_ESCAPE_SEQUENCE = \\[^] THREE_QUO = (\"\"\") -ONE_TWO_QUO = (\"[^\"]) | (\"\"[^\"]) +ONE_TWO_QUO = (\"[^\"]) | (\"\\[^]) | (\"\"[^\"]) | (\"\"\\[^]) QUO_STRING_CHAR = [^\\\"] | {ANY_ESCAPE_SEQUENCE} | {ONE_TWO_QUO} TRIPLE_QUOTED_LITERAL = {THREE_QUO} {QUO_STRING_CHAR}* {THREE_QUO}? THREE_APOS = (\'\'\') -ONE_TWO_APOS = ('[^']) | (''[^']) +ONE_TWO_APOS = ('[^']) | ('\\[^]) | (''[^']) | (''\\[^]) APOS_STRING_CHAR = [^\\'] | {ANY_ESCAPE_SEQUENCE} | {ONE_TWO_APOS} TRIPLE_APOS_LITERAL = {THREE_APOS} {APOS_STRING_CHAR}* {THREE_APOS}? diff --git a/python/src/com/jetbrains/python/lexer/_PythonLexer.java b/python/src/com/jetbrains/python/lexer/_PythonLexer.java index 57a03a46276e..8be7a2def2b3 100644 --- a/python/src/com/jetbrains/python/lexer/_PythonLexer.java +++ b/python/src/com/jetbrains/python/lexer/_PythonLexer.java @@ -1,4 +1,4 @@ -/* The following code was generated by JFlex 1.4.3 on 08.09.10 17:19 */ +/* The following code was generated by JFlex 1.4.3 on 22.02.11 16:31 */ /* It's an automatically generated code. Do not modify it. */ package com.jetbrains.python.lexer; @@ -11,7 +11,7 @@ import com.jetbrains.python.PyTokenTypes; /** * This class is a scanner generated by * JFlex 1.4.3 - * on 08.09.10 17:19 from the specification file + * on 22.02.11 16:31 from the specification file * C:/JetBrains/IDEA/tools/lexer/../../python/src/com/jetbrains/python/lexer/Python.flex */ class _PythonLexer implements FlexLexer { @@ -226,13 +226,13 @@ class _PythonLexer implements FlexLexer { "\1\70\1\71\3\2\1\43\1\0\1\72\11\3\3\7"+ "\1\73\1\3\1\74\1\75\1\76\1\77\7\3\1\100"+ "\1\101\1\102\1\103\1\104\4\3\1\105\1\106\2\3"+ - "\4\0\4\3\1\107\4\3\1\110\3\3\1\111\2\0"+ - "\2\3\1\112\1\3\1\113\1\3\1\114\1\3\1\115"+ - "\1\116\1\117\1\120\1\3\1\121\1\122\1\123\1\3"+ - "\1\124"; + "\4\0\4\3\1\107\4\3\1\110\3\3\1\111\1\0"+ + "\2\7\1\0\2\3\1\112\1\3\1\113\1\3\1\114"+ + "\1\3\1\115\1\116\1\117\2\7\1\120\1\3\1\121"+ + "\1\122\1\123\2\0\1\3\2\7\1\124"; private static int [] zzUnpackAction() { - int [] result = new int[195]; + int [] result = new int[203]; int offset = 0; offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result); return result; @@ -279,12 +279,13 @@ class _PythonLexer implements FlexLexer { "\0\u1d40\0\u0168\0\u0168\0\u1d88\0\u1dd0\0\u1e18\0\u1e60\0\u1ea8"+ "\0\u1ef0\0\u1f38\0\u1f80\0\u1fc8\0\u2010\0\u0168\0\u2058\0\u20a0"+ "\0\u20e8\0\u2130\0\u0168\0\u2178\0\u21c0\0\u2208\0\u0168\0\u2250"+ - "\0\u2298\0\u22e0\0\u2328\0\u0168\0\u2370\0\u0168\0\u23b8\0\u0168"+ - "\0\u2400\0\u0168\0\u0168\0\u0168\0\u0168\0\u2448\0\u0168\0\u0168"+ - "\0\u0168\0\u2490\0\u0168"; + "\0\u2298\0\u22e0\0\u2328\0\u2370\0\u23b8\0\u0168\0\u2400\0\u0168"+ + "\0\u2448\0\u0168\0\u2490\0\u0168\0\u0168\0\u0168\0\u24d8\0\u2520"+ + "\0\u0168\0\u2568\0\u0168\0\u0168\0\u0168\0\u25b0\0\u25f8\0\u2640"+ + "\0\u2250\0\u2328\0\u0168"; private static int [] zzUnpackRowMap() { - int [] result = new int[195]; + int [] result = new int[203]; int offset = 0; offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result); return result; @@ -441,37 +442,41 @@ class _PythonLexer implements FlexLexer { "\30\0\11\6\2\0\7\6\1\0\2\6\1\0\4\6"+ "\1\256\7\0\16\6\30\0\11\6\2\0\7\6\1\0"+ "\1\257\1\6\1\0\5\6\7\0\16\6\27\0\34\202"+ - "\1\260\163\202\146\203\1\261\51\203\1\0\11\6\2\0"+ - "\7\6\1\0\2\6\1\0\4\6\1\262\7\0\16\6"+ + "\1\260\1\261\162\202\145\203\1\262\1\263\51\203\1\0"+ + "\11\6\2\0\7\6\1\0\2\6\1\0\4\6\1\264"+ + "\7\0\16\6\30\0\11\6\2\0\7\6\1\0\2\6"+ + "\1\0\5\6\7\0\7\6\1\265\6\6\30\0\11\6"+ + "\2\0\7\6\1\0\2\6\1\0\5\6\7\0\3\6"+ + "\1\266\12\6\30\0\11\6\2\0\7\6\1\0\2\6"+ + "\1\0\4\6\1\267\7\0\16\6\30\0\11\6\2\0"+ + "\7\6\1\0\2\6\1\0\5\6\7\0\2\6\1\270"+ + "\13\6\30\0\11\6\2\0\7\6\1\0\2\6\1\0"+ + "\5\6\7\0\1\271\15\6\30\0\11\6\2\0\7\6"+ + "\1\0\1\272\1\6\1\0\5\6\7\0\16\6\30\0"+ + "\10\6\1\273\2\0\7\6\1\0\2\6\1\0\5\6"+ + "\7\0\16\6\30\0\11\6\2\0\7\6\1\0\2\6"+ + "\1\0\5\6\7\0\1\274\15\6\30\0\11\6\2\0"+ + "\7\6\1\0\2\6\1\0\5\6\7\0\4\6\1\275"+ + "\11\6\30\0\11\6\2\0\7\6\1\0\2\6\1\0"+ + "\5\6\7\0\1\6\1\276\14\6\27\0\34\202\1\201"+ + "\1\261\106\202\1\277\1\261\52\202\35\203\1\262\1\300"+ + "\106\203\1\262\1\201\51\203\1\0\11\6\2\0\7\6"+ + "\1\0\2\6\1\0\5\6\7\0\4\6\1\301\11\6"+ "\30\0\11\6\2\0\7\6\1\0\2\6\1\0\5\6"+ - "\7\0\7\6\1\263\6\6\30\0\11\6\2\0\7\6"+ - "\1\0\2\6\1\0\5\6\7\0\3\6\1\264\12\6"+ - "\30\0\11\6\2\0\7\6\1\0\2\6\1\0\4\6"+ - "\1\265\7\0\16\6\30\0\11\6\2\0\7\6\1\0"+ - "\2\6\1\0\5\6\7\0\2\6\1\266\13\6\30\0"+ - "\11\6\2\0\7\6\1\0\2\6\1\0\5\6\7\0"+ - "\1\267\15\6\30\0\11\6\2\0\7\6\1\0\1\270"+ - "\1\6\1\0\5\6\7\0\16\6\30\0\10\6\1\271"+ - "\2\0\7\6\1\0\2\6\1\0\5\6\7\0\16\6"+ - "\30\0\11\6\2\0\7\6\1\0\2\6\1\0\5\6"+ - "\7\0\1\272\15\6\30\0\11\6\2\0\7\6\1\0"+ - "\2\6\1\0\5\6\7\0\4\6\1\273\11\6\30\0"+ - "\11\6\2\0\7\6\1\0\2\6\1\0\5\6\7\0"+ - "\1\6\1\274\14\6\27\0\34\202\1\201\53\202\36\203"+ - "\1\201\51\203\1\0\11\6\2\0\7\6\1\0\2\6"+ - "\1\0\5\6\7\0\4\6\1\275\11\6\30\0\11\6"+ - "\2\0\7\6\1\0\2\6\1\0\5\6\7\0\1\6"+ - "\1\276\14\6\30\0\11\6\2\0\7\6\1\0\2\6"+ - "\1\0\5\6\7\0\4\6\1\277\11\6\30\0\10\6"+ - "\1\300\2\0\7\6\1\0\2\6\1\0\5\6\7\0"+ - "\16\6\30\0\11\6\2\0\7\6\1\0\2\6\1\0"+ - "\5\6\7\0\11\6\1\301\4\6\30\0\11\6\2\0"+ - "\7\6\1\0\2\6\1\0\2\6\1\302\2\6\7\0"+ - "\16\6\30\0\11\6\2\0\7\6\1\0\1\303\1\6"+ - "\1\0\5\6\7\0\16\6\27\0"; + "\7\0\1\6\1\302\14\6\30\0\11\6\2\0\7\6"+ + "\1\0\2\6\1\0\5\6\7\0\4\6\1\303\11\6"+ + "\30\0\10\6\1\304\2\0\7\6\1\0\2\6\1\0"+ + "\5\6\7\0\16\6\30\0\11\6\2\0\7\6\1\0"+ + "\2\6\1\0\5\6\7\0\11\6\1\305\4\6\27\0"+ + "\34\202\1\306\1\261\52\202\35\203\1\262\1\307\51\203"+ + "\1\0\11\6\2\0\7\6\1\0\2\6\1\0\2\6"+ + "\1\310\2\6\7\0\16\6\27\0\34\202\1\311\1\261"+ + "\52\202\35\203\1\262\1\312\51\203\1\0\11\6\2\0"+ + "\7\6\1\0\1\313\1\6\1\0\5\6\7\0\16\6"+ + "\27\0"; private static int [] zzUnpackTrans() { - int [] result = new int[9432]; + int [] result = new int[9864]; int offset = 0; offset = zzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result); return result; @@ -516,10 +521,11 @@ class _PythonLexer implements FlexLexer { "\1\1\4\0\1\11\1\1\1\0\1\11\10\1\1\11"+ "\27\1\3\11\1\1\1\11\1\1\5\11\1\1\2\11"+ "\1\1\1\11\4\1\1\0\12\1\1\11\17\1\4\11"+ - "\11\1\4\0\16\1\2\0\22\1"; + "\11\1\4\0\16\1\1\0\2\1\1\0\22\1\2\0"+ + "\4\1"; private static int [] zzUnpackAttribute() { - int [] result = new int[195]; + int [] result = new int[203]; int offset = 0; offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result); return result; diff --git a/python/testSrc/com/jetbrains/python/PythonLexerTest.java b/python/testSrc/com/jetbrains/python/PythonLexerTest.java index 454477fe4434..322ff23b144f 100644 --- a/python/testSrc/com/jetbrains/python/PythonLexerTest.java +++ b/python/testSrc/com/jetbrains/python/PythonLexerTest.java @@ -158,6 +158,14 @@ public class PythonLexerTest extends PyLexerTestCase { doTest("'''abc\nd", "Py:STRING_LITERAL"); } + public void testEscapedClosingTripleApos() { // PY-1777 + doTest("''' foo '\\''' bar '''", "Py:STRING_LITERAL"); + } + + public void testEscapedClosingTripleQuote() { // PY-1777 + doTest(THREE_QUOTES + " foo \"\\" + THREE_QUOTES + " bar " + THREE_QUOTES, "Py:STRING_LITERAL"); + } + public void testDedentBeforeComment() { // PY-2209 & friends doTest("class UserProfile:\n" + " pass\n" +