From ab9622ffc6694dd33600832b68cd655ead6d6d89 Mon Sep 17 00:00:00 2001 From: Ekaterina Tuzova Date: Fri, 29 Mar 2013 16:01:27 +0400 Subject: [PATCH] fixed PY-9274 Broken raw bytes literal parsing for Python 3.3 refactored lexer, added compatibility warnings, quickfix to remove not only U prefix --- .../com/jetbrains/python/PyBundle.properties | 3 +- .../PyCompatibilityInspection.java | 19 +- ...uickFix.java => RemovePrefixQuickFix.java} | 17 +- .../python/lexer/PyStringLiteralLexer.java | 38 +- .../com/jetbrains/python/lexer/Python.flex | 4 +- .../jetbrains/python/lexer/_PythonLexer.java | 1117 +++++++++-------- .../impl/PyStringLiteralExpressionImpl.java | 29 +- .../validation/CompatibilityVisitor.java | 67 +- .../validation/UnsupportedFeatures.java | 28 +- .../unsupportedFeaturesInPython3.py | 2 +- .../stringLiteralExpression.py | 43 +- .../com/jetbrains/python/PyIntentionTest.java | 2 +- 12 files changed, 745 insertions(+), 624 deletions(-) rename python/src/com/jetbrains/python/inspections/quickfix/{RemoveLeadingUQuickFix.java => RemovePrefixQuickFix.java} (65%) diff --git a/python/src/com/jetbrains/python/PyBundle.properties b/python/src/com/jetbrains/python/PyBundle.properties index def59ebbf6d8..118009781607 100644 --- a/python/src/com/jetbrains/python/PyBundle.properties +++ b/python/src/com/jetbrains/python/PyBundle.properties @@ -140,7 +140,8 @@ INTN.convert.dict.comp.to=Convert dictionary comprehension to 'dict' method call INTN.replace.noteq.operator=Replace not equal operator -INTN.remove.leading.u=Remove leading U +INTN.remove.leading.$0=Remove leading {0} +INTN.remove.leading.prefix=Remove prefix INTN.remove.trailing.l=Remove trailing L diff --git a/python/src/com/jetbrains/python/inspections/PyCompatibilityInspection.java b/python/src/com/jetbrains/python/inspections/PyCompatibilityInspection.java index 632525a88747..11d950fba567 100644 --- a/python/src/com/jetbrains/python/inspections/PyCompatibilityInspection.java +++ b/python/src/com/jetbrains/python/inspections/PyCompatibilityInspection.java @@ -7,6 +7,7 @@ import com.intellij.openapi.application.ApplicationManager; import com.intellij.openapi.roots.ProjectFileIndex; import com.intellij.openapi.roots.ProjectRootManager; import com.intellij.openapi.util.JDOMExternalizableStringList; +import com.intellij.openapi.util.TextRange; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.psi.PsiElement; import com.intellij.psi.PsiElementVisitor; @@ -128,14 +129,22 @@ public class PyCompatibilityInspection extends PyInspection { @Override protected final void registerProblem(@Nullable final PsiElement element, @NotNull final String message, - @Nullable final LocalQuickFix quickFix, final boolean asError){ - if (element == null || element.getTextLength() == 0){ - return; + @Nullable final LocalQuickFix quickFix, final boolean asError) { + if (element == null) return; + registerProblem(element, element.getTextRange(), message, quickFix, asError); + } + + @Override + protected void registerProblem(@NotNull final PsiElement element, @NotNull TextRange range, String message, + @Nullable LocalQuickFix quickFix, boolean asError) { + if (element.getTextLength() == 0) { + return; } + range = TextRange.create(range.getStartOffset() - element.getTextOffset(), range.getEndOffset() - element.getTextOffset()); if (quickFix != null) - myHolder.registerProblem(element, message, quickFix); + myHolder.registerProblem(element, range, message, quickFix); else - myHolder.registerProblem(element, message); + myHolder.registerProblem(element, range, message); } @Override diff --git a/python/src/com/jetbrains/python/inspections/quickfix/RemoveLeadingUQuickFix.java b/python/src/com/jetbrains/python/inspections/quickfix/RemovePrefixQuickFix.java similarity index 65% rename from python/src/com/jetbrains/python/inspections/quickfix/RemoveLeadingUQuickFix.java rename to python/src/com/jetbrains/python/inspections/quickfix/RemovePrefixQuickFix.java index f08543de8cf2..8ae93dfb69d9 100644 --- a/python/src/com/jetbrains/python/inspections/quickfix/RemoveLeadingUQuickFix.java +++ b/python/src/com/jetbrains/python/inspections/quickfix/RemovePrefixQuickFix.java @@ -7,6 +7,7 @@ import com.intellij.psi.PsiElement; import com.jetbrains.python.PyBundle; import com.jetbrains.python.psi.PyElementGenerator; import com.jetbrains.python.psi.PyStringLiteralExpression; +import com.jetbrains.python.psi.impl.PyStringLiteralExpressionImpl; import org.jetbrains.annotations.NotNull; /** @@ -15,16 +16,23 @@ import org.jetbrains.annotations.NotNull; * Date: 06.03.2010 * Time: 16:50:53 */ -public class RemoveLeadingUQuickFix implements LocalQuickFix { +public class RemovePrefixQuickFix implements LocalQuickFix { + private final String myPrefix; + + public RemovePrefixQuickFix(String prefix) { + myPrefix = prefix; + } + @NotNull + @Override public String getName() { - return PyBundle.message("INTN.remove.leading.u"); + return PyBundle.message("INTN.remove.leading.$0", myPrefix); } @NotNull public String getFamilyName() { - return getName(); + return PyBundle.message("INTN.remove.leading.prefix"); } @Override @@ -32,7 +40,8 @@ public class RemoveLeadingUQuickFix implements LocalQuickFix { PsiElement stringLiteralExpression = descriptor.getPsiElement(); if (stringLiteralExpression instanceof PyStringLiteralExpression) { PyElementGenerator elementGenerator = PyElementGenerator.getInstance(project); - stringLiteralExpression.replace(elementGenerator.createExpressionFromText(stringLiteralExpression.getText().substring(1))); + final int length = PyStringLiteralExpressionImpl.getPrefixLength(stringLiteralExpression.getText()); + stringLiteralExpression.replace(elementGenerator.createExpressionFromText(stringLiteralExpression.getText().substring(length))); } } } \ No newline at end of file diff --git a/python/src/com/jetbrains/python/lexer/PyStringLiteralLexer.java b/python/src/com/jetbrains/python/lexer/PyStringLiteralLexer.java index 6aec4fd6557b..1b35d64ac8a2 100644 --- a/python/src/com/jetbrains/python/lexer/PyStringLiteralLexer.java +++ b/python/src/com/jetbrains/python/lexer/PyStringLiteralLexer.java @@ -57,22 +57,18 @@ public class PyStringLiteralLexer extends LexerBase { // the following could be parsing steps if we wanted this info as tokens int i = myStart; - // unicode flag - char c = buffer.charAt(i); - if (c == 'u' || c == 'U' || c == 'b' || c == 'B' || c == 'c' || c == 'C') - i += 1; - - // raw flag - c = buffer.charAt(i); - if (c == 'r' || c == 'R') { - myIsRaw = true; - i += 1; - } - else myIsRaw = false; + i = skipEncodingPrefix(buffer, i); + int offset = skipRawPrefix(buffer, i); + if (offset > i) myIsRaw = true; + i = offset; + i = skipEncodingPrefix(buffer, i); + offset = skipRawPrefix(buffer, i); + if (offset > i) myIsRaw = true; + i = offset; // which quote char? - c = buffer.charAt(i); + char c = buffer.charAt(i); assert (c == '"') || (c == '\'') : "String must be quoted by single or double quote"; myQuoteChar = c; @@ -80,6 +76,22 @@ public class PyStringLiteralLexer extends LexerBase { myEnd = locateToken(myStart); } + public static int skipRawPrefix(CharSequence text, int startOffset) { + char c = Character.toUpperCase(text.charAt(startOffset)); + if (c == 'R') { + startOffset++; + } + return startOffset; + } + + public static int skipEncodingPrefix(CharSequence text, int startOffset) { + char c = Character.toUpperCase(text.charAt(startOffset)); + if (c == 'U' || c == 'B' || c == 'C') { + startOffset++; + } + return startOffset; + } + public int getState() { return myLastState; } diff --git a/python/src/com/jetbrains/python/lexer/Python.flex b/python/src/com/jetbrains/python/lexer/Python.flex index c7dcb46acc12..7ccfa97ae3f6 100644 --- a/python/src/com/jetbrains/python/lexer/Python.flex +++ b/python/src/com/jetbrains/python/lexer/Python.flex @@ -48,8 +48,8 @@ IMAGNUMBER=(({FLOATNUMBER})|({INTPART}))[Jj] //RAW_STRING=[Rr]{QUOTED_STRING} //QUOTED_STRING=({TRIPLE_APOS_LITERAL})|({QUOTED_LITERAL})|({DOUBLE_QUOTED_LITERAL})|({TRIPLE_QUOTED_LITERAL}) -SINGLE_QUOTED_STRING=[UuBbCc]?[Rr]?({QUOTED_LITERAL} | {DOUBLE_QUOTED_LITERAL}) -TRIPLE_QUOTED_STRING=[UuBbCc]?[Rr]?({TRIPLE_QUOTED_LITERAL}|{TRIPLE_APOS_LITERAL}) +SINGLE_QUOTED_STRING=[UuBbCcRr]{0,2}({QUOTED_LITERAL} | {DOUBLE_QUOTED_LITERAL}) +TRIPLE_QUOTED_STRING=[UuBbCcRr]{0,2}[UuBbCcRr]?({TRIPLE_QUOTED_LITERAL}|{TRIPLE_APOS_LITERAL}) DOCSTRING_LITERAL=({SINGLE_QUOTED_STRING}|{TRIPLE_QUOTED_STRING}) diff --git a/python/src/com/jetbrains/python/lexer/_PythonLexer.java b/python/src/com/jetbrains/python/lexer/_PythonLexer.java index d02de6d37b9d..ae682695afb8 100644 --- a/python/src/com/jetbrains/python/lexer/_PythonLexer.java +++ b/python/src/com/jetbrains/python/lexer/_PythonLexer.java @@ -1,4 +1,4 @@ -/* The following code was generated by JFlex 1.4.3 on 8/21/12 6:27 PM */ +/* The following code was generated by JFlex 1.4.3 on 3/28/13 3:36 PM */ /* It's an automatically generated code. Do not modify it. */ package com.jetbrains.python.lexer; @@ -12,7 +12,7 @@ import com.intellij.openapi.util.text.StringUtil; /** * This class is a scanner generated by * JFlex 1.4.3 - * on 8/21/12 6:27 PM from the specification file + * on 3/28/13 3:36 PM from the specification file * ./Python.flex */ class _PythonLexer implements FlexLexer { @@ -31,23 +31,23 @@ class _PythonLexer implements FlexLexer { * at the beginning of a line * l is of the form l = 2*k, k a non negative integer */ - private static final int ZZ_LEXSTATE[] = { - 0, 0, 1, 1, 2, 2, 3, 3 + private static final int ZZ_LEXSTATE[] = { + 0, 0, 1, 1, 2, 2, 3, 3 }; - /** + /** * Translates characters to character classes */ - private static final String ZZ_CMAP_PACKED = - "\11\0\1\43\1\41\1\0\1\44\1\13\22\0\1\42\1\74\1\40"+ - "\1\12\1\0\1\66\1\67\1\36\1\76\1\77\1\64\1\62\1\105"+ - "\1\26\1\23\1\65\1\1\1\5\6\3\2\2\1\106\1\110\1\73"+ - "\1\63\1\72\1\0\1\104\1\15\1\20\1\31\1\15\1\25\1\15"+ - "\3\14\1\27\1\14\1\21\2\14\1\17\2\14\1\34\2\14\1\32"+ - "\2\14\1\16\2\14\1\100\1\37\1\101\1\71\1\14\1\107\1\45"+ - "\1\10\1\30\1\47\1\24\1\4\1\57\1\61\1\53\1\27\1\52"+ - "\1\11\1\56\1\46\1\7\1\54\1\14\1\35\1\50\1\51\1\33"+ - "\1\14\1\60\1\6\1\55\1\14\1\102\1\70\1\103\1\75\53\0"+ + private static final String ZZ_CMAP_PACKED = + "\11\0\1\41\1\37\1\0\1\42\1\13\22\0\1\40\1\73\1\36"+ + "\1\12\1\0\1\65\1\66\1\34\1\75\1\76\1\63\1\61\1\104"+ + "\1\26\1\23\1\64\1\1\1\5\6\3\2\2\1\105\1\107\1\72"+ + "\1\62\1\71\1\0\1\103\1\15\1\20\1\31\1\15\1\25\1\15"+ + "\3\14\1\27\1\14\1\21\2\14\1\17\2\14\1\32\2\14\1\32"+ + "\2\14\1\16\2\14\1\77\1\35\1\100\1\70\1\14\1\106\1\43"+ + "\1\10\1\30\1\45\1\24\1\4\1\56\1\60\1\52\1\27\1\51"+ + "\1\11\1\55\1\44\1\7\1\53\1\14\1\47\1\46\1\50\1\33"+ + "\1\14\1\57\1\6\1\54\1\14\1\101\1\67\1\102\1\74\53\0"+ "\1\14\12\0\1\14\4\0\1\14\5\0\27\14\1\0\7\14\30\14"+ "\1\0\10\14\1\14\1\14\1\14\1\14\1\14\1\14\1\14\1\14"+ "\1\14\1\14\1\14\1\14\1\14\1\14\1\14\1\14\1\14\1\14"+ @@ -207,39 +207,40 @@ class _PythonLexer implements FlexLexer { "\1\22\1\0\32\14\12\0\1\22\12\14\1\14\55\14\2\14\37\14"+ "\3\0\6\14\2\0\6\14\2\0\6\14\2\0\3\14\43\0"; - /** + /** * Translates characters to character classes */ private static final char [] ZZ_CMAP = zzUnpackCMap(ZZ_CMAP_PACKED); - /** + /** * Translates DFA states to action switch labels. */ private static final int [] ZZ_ACTION = zzUnpackAction(); private static final String ZZ_ACTION_PACKED_0 = "\4\0\1\1\1\2\1\3\1\4\1\5\1\6\1\7"+ - "\3\10\2\11\1\12\2\13\6\14\1\15\1\14\1\16"+ - "\3\14\2\17\11\14\1\20\1\21\1\22\1\23\1\24"+ + "\2\10\2\11\1\12\2\13\6\14\1\15\1\14\1\16"+ + "\1\14\2\17\12\14\1\20\1\21\1\22\1\23\1\24"+ "\1\25\1\26\1\27\1\30\1\31\1\12\1\32\1\33"+ "\1\34\1\35\1\36\1\37\1\40\1\41\1\42\1\43"+ "\1\44\1\45\1\43\1\0\3\11\1\46\3\11\1\13"+ - "\4\0\1\13\1\47\1\0\1\50\3\14\1\51\4\14"+ - "\1\52\4\14\6\17\5\14\1\53\1\54\1\55\5\14"+ + "\4\0\1\13\1\47\1\0\1\50\3\14\1\51\5\14"+ + "\1\52\2\14\6\17\7\14\1\53\1\54\1\55\5\14"+ "\1\56\1\57\1\60\1\61\1\62\1\63\1\64\1\65"+ "\1\66\1\67\1\70\1\71\1\72\1\73\1\74\1\75"+ - "\1\76\1\0\1\11\1\77\1\100\1\0\1\77\3\13"+ - "\1\47\1\0\1\101\13\14\1\17\2\102\1\103\1\14"+ - "\1\104\1\105\1\106\1\107\5\14\1\110\1\111\1\112"+ - "\1\113\1\0\2\100\1\0\1\114\4\14\1\115\1\116"+ - "\4\14\4\0\2\14\1\117\3\14\1\0\2\77\1\0"+ - "\1\14\1\120\3\14\1\121\1\14\1\122\1\0\2\102"+ - "\1\0\2\14\1\123\1\14\1\124\3\77\1\14\1\125"+ - "\1\126\1\14\1\127\3\102\1\130\1\131\1\132\2\0"+ - "\1\133\1\14\2\0\2\77\1\134\2\102"; + "\1\76\2\0\1\11\1\77\1\100\1\0\1\77\3\13"+ + "\1\47\1\0\1\101\12\14\1\17\2\102\1\103\1\14"+ + "\1\104\1\105\1\106\2\14\1\107\5\14\1\110\1\111"+ + "\1\112\1\113\3\0\2\100\1\0\1\114\1\14\2\0"+ + "\3\14\1\115\1\116\2\14\4\0\4\14\1\117\3\14"+ + "\3\0\2\77\1\0\1\14\2\0\1\120\3\14\1\121"+ + "\1\0\2\102\1\0\2\14\1\122\1\14\1\123\1\14"+ + "\1\124\3\77\1\14\1\125\1\126\1\14\3\102\1\127"+ + "\1\130\1\131\1\132\2\0\1\133\1\14\2\0\2\77"+ + "\1\134\2\102"; private static int [] zzUnpackAction() { - int [] result = new int[244]; + int [] result = new int[253]; int offset = 0; offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result); return result; @@ -258,46 +259,47 @@ class _PythonLexer implements FlexLexer { } - /** + /** * Translates a state to a row index in the transition table */ private static final int [] ZZ_ROWMAP = zzUnpackRowMap(); private static final String ZZ_ROWMAP_PACKED_0 = - "\0\0\0\111\0\222\0\333\0\u0124\0\u016d\0\u0124\0\u0124"+ - "\0\u0124\0\u0124\0\u0124\0\u0124\0\u01b6\0\u01ff\0\u0248\0\u0291"+ - "\0\u0124\0\u02da\0\u0323\0\u036c\0\u03b5\0\u03fe\0\u0447\0\u0490"+ - "\0\u04d9\0\u0522\0\u056b\0\u05b4\0\u05fd\0\u0646\0\u068f\0\u06d8"+ - "\0\u0721\0\u076a\0\u07b3\0\u07fc\0\u0845\0\u088e\0\u08d7\0\u0920"+ - "\0\u0969\0\u09b2\0\u09fb\0\u0a44\0\u0a8d\0\u0ad6\0\u0b1f\0\u0b68"+ - "\0\u0bb1\0\u0bfa\0\u0c43\0\u0c8c\0\u0cd5\0\u0124\0\u0124\0\u0124"+ - "\0\u0124\0\u0124\0\u0124\0\u0124\0\u0124\0\u0124\0\u0d1e\0\u0124"+ - "\0\u0124\0\u0124\0\u01ff\0\u0d67\0\u0db0\0\u0df9\0\u0124\0\u0e42"+ - "\0\u0e8b\0\u0ed4\0\u0f1d\0\u0f66\0\u0faf\0\u0ff8\0\u1041\0\u0124"+ - "\0\u108a\0\u10d3\0\u0124\0\u111c\0\u1165\0\u11ae\0\u03b5\0\u11f7"+ - "\0\u1240\0\u1289\0\u12d2\0\u0124\0\u131b\0\u1364\0\u13ad\0\u13f6"+ - "\0\u143f\0\u1488\0\u14d1\0\u151a\0\u1563\0\u15ac\0\u15f5\0\u163e"+ - "\0\u1687\0\u16d0\0\u1719\0\u03b5\0\u03b5\0\u03b5\0\u1762\0\u17ab"+ - "\0\u17f4\0\u183d\0\u1886\0\u0124\0\u0124\0\u0124\0\u18cf\0\u0124"+ - "\0\u1918\0\u0124\0\u0124\0\u0124\0\u0124\0\u0124\0\u1961\0\u0124"+ - "\0\u0124\0\u19aa\0\u0124\0\u0124\0\u0d1e\0\u19f3\0\u1a3c\0\u0124"+ - "\0\u19f3\0\u1a85\0\u1ace\0\u1b17\0\u1b60\0\u1ba9\0\u1bf2\0\u03b5"+ - "\0\u1c3b\0\u1c84\0\u1ccd\0\u1d16\0\u1d5f\0\u1da8\0\u1df1\0\u1e3a"+ - "\0\u1e83\0\u1ecc\0\u1f15\0\u0124\0\u1f5e\0\u1fa7\0\u03b5\0\u1ff0"+ - "\0\u03b5\0\u03b5\0\u03b5\0\u03b5\0\u2039\0\u2082\0\u20cb\0\u2114"+ - "\0\u215d\0\u0124\0\u0124\0\u0124\0\u0124\0\u21a6\0\u21ef\0\u2238"+ - "\0\u2281\0\u03b5\0\u22ca\0\u2313\0\u235c\0\u23a5\0\u03b5\0\u03b5"+ - "\0\u23ee\0\u2437\0\u2480\0\u24c9\0\u2512\0\u255b\0\u25a4\0\u25ed"+ - "\0\u2636\0\u267f\0\u03b5\0\u26c8\0\u2711\0\u275a\0\u27a3\0\u27ec"+ - "\0\u2835\0\u287e\0\u28c7\0\u03b5\0\u2910\0\u2959\0\u29a2\0\u03b5"+ - "\0\u29eb\0\u03b5\0\u2a34\0\u2a7d\0\u2ac6\0\u2b0f\0\u2b58\0\u2ba1"+ - "\0\u03b5\0\u2bea\0\u03b5\0\u19f3\0\u2c33\0\u2c7c\0\u2cc5\0\u03b5"+ - "\0\u03b5\0\u2d0e\0\u03b5\0\u0124\0\u2d57\0\u2da0\0\u03b5\0\u03b5"+ - "\0\u03b5\0\u2de9\0\u2e32\0\u03b5\0\u2e7b\0\u2ec4\0\u2f0d\0\u27a3"+ - "\0\u287e\0\u03b5\0\u2a34\0\u2b0f"; + "\0\0\0\110\0\220\0\330\0\u0120\0\u0168\0\u0120\0\u0120"+ + "\0\u0120\0\u0120\0\u0120\0\u0120\0\u01b0\0\u01f8\0\u0240\0\u0120"+ + "\0\u0288\0\u02d0\0\u0318\0\u0360\0\u03a8\0\u03f0\0\u0438\0\u0480"+ + "\0\u04c8\0\u0510\0\u0558\0\u05a0\0\u05e8\0\u0630\0\u0678\0\u06c0"+ + "\0\u0708\0\u0750\0\u0798\0\u07e0\0\u0828\0\u0870\0\u08b8\0\u0900"+ + "\0\u0948\0\u0990\0\u09d8\0\u0a20\0\u0a68\0\u0ab0\0\u0af8\0\u0b40"+ + "\0\u0b88\0\u0bd0\0\u0c18\0\u0120\0\u0120\0\u0120\0\u0120\0\u0120"+ + "\0\u0120\0\u0120\0\u0120\0\u0120\0\u0c60\0\u0120\0\u0120\0\u0120"+ + "\0\u0ca8\0\u0cf0\0\u0d38\0\u0d80\0\u0120\0\u0dc8\0\u0e10\0\u0e58"+ + "\0\u0ea0\0\u0ee8\0\u0f30\0\u0f78\0\u0fc0\0\u0120\0\u1008\0\u1050"+ + "\0\u0120\0\u1098\0\u10e0\0\u1128\0\u0360\0\u1170\0\u11b8\0\u1200"+ + "\0\u1248\0\u1290\0\u0120\0\u12d8\0\u1320\0\u1368\0\u13b0\0\u13f8"+ + "\0\u1440\0\u1488\0\u14d0\0\u1518\0\u1560\0\u15a8\0\u15f0\0\u1638"+ + "\0\u1680\0\u16c8\0\u0360\0\u0360\0\u0360\0\u1710\0\u1758\0\u17a0"+ + "\0\u17e8\0\u1830\0\u0120\0\u0120\0\u0120\0\u1878\0\u0120\0\u18c0"+ + "\0\u0120\0\u0120\0\u0120\0\u0120\0\u0120\0\u1908\0\u0120\0\u0120"+ + "\0\u1950\0\u0120\0\u0120\0\u0c60\0\u1998\0\u19e0\0\u1a28\0\u0120"+ + "\0\u19e0\0\u1a70\0\u1ab8\0\u1b00\0\u1b48\0\u1b90\0\u1bd8\0\u0360"+ + "\0\u1c20\0\u1c68\0\u1cb0\0\u1cf8\0\u1d40\0\u1d88\0\u1dd0\0\u1e18"+ + "\0\u1e60\0\u1ea8\0\u0120\0\u1ef0\0\u1f38\0\u0360\0\u1f80\0\u0360"+ + "\0\u0360\0\u0360\0\u1fc8\0\u2010\0\u0360\0\u2058\0\u20a0\0\u20e8"+ + "\0\u2130\0\u2178\0\u0120\0\u0120\0\u0120\0\u0120\0\u21c0\0\u2208"+ + "\0\u2250\0\u2298\0\u22e0\0\u2328\0\u0360\0\u2370\0\u23b8\0\u2400"+ + "\0\u2448\0\u2490\0\u24d8\0\u0360\0\u0360\0\u2520\0\u2568\0\u25b0"+ + "\0\u25f8\0\u2640\0\u2688\0\u26d0\0\u2718\0\u2760\0\u27a8\0\u0360"+ + "\0\u27f0\0\u2838\0\u2880\0\u28c8\0\u2910\0\u2958\0\u29a0\0\u29e8"+ + "\0\u2a30\0\u2a78\0\u13b0\0\u14d0\0\u0360\0\u2ac0\0\u2b08\0\u2b50"+ + "\0\u0360\0\u2b98\0\u2be0\0\u2c28\0\u2c70\0\u2cb8\0\u2d00\0\u0360"+ + "\0\u2d48\0\u0360\0\u2d90\0\u0360\0\u19e0\0\u2dd8\0\u2e20\0\u2e68"+ + "\0\u0360\0\u0360\0\u2eb0\0\u0120\0\u2ef8\0\u2f40\0\u0360\0\u0360"+ + "\0\u0360\0\u0360\0\u2f88\0\u2fd0\0\u0360\0\u3018\0\u3060\0\u30a8"+ + "\0\u2958\0\u2a30\0\u0360\0\u2b98\0\u2c70"; private static int [] zzUnpackRowMap() { - int [] result = new int[244]; + int [] result = new int[253]; int offset = 0; offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result); return result; @@ -314,207 +316,217 @@ class _PythonLexer implements FlexLexer { return j; } - /** + /** * The transition table of the DFA */ private static final int [] ZZ_TRANS = zzUnpackTrans(); private static final String ZZ_TRANS_PACKED_0 = - "\12\5\1\6\24\5\1\7\1\5\1\10\1\11\1\12"+ - "\1\13\44\5\10\14\1\15\1\14\1\6\5\14\1\15"+ - "\7\14\4\15\2\16\1\17\1\7\1\20\1\10\1\11"+ - "\1\12\1\13\44\14\1\21\1\22\2\23\1\24\1\23"+ - "\1\25\1\26\1\27\1\30\1\6\1\21\4\25\1\31"+ - "\1\25\1\21\1\32\1\33\1\25\1\34\1\25\1\35"+ - "\3\31\1\36\1\37\1\40\1\7\1\41\1\10\1\11"+ - "\1\12\1\13\1\42\1\43\1\44\1\25\1\45\1\25"+ - "\1\46\1\47\1\50\1\25\1\51\1\52\1\25\1\53"+ - "\1\54\1\55\1\56\1\57\1\60\1\61\1\62\1\63"+ - "\1\64\1\65\1\66\1\67\1\70\1\71\1\72\1\73"+ - "\1\74\1\75\1\76\1\77\1\100\1\101\1\21\1\22"+ - "\2\23\1\24\1\23\1\25\1\26\1\27\1\30\1\6"+ - "\1\21\4\25\1\31\1\25\1\21\1\32\1\33\1\25"+ - "\1\34\1\25\1\35\3\31\1\36\1\37\1\40\1\7"+ - "\1\41\1\10\1\11\1\12\1\13\1\42\1\43\1\44"+ - "\1\25\1\45\1\25\1\46\1\47\1\50\1\25\1\51"+ - "\1\52\1\25\1\53\1\54\1\55\1\56\1\57\1\60"+ - "\1\61\1\62\1\63\1\64\1\65\1\66\1\67\1\70"+ - "\1\71\1\72\1\73\1\74\1\75\1\76\1\102\1\100"+ - "\1\101\111\0\13\6\1\0\25\6\1\0\47\6\34\0"+ - "\2\103\1\17\1\0\1\20\106\0\1\17\1\0\1\20"+ - "\50\0\13\104\1\0\22\104\1\105\1\106\1\104\1\107"+ - "\47\104\13\110\1\0\23\110\1\111\1\112\1\107\47\110"+ - "\1\0\1\113\1\114\1\113\1\0\1\113\1\115\1\116"+ - "\1\117\1\120\4\0\1\115\1\116\1\117\1\120\1\0"+ - "\1\121\2\122\1\0\1\123\62\0\3\23\1\0\1\23"+ - "\3\0\1\120\7\0\1\120\1\0\1\121\2\122\1\0"+ - "\1\123\62\0\6\25\1\124\2\25\2\0\7\25\1\0"+ - "\2\25\1\0\6\25\1\125\7\0\6\25\1\126\6\25"+ - "\30\0\11\25\2\0\7\25\1\0\2\25\1\0\7\25"+ - "\7\0\15\25\30\0\11\25\2\0\7\25\1\0\2\25"+ - "\1\0\6\25\1\127\7\0\15\25\30\0\11\25\2\0"+ - "\7\25\1\0\2\25\1\0\5\25\1\36\1\130\1\40"+ - "\1\0\1\41\4\0\15\25\30\0\11\25\2\0\7\25"+ - "\1\0\2\25\1\0\7\25\7\0\1\131\14\25\30\0"+ - "\11\25\2\0\7\25\1\0\2\25\1\0\5\25\2\36"+ - "\1\40\1\0\1\41\4\0\15\25\30\0\3\121\1\0"+ - "\1\121\104\0\5\25\1\132\2\25\1\133\2\0\7\25"+ - "\1\0\2\25\1\0\7\25\7\0\15\25\112\0\1\134"+ - "\26\0\6\25\1\135\1\25\1\136\2\0\7\25\1\0"+ - "\2\25\1\0\5\25\2\36\1\40\1\0\1\41\4\0"+ - "\15\25\30\0\11\25\2\0\7\25\1\0\2\25\1\0"+ - "\7\25\1\40\1\0\1\41\4\0\15\25\30\0\11\25"+ - "\2\0\7\25\1\0\1\137\1\25\1\0\7\25\1\40"+ - "\1\0\1\41\4\0\1\140\14\25\27\0\13\141\1\0"+ - "\22\141\1\142\1\143\1\141\1\0\47\141\13\144\1\0"+ - "\23\144\1\145\1\146\1\0\47\144\1\0\11\25\2\0"+ - "\7\25\1\0\2\25\1\0\7\25\7\0\1\25\1\147"+ - "\1\25\1\150\11\25\30\0\6\25\1\151\2\25\2\0"+ - "\7\25\1\0\2\25\1\0\7\25\7\0\15\25\30\0"+ - "\11\25\2\0\7\25\1\0\1\152\1\25\1\0\7\25"+ - "\7\0\15\25\30\0\11\25\2\0\7\25\1\0\2\25"+ - "\1\0\6\25\1\153\7\0\15\25\30\0\3\25\1\154"+ - "\5\25\2\0\7\25\1\0\2\25\1\0\7\25\7\0"+ - "\1\25\1\155\1\25\1\156\5\25\1\157\3\25\30\0"+ - "\11\25\2\0\7\25\1\0\2\25\1\0\7\25\7\0"+ - "\1\160\14\25\30\0\11\25\2\0\7\25\1\0\2\25"+ - "\1\0\7\25\7\0\6\25\1\161\6\25\30\0\10\25"+ - "\1\162\2\0\7\25\1\0\2\25\1\0\7\25\7\0"+ - "\15\25\30\0\11\25\2\0\7\25\1\0\2\25\1\0"+ - "\7\25\7\0\14\25\1\163\112\0\1\164\110\0\1\165"+ - "\110\0\1\166\1\167\107\0\1\170\1\0\1\171\106\0"+ - "\1\172\110\0\1\173\110\0\1\174\110\0\1\175\110\0"+ - "\1\176\6\0\1\177\101\0\1\200\6\0\1\201\1\202"+ - "\100\0\1\203\66\0\1\204\1\205\46\0\13\104\1\0"+ - "\22\104\1\206\1\106\1\104\1\107\47\104\36\0\1\207"+ - "\1\210\1\0\1\107\2\211\44\0\1\107\111\104\13\110"+ - "\1\0\23\110\1\111\1\206\1\107\160\110\37\0\1\210"+ - "\1\212\1\107\2\211\44\0\1\107\1\0\1\113\1\114"+ - "\1\113\1\0\1\113\3\0\1\120\7\0\1\120\1\0"+ - "\1\121\2\122\1\0\1\123\62\0\3\114\1\0\1\114"+ - "\15\0\1\121\2\122\1\0\1\123\62\0\5\213\2\0"+ + "\12\5\1\6\22\5\1\7\1\5\1\10\1\11\1\12"+ + "\1\13\45\5\10\14\1\15\1\14\1\6\5\14\1\15"+ + "\7\14\4\15\1\16\1\7\1\17\1\10\1\11\1\12"+ + "\1\13\4\14\1\15\40\14\1\20\1\21\2\22\1\23"+ + "\1\22\1\24\1\25\1\26\1\27\1\6\1\20\4\24"+ + "\1\30\1\24\1\20\1\31\1\32\1\24\1\33\1\24"+ + "\1\34\3\30\1\35\1\7\1\36\1\10\1\11\1\12"+ + "\1\13\1\37\1\40\1\41\1\24\1\42\1\43\1\24"+ + "\1\44\1\45\1\46\1\24\1\47\1\50\1\24\1\51"+ + "\1\52\1\53\1\54\1\55\1\56\1\57\1\60\1\61"+ + "\1\62\1\63\1\64\1\65\1\66\1\67\1\70\1\71"+ + "\1\72\1\73\1\74\1\75\1\76\1\77\1\20\1\21"+ + "\2\22\1\23\1\22\1\24\1\25\1\26\1\27\1\6"+ + "\1\20\4\24\1\30\1\24\1\20\1\31\1\32\1\24"+ + "\1\33\1\24\1\34\3\30\1\35\1\7\1\36\1\10"+ + "\1\11\1\12\1\13\1\37\1\40\1\41\1\24\1\42"+ + "\1\43\1\24\1\44\1\45\1\46\1\24\1\47\1\50"+ + "\1\24\1\51\1\52\1\53\1\54\1\55\1\56\1\57"+ + "\1\60\1\61\1\62\1\63\1\64\1\65\1\66\1\67"+ + "\1\70\1\71\1\72\1\73\1\74\1\100\1\76\1\77"+ + "\110\0\13\6\1\0\23\6\1\0\50\6\10\0\1\101"+ + "\7\0\1\101\7\0\4\101\1\16\1\0\1\17\10\0"+ + "\1\101\40\0\13\102\1\0\20\102\1\103\1\104\1\102"+ + "\1\105\50\102\13\106\1\0\21\106\1\107\1\110\1\105"+ + "\50\106\1\0\1\111\1\112\1\111\1\0\1\111\1\113"+ + "\1\114\1\115\1\116\4\0\1\113\1\114\1\115\1\116"+ + "\1\0\1\117\2\120\1\0\1\121\61\0\3\22\1\0"+ + "\1\22\3\0\1\116\7\0\1\116\1\0\1\117\2\120"+ + "\1\0\1\121\61\0\6\24\1\122\2\24\2\0\7\24"+ + "\1\0\2\24\1\0\5\24\7\0\4\24\1\123\2\24"+ + "\1\124\6\24\30\0\11\24\2\0\7\24\1\0\2\24"+ + "\1\0\5\24\7\0\16\24\30\0\11\24\2\0\7\24"+ + "\1\0\2\24\1\0\5\24\7\0\4\24\1\125\11\24"+ + "\30\0\7\24\1\126\1\24\2\0\4\24\1\126\2\24"+ + "\1\0\2\24\1\0\1\24\4\126\1\35\1\0\1\36"+ + "\4\0\4\24\1\127\11\24\30\0\11\24\2\0\7\24"+ + "\1\0\2\24\1\0\5\24\7\0\1\130\15\24\30\0"+ + "\7\24\1\126\1\24\2\0\4\24\1\126\2\24\1\0"+ + "\2\24\1\0\1\24\4\126\1\35\1\0\1\36\4\0"+ + "\4\24\1\126\11\24\30\0\3\117\1\0\1\117\103\0"+ + "\5\24\1\131\2\24\1\132\2\0\7\24\1\0\2\24"+ + "\1\0\5\24\7\0\16\24\111\0\1\133\26\0\6\24"+ + "\1\134\1\126\1\135\2\0\4\24\1\126\2\24\1\0"+ + "\2\24\1\0\1\24\4\126\1\35\1\0\1\36\4\0"+ + "\4\24\1\126\11\24\27\0\13\136\1\0\20\136\1\137"+ + "\1\140\1\136\1\0\50\136\13\141\1\0\21\141\1\142"+ + "\1\143\1\0\50\141\1\0\11\24\2\0\7\24\1\0"+ + "\2\24\1\0\5\24\7\0\1\24\1\144\1\24\1\145"+ + "\12\24\30\0\6\24\1\146\2\24\2\0\7\24\1\0"+ + "\2\24\1\0\5\24\7\0\16\24\30\0\11\24\2\0"+ + "\7\24\1\0\1\147\1\24\1\0\5\24\7\0\16\24"+ + "\30\0\7\24\1\126\1\24\2\0\4\24\1\126\2\24"+ + "\1\0\1\150\1\24\1\0\1\24\4\126\1\35\1\0"+ + "\1\36\4\0\1\151\3\24\1\126\11\24\30\0\11\24"+ + "\2\0\7\24\1\0\2\24\1\0\5\24\7\0\4\24"+ + "\1\152\11\24\30\0\3\24\1\153\5\24\2\0\7\24"+ + "\1\0\2\24\1\0\5\24\7\0\1\24\1\154\1\24"+ + "\1\155\6\24\1\156\3\24\30\0\11\24\2\0\7\24"+ + "\1\0\2\24\1\0\5\24\7\0\1\157\15\24\30\0"+ + "\11\24\2\0\7\24\1\0\2\24\1\0\5\24\7\0"+ + "\7\24\1\160\6\24\30\0\10\24\1\161\2\0\7\24"+ + "\1\0\2\24\1\0\5\24\7\0\16\24\30\0\11\24"+ + "\2\0\7\24\1\0\2\24\1\0\5\24\7\0\15\24"+ + "\1\162\111\0\1\163\107\0\1\164\107\0\1\165\1\166"+ + "\106\0\1\167\1\0\1\170\105\0\1\171\107\0\1\172"+ + "\107\0\1\173\107\0\1\174\107\0\1\175\6\0\1\176"+ + "\100\0\1\177\6\0\1\200\1\201\77\0\1\202\64\0"+ + "\1\203\1\204\57\0\1\205\7\0\1\205\7\0\4\205"+ + "\1\16\1\0\1\17\10\0\1\205\40\0\13\102\1\0"+ + "\20\102\1\206\1\104\1\102\1\105\50\102\34\0\1\207"+ + "\1\210\1\0\1\105\2\211\45\0\1\105\110\102\13\106"+ + "\1\0\21\106\1\107\1\206\1\105\160\106\35\0\1\210"+ + "\1\212\1\105\2\211\45\0\1\105\1\0\1\111\1\112"+ + "\1\111\1\0\1\111\3\0\1\116\7\0\1\116\1\0"+ + "\1\117\2\120\1\0\1\121\61\0\3\112\1\0\1\112"+ + "\15\0\1\117\2\120\1\0\1\121\61\0\5\213\2\0"+ "\1\213\4\0\1\213\2\0\1\213\3\0\2\213\2\0"+ - "\2\213\13\0\1\213\1\0\1\213\42\0\1\214\1\0"+ - "\1\214\1\0\1\214\104\0\1\215\3\0\1\215\104\0"+ - "\3\121\1\0\1\121\16\0\2\122\1\0\1\123\62\0"+ - "\3\216\1\0\1\216\20\0\1\217\33\0\1\217\27\0"+ - "\11\25\2\0\7\25\1\0\2\25\1\0\6\25\1\220"+ - "\7\0\15\25\30\0\6\25\1\221\2\25\2\0\7\25"+ - "\1\0\2\25\1\0\7\25\7\0\15\25\30\0\11\25"+ - "\2\0\7\25\1\0\2\25\1\0\7\25\7\0\1\25"+ - "\1\222\13\25\30\0\11\25\2\0\7\25\1\0\1\223"+ - "\1\25\1\0\7\25\1\40\1\0\1\41\4\0\15\25"+ - "\30\0\11\25\2\0\7\25\1\0\2\25\1\0\7\25"+ - "\7\0\11\25\1\224\3\25\30\0\11\25\2\0\7\25"+ - "\1\0\2\25\1\0\1\25\1\225\5\25\7\0\15\25"+ - "\30\0\11\25\2\0\7\25\1\0\2\25\1\0\7\25"+ - "\7\0\3\25\1\226\2\25\1\227\6\25\30\0\11\25"+ - "\2\0\7\25\1\0\2\25\1\0\7\25\7\0\1\25"+ - "\1\230\13\25\30\0\11\25\2\0\7\25\1\0\2\25"+ - "\1\0\7\25\7\0\1\231\14\25\30\0\11\25\2\0"+ - "\7\25\1\0\2\25\1\0\7\25\7\0\4\25\1\232"+ - "\10\25\30\0\11\25\2\0\7\25\1\0\2\25\1\0"+ - "\7\25\7\0\6\25\1\233\6\25\27\0\13\141\1\0"+ - "\22\141\1\234\1\143\1\141\1\0\47\141\36\0\1\235"+ - "\52\0\111\141\13\144\1\0\23\144\1\145\1\234\1\0"+ - "\160\144\40\0\1\236\51\0\11\25\2\0\7\25\1\0"+ - "\2\25\1\0\7\25\7\0\2\25\1\237\12\25\30\0"+ - "\11\25\2\0\7\25\1\0\2\25\1\0\7\25\7\0"+ - "\3\25\1\240\11\25\30\0\11\25\2\0\7\25\1\0"+ - "\2\25\1\0\7\25\7\0\4\25\1\241\10\25\30\0"+ - "\3\25\1\242\4\25\1\243\2\0\7\25\1\0\2\25"+ - "\1\0\7\25\7\0\15\25\30\0\11\25\2\0\7\25"+ - "\1\0\2\25\1\0\7\25\7\0\10\25\1\244\4\25"+ - "\30\0\11\25\2\0\7\25\1\0\2\25\1\0\7\25"+ - "\7\0\7\25\1\245\5\25\30\0\11\25\2\0\7\25"+ - "\1\0\2\25\1\0\7\25\7\0\3\25\1\246\11\25"+ - "\30\0\11\25\2\0\7\25\1\0\1\247\1\25\1\0"+ - "\7\25\7\0\15\25\30\0\6\25\1\250\2\25\2\0"+ - "\7\25\1\0\2\25\1\0\7\25\7\0\15\25\30\0"+ - "\11\25\2\0\7\25\1\0\2\25\1\0\7\25\7\0"+ - "\6\25\1\251\6\25\112\0\1\252\110\0\1\253\110\0"+ - "\1\254\110\0\1\255\64\0\1\210\1\0\1\107\2\211"+ - "\44\0\1\107\36\207\1\256\1\257\51\207\37\212\1\260"+ - "\1\261\50\212\1\0\5\213\2\0\1\213\1\120\3\0"+ - "\1\213\2\0\1\213\1\120\2\0\2\213\2\0\2\213"+ - "\13\0\1\213\1\0\1\213\42\0\1\214\1\0\1\214"+ - "\1\0\1\214\3\0\1\120\7\0\1\120\70\0\1\215"+ - "\3\0\1\215\3\0\1\120\7\0\1\120\70\0\3\216"+ - "\1\0\1\216\21\0\1\123\62\0\3\216\1\0\1\216"+ - "\104\0\11\25\2\0\7\25\1\0\2\25\1\0\7\25"+ - "\7\0\11\25\1\262\3\25\30\0\11\25\2\0\7\25"+ - "\1\0\2\25\1\0\7\25\7\0\1\263\14\25\30\0"+ - "\11\25\2\0\7\25\1\0\2\25\1\0\7\25\7\0"+ - "\1\264\14\25\30\0\7\25\1\265\1\25\2\0\7\25"+ - "\1\0\2\25\1\0\7\25\7\0\15\25\30\0\11\25"+ - "\2\0\7\25\1\0\1\266\1\25\1\0\7\25\7\0"+ - "\15\25\30\0\11\25\2\0\7\25\1\0\1\267\1\25"+ - "\1\0\7\25\7\0\15\25\30\0\3\25\1\270\5\25"+ - "\2\0\7\25\1\0\2\25\1\0\7\25\7\0\15\25"+ - "\30\0\11\25\2\0\7\25\1\0\2\25\1\0\7\25"+ - "\7\0\4\25\1\271\10\25\30\0\11\25\2\0\7\25"+ - "\1\0\2\25\1\0\7\25\7\0\3\25\1\272\11\25"+ - "\30\0\11\25\2\0\7\25\1\0\2\25\1\0\4\25"+ - "\1\273\2\25\7\0\15\25\30\0\11\25\2\0\7\25"+ - "\1\0\2\25\1\0\7\25\7\0\3\25\1\274\11\25"+ - "\27\0\36\235\1\275\1\276\51\235\37\236\1\277\1\300"+ - "\50\236\1\0\11\25\2\0\7\25\1\0\1\301\1\25"+ - "\1\0\7\25\7\0\15\25\30\0\6\25\1\302\2\25"+ - "\2\0\7\25\1\0\2\25\1\0\7\25\7\0\15\25"+ - "\30\0\11\25\2\0\7\25\1\0\2\25\1\0\7\25"+ - "\7\0\3\25\1\303\11\25\30\0\10\25\1\304\2\0"+ - "\7\25\1\0\2\25\1\0\7\25\7\0\15\25\30\0"+ - "\7\25\1\305\1\25\2\0\7\25\1\0\2\25\1\0"+ - "\7\25\7\0\15\25\30\0\10\25\1\306\2\0\7\25"+ - "\1\0\2\25\1\0\7\25\7\0\15\25\27\0\36\207"+ - "\1\307\1\310\162\207\150\212\1\311\1\312\50\212\1\0"+ - "\10\25\1\313\2\0\7\25\1\0\2\25\1\0\7\25"+ - "\7\0\15\25\30\0\11\25\2\0\7\25\1\0\2\25"+ - "\1\0\7\25\7\0\5\25\1\314\7\25\30\0\11\25"+ - "\2\0\7\25\1\0\2\25\1\0\7\25\7\0\2\25"+ - "\1\315\12\25\30\0\11\25\2\0\7\25\1\0\2\25"+ - "\1\0\7\25\7\0\7\25\1\316\5\25\30\0\11\25"+ - "\2\0\7\25\1\0\2\25\1\0\7\25\7\0\6\25"+ - "\1\317\6\25\30\0\11\25\2\0\7\25\1\0\2\25"+ - "\1\0\7\25\7\0\3\25\1\320\11\25\30\0\11\25"+ - "\2\0\7\25\1\0\2\25\1\0\6\25\1\321\7\0"+ - "\15\25\30\0\11\25\2\0\7\25\1\0\1\322\1\25"+ - "\1\0\7\25\7\0\15\25\27\0\36\235\1\323\1\324"+ - "\162\235\150\236\1\325\1\326\50\236\1\0\11\25\2\0"+ - "\7\25\1\0\2\25\1\0\6\25\1\327\7\0\15\25"+ - "\30\0\11\25\2\0\7\25\1\0\2\25\1\0\6\25"+ - "\1\330\7\0\15\25\30\0\11\25\2\0\7\25\1\0"+ - "\2\25\1\0\7\25\7\0\2\25\1\331\12\25\30\0"+ - "\11\25\2\0\7\25\1\0\2\25\1\0\7\25\7\0"+ - "\1\332\14\25\30\0\11\25\2\0\7\25\1\0\1\333"+ - "\1\25\1\0\7\25\7\0\15\25\27\0\36\207\1\334"+ - "\1\310\107\207\1\335\1\310\51\207\37\212\1\311\1\336"+ - "\107\212\1\311\1\334\50\212\1\0\10\25\1\337\2\0"+ - "\7\25\1\0\2\25\1\0\7\25\7\0\15\25\30\0"+ - "\11\25\2\0\7\25\1\0\2\25\1\0\7\25\7\0"+ - "\1\340\14\25\30\0\11\25\2\0\7\25\1\0\2\25"+ - "\1\0\7\25\7\0\4\25\1\341\10\25\30\0\11\25"+ - "\2\0\7\25\1\0\2\25\1\0\7\25\7\0\1\25"+ - "\1\342\13\25\30\0\11\25\2\0\7\25\1\0\2\25"+ - "\1\0\7\25\7\0\1\25\1\343\13\25\27\0\36\235"+ - "\1\344\1\324\107\235\1\345\1\324\51\235\37\236\1\325"+ - "\1\346\107\236\1\325\1\344\50\236\1\0\11\25\2\0"+ - "\7\25\1\0\2\25\1\0\7\25\7\0\4\25\1\347"+ - "\10\25\30\0\11\25\2\0\7\25\1\0\2\25\1\0"+ - "\7\25\7\0\4\25\1\350\10\25\30\0\10\25\1\351"+ - "\2\0\7\25\1\0\2\25\1\0\7\25\7\0\15\25"+ - "\27\0\36\207\1\352\1\310\51\207\37\212\1\311\1\353"+ - "\50\212\1\0\11\25\2\0\7\25\1\0\2\25\1\0"+ - "\7\25\7\0\10\25\1\354\4\25\30\0\11\25\2\0"+ - "\7\25\1\0\2\25\1\0\4\25\1\355\2\25\7\0"+ - "\15\25\27\0\36\235\1\356\1\324\51\235\37\236\1\325"+ - "\1\357\50\236\36\207\1\360\1\310\51\207\37\212\1\311"+ - "\1\361\50\212\1\0\11\25\2\0\7\25\1\0\1\362"+ - "\1\25\1\0\7\25\7\0\15\25\27\0\36\235\1\363"+ - "\1\324\51\235\37\236\1\325\1\364\50\236"; + "\2\213\11\0\1\213\1\0\1\213\43\0\1\214\1\0"+ + "\1\214\1\0\1\214\103\0\1\215\3\0\1\215\103\0"+ + "\3\117\1\0\1\117\16\0\2\120\1\0\1\121\61\0"+ + "\3\216\1\0\1\216\20\0\1\217\32\0\1\217\27\0"+ + "\11\24\2\0\7\24\1\0\2\24\1\0\5\24\7\0"+ + "\4\24\1\220\11\24\30\0\6\24\1\221\2\24\2\0"+ + "\7\24\1\0\2\24\1\0\5\24\7\0\16\24\30\0"+ + "\11\24\2\0\7\24\1\0\2\24\1\0\5\24\7\0"+ + "\1\24\1\222\14\24\30\0\7\24\1\223\1\24\2\0"+ + "\4\24\1\223\2\24\1\0\2\24\1\0\1\24\4\223"+ + "\1\35\1\0\1\36\4\0\4\24\1\223\11\24\30\0"+ + "\7\24\1\223\1\24\2\0\4\24\1\223\2\24\1\0"+ + "\1\224\1\24\1\0\1\24\4\223\1\35\1\0\1\36"+ + "\4\0\4\24\1\223\11\24\30\0\11\24\2\0\7\24"+ + "\1\0\2\24\1\0\5\24\7\0\12\24\1\225\3\24"+ + "\30\0\11\24\2\0\7\24\1\0\2\24\1\0\1\24"+ + "\1\226\3\24\7\0\16\24\30\0\11\24\2\0\7\24"+ + "\1\0\2\24\1\0\5\24\7\0\3\24\1\227\3\24"+ + "\1\230\6\24\30\0\11\24\2\0\7\24\1\0\2\24"+ + "\1\0\5\24\7\0\1\24\1\231\14\24\30\0\11\24"+ + "\2\0\7\24\1\0\2\24\1\0\5\24\7\0\1\232"+ + "\15\24\27\0\13\136\1\0\20\136\1\233\1\140\1\136"+ + "\1\0\50\136\34\0\1\234\53\0\110\136\13\141\1\0"+ + "\21\141\1\142\1\233\1\0\160\141\36\0\1\235\52\0"+ + "\11\24\2\0\7\24\1\0\2\24\1\0\5\24\7\0"+ + "\2\24\1\236\13\24\30\0\11\24\2\0\7\24\1\0"+ + "\2\24\1\0\5\24\7\0\3\24\1\237\12\24\30\0"+ + "\11\24\2\0\7\24\1\0\2\24\1\0\5\24\7\0"+ + "\5\24\1\240\10\24\30\0\3\24\1\241\4\24\1\242"+ + "\2\0\7\24\1\0\2\24\1\0\5\24\7\0\16\24"+ + "\30\0\11\24\2\0\7\24\1\0\2\24\1\0\5\24"+ + "\7\0\5\24\1\243\10\24\30\0\11\24\2\0\7\24"+ + "\1\0\2\24\1\0\5\24\7\0\7\24\1\244\6\24"+ + "\30\0\11\24\2\0\7\24\1\0\2\24\1\0\5\24"+ + "\7\0\11\24\1\245\4\24\30\0\11\24\2\0\7\24"+ + "\1\0\2\24\1\0\5\24\7\0\10\24\1\246\5\24"+ + "\30\0\11\24\2\0\7\24\1\0\2\24\1\0\5\24"+ + "\7\0\3\24\1\247\12\24\30\0\11\24\2\0\7\24"+ + "\1\0\1\250\1\24\1\0\5\24\7\0\16\24\30\0"+ + "\6\24\1\251\2\24\2\0\7\24\1\0\2\24\1\0"+ + "\5\24\7\0\16\24\30\0\11\24\2\0\7\24\1\0"+ + "\2\24\1\0\5\24\7\0\7\24\1\252\6\24\111\0"+ + "\1\253\107\0\1\254\107\0\1\255\107\0\1\256\61\0"+ + "\1\257\1\0\1\260\106\0\1\210\1\0\1\105\2\211"+ + "\45\0\1\105\34\207\1\261\1\262\52\207\35\212\1\263"+ + "\1\264\51\212\1\0\5\213\2\0\1\213\1\116\3\0"+ + "\1\213\2\0\1\213\1\116\2\0\2\213\2\0\2\213"+ + "\11\0\1\213\1\0\1\213\43\0\1\214\1\0\1\214"+ + "\1\0\1\214\3\0\1\116\7\0\1\116\67\0\1\215"+ + "\3\0\1\215\3\0\1\116\7\0\1\116\67\0\3\216"+ + "\1\0\1\216\21\0\1\121\61\0\3\216\1\0\1\216"+ + "\103\0\11\24\2\0\7\24\1\0\2\24\1\0\5\24"+ + "\7\0\12\24\1\265\3\24\30\0\11\24\2\0\7\24"+ + "\1\0\2\24\1\0\5\24\7\0\1\266\15\24\30\0"+ + "\11\24\2\0\7\24\1\0\2\24\1\0\5\24\1\267"+ + "\1\0\1\270\4\0\16\24\30\0\11\24\2\0\7\24"+ + "\1\0\2\24\1\0\5\24\7\0\1\271\15\24\30\0"+ + "\7\24\1\272\1\24\2\0\7\24\1\0\2\24\1\0"+ + "\5\24\7\0\16\24\30\0\11\24\2\0\7\24\1\0"+ + "\1\273\1\24\1\0\5\24\7\0\16\24\30\0\11\24"+ + "\2\0\7\24\1\0\1\274\1\24\1\0\5\24\7\0"+ + "\16\24\30\0\3\24\1\275\5\24\2\0\7\24\1\0"+ + "\2\24\1\0\5\24\7\0\16\24\30\0\11\24\2\0"+ + "\7\24\1\0\2\24\1\0\5\24\7\0\5\24\1\276"+ + "\10\24\30\0\11\24\2\0\7\24\1\0\2\24\1\0"+ + "\5\24\7\0\3\24\1\277\12\24\27\0\34\234\1\300"+ + "\1\301\52\234\35\235\1\302\1\303\51\235\1\0\11\24"+ + "\2\0\7\24\1\0\1\304\1\24\1\0\5\24\7\0"+ + "\16\24\30\0\11\24\2\0\7\24\1\0\2\24\1\0"+ + "\4\24\1\305\7\0\16\24\30\0\11\24\2\0\7\24"+ + "\1\0\2\24\1\0\5\24\7\0\3\24\1\306\12\24"+ + "\30\0\6\24\1\307\2\24\2\0\7\24\1\0\2\24"+ + "\1\0\5\24\7\0\16\24\30\0\11\24\2\0\7\24"+ + "\1\0\2\24\1\0\5\24\7\0\3\24\1\310\12\24"+ + "\30\0\10\24\1\311\2\0\7\24\1\0\2\24\1\0"+ + "\5\24\7\0\16\24\30\0\7\24\1\312\1\24\2\0"+ + "\7\24\1\0\2\24\1\0\5\24\7\0\16\24\30\0"+ + "\10\24\1\313\2\0\7\24\1\0\2\24\1\0\5\24"+ + "\7\0\16\24\63\0\1\314\111\0\1\315\51\0\34\207"+ + "\1\316\1\317\162\207\145\212\1\320\1\321\51\212\1\0"+ + "\10\24\1\322\2\0\7\24\1\0\2\24\1\0\5\24"+ + "\7\0\16\24\63\0\1\323\111\0\1\324\52\0\11\24"+ + "\2\0\7\24\1\0\2\24\1\0\5\24\7\0\6\24"+ + "\1\325\7\24\30\0\11\24\2\0\7\24\1\0\2\24"+ + "\1\0\5\24\7\0\2\24\1\326\13\24\30\0\11\24"+ + "\2\0\7\24\1\0\2\24\1\0\5\24\7\0\10\24"+ + "\1\327\5\24\30\0\11\24\2\0\7\24\1\0\2\24"+ + "\1\0\5\24\7\0\7\24\1\330\6\24\30\0\11\24"+ + "\2\0\7\24\1\0\2\24\1\0\5\24\7\0\3\24"+ + "\1\331\12\24\27\0\34\234\1\332\1\333\162\234\145\235"+ + "\1\334\1\335\51\235\1\0\11\24\2\0\7\24\1\0"+ + "\2\24\1\0\5\24\7\0\4\24\1\336\11\24\30\0"+ + "\11\24\2\0\7\24\1\0\2\24\1\0\5\24\7\0"+ + "\4\24\1\337\11\24\30\0\11\24\2\0\7\24\1\0"+ + "\1\340\1\24\1\0\5\24\7\0\16\24\30\0\11\24"+ + "\2\0\7\24\1\0\2\24\1\0\5\24\7\0\4\24"+ + "\1\341\11\24\30\0\11\24\2\0\7\24\1\0\2\24"+ + "\1\0\5\24\7\0\2\24\1\342\13\24\30\0\11\24"+ + "\2\0\7\24\1\0\2\24\1\0\5\24\7\0\1\343"+ + "\15\24\30\0\11\24\2\0\7\24\1\0\1\344\1\24"+ + "\1\0\5\24\7\0\16\24\63\0\1\207\111\0\1\212"+ + "\51\0\34\207\1\345\1\317\106\207\1\346\1\317\52\207"+ + "\35\212\1\320\1\347\106\212\1\320\1\345\51\212\1\0"+ + "\10\24\1\350\2\0\7\24\1\0\2\24\1\0\5\24"+ + "\7\0\16\24\30\0\11\24\2\0\7\24\1\0\2\24"+ + "\1\0\5\24\7\0\1\351\15\24\30\0\11\24\2\0"+ + "\7\24\1\0\2\24\1\0\5\24\7\0\5\24\1\352"+ + "\10\24\30\0\11\24\2\0\7\24\1\0\2\24\1\0"+ + "\5\24\7\0\1\24\1\353\14\24\27\0\34\234\1\354"+ + "\1\333\106\234\1\355\1\333\52\234\35\235\1\334\1\356"+ + "\106\235\1\334\1\354\51\235\1\0\11\24\2\0\7\24"+ + "\1\0\2\24\1\0\5\24\7\0\5\24\1\357\10\24"+ + "\30\0\11\24\2\0\7\24\1\0\2\24\1\0\5\24"+ + "\7\0\1\24\1\360\14\24\30\0\11\24\2\0\7\24"+ + "\1\0\2\24\1\0\5\24\7\0\5\24\1\361\10\24"+ + "\30\0\10\24\1\362\2\0\7\24\1\0\2\24\1\0"+ + "\5\24\7\0\16\24\27\0\34\207\1\363\1\317\52\207"+ + "\35\212\1\320\1\364\51\212\1\0\11\24\2\0\7\24"+ + "\1\0\2\24\1\0\5\24\7\0\11\24\1\365\4\24"+ + "\30\0\11\24\2\0\7\24\1\0\2\24\1\0\4\24"+ + "\1\366\7\0\16\24\27\0\34\234\1\367\1\333\52\234"+ + "\35\235\1\334\1\370\51\235\34\207\1\371\1\317\52\207"+ + "\35\212\1\320\1\372\51\212\1\0\11\24\2\0\7\24"+ + "\1\0\1\373\1\24\1\0\5\24\7\0\16\24\27\0"+ + "\34\234\1\374\1\333\52\234\35\235\1\334\1\375\51\235"; private static int [] zzUnpackTrans() { - int [] result = new int[12118]; + int [] result = new int[12528]; int offset = 0; offset = zzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result); return result; @@ -555,17 +567,18 @@ class _PythonLexer implements FlexLexer { private static final int [] ZZ_ATTRIBUTE = zzUnpackAttribute(); private static final String ZZ_ATTRIBUTE_PACKED_0 = - "\4\0\1\11\1\1\6\11\4\1\1\11\44\1\11\11"+ + "\4\0\1\11\1\1\6\11\3\1\1\11\43\1\11\11"+ "\1\1\3\11\1\0\3\1\1\11\4\1\4\0\1\11"+ - "\1\1\1\0\1\11\10\1\1\11\27\1\3\11\1\1"+ - "\1\11\1\1\5\11\1\1\2\11\1\1\2\11\1\0"+ - "\2\1\1\11\1\0\5\1\1\0\14\1\1\11\15\1"+ - "\4\11\1\0\2\1\1\0\13\1\4\0\6\1\1\0"+ - "\2\1\1\0\10\1\1\0\2\1\1\0\15\1\1\11"+ - "\5\1\2\0\2\1\2\0\5\1"; + "\1\1\1\0\1\11\11\1\1\11\27\1\3\11\1\1"+ + "\1\11\1\1\5\11\1\1\2\11\1\1\2\11\2\0"+ + "\2\1\1\11\1\0\5\1\1\0\13\1\1\11\17\1"+ + "\4\11\3\0\2\1\1\0\2\1\2\0\7\1\4\0"+ + "\10\1\3\0\2\1\1\0\1\1\2\0\5\1\1\0"+ + "\2\1\1\0\16\1\1\11\6\1\2\0\2\1\2\0"+ + "\5\1"; private static int [] zzUnpackAttribute() { - int [] result = new int[244]; + int [] result = new int[253]; int offset = 0; offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result); return result; @@ -590,7 +603,7 @@ class _PythonLexer implements FlexLexer { private int zzLexicalState = YYINITIAL; /** this buffer contains the current text to be matched and is - the source of the yytext() string */ + the source of the yytext() string */ private CharSequence zzBuffer = ""; /** this buffer may contains the current text array to be matched when it is cheap to acquire it */ @@ -609,7 +622,7 @@ class _PythonLexer implements FlexLexer { private int zzStartRead; /** endRead marks the last character in the buffer, that has been read - from input */ + from input */ private int zzEndRead; /** @@ -624,11 +637,11 @@ class _PythonLexer implements FlexLexer { private boolean zzEOFDone; /* user code: */ - private int getSpaceLength(CharSequence string) { - int i = Math.max(StringUtil.lastIndexOf(string, '"', 0, string.length()), StringUtil.lastIndexOf(string, '\'', 0, string.length())); - return yylength()-i-1; +private int getSpaceLength(CharSequence string) { +int i = Math.max(StringUtil.lastIndexOf(string, '"', 0, string.length()), StringUtil.lastIndexOf(string, '\'', 0, string.length())); +return yylength()-i-1; - } +} _PythonLexer(java.io.Reader in) { @@ -645,7 +658,7 @@ class _PythonLexer implements FlexLexer { this(new java.io.InputStreamReader(in)); } - /** + /** * Unpacks the compressed character translation table. * * @param packed the packed character translation table @@ -794,7 +807,7 @@ class _PythonLexer implements FlexLexer { private void zzDoEOF() { if (!zzEOFDone) { zzEOFDone = true; - + } } @@ -877,375 +890,375 @@ class _PythonLexer implements FlexLexer { zzMarkedPos = zzMarkedPosL; switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) { - case 83: - { return PyTokenTypes.YIELD_KEYWORD; - } + case 83: + { return PyTokenTypes.YIELD_KEYWORD; + } case 93: break; - case 11: - { return PyTokenTypes.INTEGER_LITERAL; - } + case 11: + { return PyTokenTypes.INTEGER_LITERAL; + } case 94: break; - case 19: - { return PyTokenTypes.DIV; - } + case 19: + { return PyTokenTypes.DIV; + } case 95: break; - case 20: - { return PyTokenTypes.PERC; - } + case 20: + { return PyTokenTypes.PERC; + } case 96: break; - case 3: - { return PyTokenTypes.BACKSLASH; - } + case 3: + { return PyTokenTypes.BACKSLASH; + } case 97: break; - case 28: - { return PyTokenTypes.RPAR; - } + case 28: + { return PyTokenTypes.RPAR; + } case 98: break; - case 10: - { return PyTokenTypes.BAD_CHARACTER; - } + case 10: + { return PyTokenTypes.BAD_CHARACTER; + } case 99: break; - case 31: - { return PyTokenTypes.LBRACE; - } + case 31: + { return PyTokenTypes.LBRACE; + } case 100: break; - case 69: - { yybegin(IN_DOCSTRING_OWNER); return PyTokenTypes.DEF_KEYWORD; - } + case 69: + { yybegin(IN_DOCSTRING_OWNER); return PyTokenTypes.DEF_KEYWORD; + } case 101: break; - case 81: - { yybegin(IN_DOCSTRING_OWNER); return PyTokenTypes.CLASS_KEYWORD; - } + case 81: + { yybegin(IN_DOCSTRING_OWNER); return PyTokenTypes.CLASS_KEYWORD; + } case 102: break; - case 64: - { yypushback(getSpaceLength(yytext())); return PyTokenTypes.DOCSTRING; - } + case 64: + { yypushback(getSpaceLength(yytext())); return PyTokenTypes.DOCSTRING; + } case 103: break; - case 14: - { return PyTokenTypes.MINUS; - } + case 14: + { return PyTokenTypes.MINUS; + } case 104: break; - case 80: - { return PyTokenTypes.BREAK_KEYWORD; - } + case 80: + { return PyTokenTypes.BREAK_KEYWORD; + } case 105: break; - case 30: - { return PyTokenTypes.RBRACKET; - } + case 30: + { return PyTokenTypes.RBRACKET; + } case 106: break; - case 1: - { yypushback(1); yybegin(PENDING_DOCSTRING); - } + case 1: + { yypushback(1); yybegin(PENDING_DOCSTRING); + } case 107: break; - case 8: - { yypushback(1); yybegin(USUAL); - } + case 8: + { yypushback(1); yybegin(USUAL); + } case 108: break; - case 13: - { return PyTokenTypes.DOT; - } + case 13: + { return PyTokenTypes.DOT; + } case 109: break; - case 12: - { return PyTokenTypes.IDENTIFIER; - } + case 12: + { return PyTokenTypes.IDENTIFIER; + } case 110: break; - case 27: - { return PyTokenTypes.LPAR; - } + case 27: + { return PyTokenTypes.LPAR; + } case 111: break; - case 25: - { return PyTokenTypes.LT; - } + case 25: + { return PyTokenTypes.LT; + } case 112: break; - case 44: - { return PyTokenTypes.IN_KEYWORD; - } + case 44: + { return PyTokenTypes.IN_KEYWORD; + } case 113: break; - case 82: - { return PyTokenTypes.RAISE_KEYWORD; - } + case 82: + { return PyTokenTypes.RAISE_KEYWORD; + } case 114: break; - case 74: - { return PyTokenTypes.GTGTEQ; - } + case 74: + { return PyTokenTypes.GTGTEQ; + } case 115: break; - case 5: - { return PyTokenTypes.SPACE; - } + case 5: + { return PyTokenTypes.SPACE; + } case 116: break; - case 86: - { return PyTokenTypes.EXCEPT_KEYWORD; - } + case 86: + { return PyTokenTypes.EXCEPT_KEYWORD; + } case 117: break; - case 40: - { return PyTokenTypes.IMAGINARY_LITERAL; - } + case 40: + { return PyTokenTypes.IMAGINARY_LITERAL; + } case 118: break; - case 15: - { return PyTokenTypes.SINGLE_QUOTED_STRING; - } + case 15: + { return PyTokenTypes.SINGLE_QUOTED_STRING; + } case 119: break; - case 7: - { return PyTokenTypes.FORMFEED; - } + case 7: + { return PyTokenTypes.FORMFEED; + } case 120: break; - case 37: - { return PyTokenTypes.SEMICOLON; - } + case 37: + { return PyTokenTypes.SEMICOLON; + } case 121: break; - case 24: - { return PyTokenTypes.GT; - } + case 24: + { return PyTokenTypes.GT; + } case 122: break; - case 29: - { return PyTokenTypes.LBRACKET; - } + case 29: + { return PyTokenTypes.LBRACKET; + } case 123: break; - case 35: - { return PyTokenTypes.COLON; - } + case 35: + { return PyTokenTypes.COLON; + } case 124: break; - case 75: - { return PyTokenTypes.LTLTEQ; - } + case 75: + { return PyTokenTypes.LTLTEQ; + } case 125: break; - case 9: - { if (zzInput == YYEOF) return PyTokenTypes.DOCSTRING; - else yybegin(USUAL); return PyTokenTypes.SINGLE_QUOTED_STRING; - } + case 9: + { if (zzInput == YYEOF) return PyTokenTypes.DOCSTRING; + else yybegin(USUAL); return PyTokenTypes.SINGLE_QUOTED_STRING; + } case 126: break; - case 77: - { return PyTokenTypes.ELSE_KEYWORD; - } + case 77: + { return PyTokenTypes.ELSE_KEYWORD; + } case 127: break; - case 22: - { return PyTokenTypes.OR; - } + case 22: + { return PyTokenTypes.OR; + } case 128: break; - case 48: - { return PyTokenTypes.MULTEQ; - } + case 48: + { return PyTokenTypes.MULTEQ; + } case 129: break; - case 91: - { return PyTokenTypes.FINALLY_KEYWORD; - } + case 91: + { return PyTokenTypes.FINALLY_KEYWORD; + } case 130: break; - case 84: - { return PyTokenTypes.WHILE_KEYWORD; - } + case 84: + { return PyTokenTypes.WHILE_KEYWORD; + } case 131: break; - case 18: - { return PyTokenTypes.MULT; - } + case 18: + { return PyTokenTypes.MULT; + } case 132: break; - case 88: - { return PyTokenTypes.ASSERT_KEYWORD; - } + case 87: + { return PyTokenTypes.ASSERT_KEYWORD; + } case 133: break; - case 51: - { return PyTokenTypes.FLOORDIV; - } + case 51: + { return PyTokenTypes.FLOORDIV; + } case 134: break; - case 90: - { return PyTokenTypes.GLOBAL_KEYWORD; - } + case 90: + { return PyTokenTypes.GLOBAL_KEYWORD; + } case 135: break; - case 85: - { return PyTokenTypes.LAMBDA_KEYWORD; - } + case 85: + { return PyTokenTypes.LAMBDA_KEYWORD; + } case 136: break; - case 55: - { return PyTokenTypes.XOREQ; - } + case 55: + { return PyTokenTypes.XOREQ; + } case 137: break; - case 65: - { return PyTokenTypes.FOR_KEYWORD; - } + case 65: + { return PyTokenTypes.FOR_KEYWORD; + } case 138: break; - case 33: - { return PyTokenTypes.AT; - } + case 33: + { return PyTokenTypes.AT; + } case 139: break; - case 68: - { return PyTokenTypes.NOT_KEYWORD; - } + case 68: + { return PyTokenTypes.NOT_KEYWORD; + } case 140: break; - case 16: - { return PyTokenTypes.PLUS; - } + case 16: + { return PyTokenTypes.PLUS; + } case 141: break; - case 42: - { return PyTokenTypes.MINUSEQ; - } + case 42: + { return PyTokenTypes.MINUSEQ; + } case 142: break; - case 38: - { yypushback(getSpaceLength(yytext())); yybegin(USUAL); return PyTokenTypes.DOCSTRING; - } + case 38: + { yypushback(getSpaceLength(yytext())); yybegin(USUAL); return PyTokenTypes.DOCSTRING; + } case 143: break; - case 61: - { return PyTokenTypes.NE; - } + case 61: + { return PyTokenTypes.NE; + } case 144: break; - case 34: - { return PyTokenTypes.COMMA; - } + case 34: + { return PyTokenTypes.COMMA; + } case 145: break; - case 47: - { return PyTokenTypes.EQEQ; - } + case 47: + { return PyTokenTypes.EQEQ; + } case 146: break; - case 36: - { return PyTokenTypes.TICK; - } + case 36: + { return PyTokenTypes.TICK; + } case 147: break; - case 49: - { return PyTokenTypes.EXP; - } + case 49: + { return PyTokenTypes.EXP; + } case 148: break; - case 66: - { return PyTokenTypes.TRIPLE_QUOTED_STRING; - } + case 66: + { return PyTokenTypes.TRIPLE_QUOTED_STRING; + } case 149: break; - case 92: - { return PyTokenTypes.CONTINUE_KEYWORD; - } + case 92: + { return PyTokenTypes.CONTINUE_KEYWORD; + } case 150: break; - case 60: - { return PyTokenTypes.LTLT; - } + case 60: + { return PyTokenTypes.LTLT; + } case 151: break; - case 58: - { return PyTokenTypes.LE; - } + case 58: + { return PyTokenTypes.LE; + } case 152: break; - case 67: - { return PyTokenTypes.AND_KEYWORD; - } + case 67: + { return PyTokenTypes.AND_KEYWORD; + } case 153: break; - case 41: - { return PyTokenTypes.OR_KEYWORD; - } + case 41: + { return PyTokenTypes.OR_KEYWORD; + } case 154: break; - case 79: - { return PyTokenTypes.PASS_KEYWORD; - } + case 79: + { return PyTokenTypes.PASS_KEYWORD; + } case 155: break; - case 62: - { yypushback(yylength()-1); yybegin(PENDING_DOCSTRING); return PyTokenTypes.COLON; - } + case 62: + { yypushback(yylength()-1); yybegin(PENDING_DOCSTRING); return PyTokenTypes.COLON; + } case 156: break; - case 2: - { return PyTokenTypes.END_OF_LINE_COMMENT; - } + case 2: + { return PyTokenTypes.END_OF_LINE_COMMENT; + } case 157: break; - case 63: - { if (zzInput == YYEOF) return PyTokenTypes.DOCSTRING; - else yybegin(USUAL); return PyTokenTypes.TRIPLE_QUOTED_STRING; - } + case 63: + { if (zzInput == YYEOF) return PyTokenTypes.DOCSTRING; + else yybegin(USUAL); return PyTokenTypes.TRIPLE_QUOTED_STRING; + } case 158: break; - case 54: - { return PyTokenTypes.OREQ; - } + case 54: + { return PyTokenTypes.OREQ; + } case 159: break; - case 57: - { return PyTokenTypes.GTGT; - } + case 57: + { return PyTokenTypes.GTGT; + } case 160: break; - case 21: - { return PyTokenTypes.AND; - } + case 21: + { return PyTokenTypes.AND; + } case 161: break; - case 50: - { return PyTokenTypes.DIVEQ; - } + case 50: + { return PyTokenTypes.DIVEQ; + } case 162: break; - case 87: - { return PyTokenTypes.RETURN_KEYWORD; - } + case 88: + { return PyTokenTypes.RETURN_KEYWORD; + } case 163: break; - case 89: - { return PyTokenTypes.IMPORT_KEYWORD; - } + case 89: + { return PyTokenTypes.IMPORT_KEYWORD; + } case 164: break; - case 56: - { return PyTokenTypes.GE; - } + case 56: + { return PyTokenTypes.GE; + } case 165: break; - case 46: - { return PyTokenTypes.PLUSEQ; - } + case 46: + { return PyTokenTypes.PLUSEQ; + } case 166: break; - case 45: - { return PyTokenTypes.IS_KEYWORD; - } + case 45: + { return PyTokenTypes.IS_KEYWORD; + } case 167: break; - case 73: - { return PyTokenTypes.FLOORDIVEQ; - } + case 73: + { return PyTokenTypes.FLOORDIVEQ; + } case 168: break; - case 39: - { return PyTokenTypes.FLOAT_LITERAL; - } + case 39: + { return PyTokenTypes.FLOAT_LITERAL; + } case 169: break; - case 4: - { return PyTokenTypes.LINE_BREAK; - } + case 4: + { return PyTokenTypes.LINE_BREAK; + } case 170: break; - case 52: - { return PyTokenTypes.PERCEQ; - } + case 52: + { return PyTokenTypes.PERCEQ; + } case 171: break; - case 71: - { return PyTokenTypes.TRY_KEYWORD; - } + case 71: + { return PyTokenTypes.TRY_KEYWORD; + } case 172: break; - case 53: - { return PyTokenTypes.ANDEQ; - } + case 53: + { return PyTokenTypes.ANDEQ; + } case 173: break; - case 17: - { return PyTokenTypes.EQ; - } + case 17: + { return PyTokenTypes.EQ; + } case 174: break; - case 76: - { return PyTokenTypes.FROM_KEYWORD; - } + case 76: + { return PyTokenTypes.FROM_KEYWORD; + } case 175: break; - case 72: - { return PyTokenTypes.EXPEQ; - } + case 72: + { return PyTokenTypes.EXPEQ; + } case 176: break; - case 78: - { return PyTokenTypes.ELIF_KEYWORD; - } + case 78: + { return PyTokenTypes.ELIF_KEYWORD; + } case 177: break; - case 32: - { return PyTokenTypes.RBRACE; - } + case 32: + { return PyTokenTypes.RBRACE; + } case 178: break; - case 43: - { return PyTokenTypes.IF_KEYWORD; - } + case 43: + { return PyTokenTypes.IF_KEYWORD; + } case 179: break; - case 6: - { return PyTokenTypes.TAB; - } + case 6: + { return PyTokenTypes.TAB; + } case 180: break; - case 59: - { return PyTokenTypes.NE_OLD; - } + case 59: + { return PyTokenTypes.NE_OLD; + } case 181: break; - case 26: - { return PyTokenTypes.TILDE; - } + case 26: + { return PyTokenTypes.TILDE; + } case 182: break; - case 23: - { return PyTokenTypes.XOR; - } + case 23: + { return PyTokenTypes.XOR; + } case 183: break; - case 70: - { return PyTokenTypes.DEL_KEYWORD; - } + case 70: + { return PyTokenTypes.DEL_KEYWORD; + } case 184: break; default: if (zzInput == YYEOF && zzStartRead == zzCurrentPos) { @@ -1261,4 +1274,4 @@ class _PythonLexer implements FlexLexer { } -} \ No newline at end of file +} diff --git a/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java b/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java index 525fa2b70a5a..e110791a9699 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java @@ -13,6 +13,7 @@ import com.intellij.psi.tree.IElementType; import com.intellij.psi.util.PsiTreeUtil; import com.jetbrains.python.PyTokenTypes; import com.jetbrains.python.codeInsight.regexp.PythonVerboseRegexpLanguage; +import com.jetbrains.python.lexer.PyStringLiteralLexer; import com.jetbrains.python.lexer.PythonHighlightingLexer; import com.jetbrains.python.psi.*; import com.jetbrains.python.psi.types.PyType; @@ -100,34 +101,20 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt public static int getPrefixLength(String text) { int startOffset = 0; - startOffset = skipEncodingPrefix(text, startOffset); - startOffset = skipRawPrefix(text, startOffset); - return startOffset; - } - - private static int skipRawPrefix(String text, int startOffset) { - char c = Character.toUpperCase(text.charAt(startOffset)); - if (c == 'R') { - startOffset++; - } - return startOffset; - } - - private static int skipEncodingPrefix(String text, int startOffset) { - char c = Character.toUpperCase(text.charAt(startOffset)); - if (c == 'U' || c == 'B' || c == 'C') { - startOffset++; - } + startOffset = PyStringLiteralLexer.skipEncodingPrefix(text, startOffset); + startOffset = PyStringLiteralLexer.skipRawPrefix(text, startOffset); + startOffset = PyStringLiteralLexer.skipEncodingPrefix(text, startOffset); + startOffset = PyStringLiteralLexer.skipRawPrefix(text, startOffset); return startOffset; } private static boolean isRaw(String text) { - int startOffset = skipEncodingPrefix(text, 0); - return skipRawPrefix(text, startOffset) > startOffset; + int startOffset = PyStringLiteralLexer.skipEncodingPrefix(text, 0); + return PyStringLiteralLexer.skipRawPrefix(text, startOffset) > startOffset; } private static boolean isUnicode(String text) { - return text.length() > 0 && Character.toUpperCase(text.charAt(0)) == 'U'; + return text.length() > 0 && Character.toUpperCase(text.charAt(0)) == 'U'; //TODO[ktisha] } private static boolean isBytes(String text) { diff --git a/python/src/com/jetbrains/python/validation/CompatibilityVisitor.java b/python/src/com/jetbrains/python/validation/CompatibilityVisitor.java index 81febf3554f2..89745f3c97d9 100644 --- a/python/src/com/jetbrains/python/validation/CompatibilityVisitor.java +++ b/python/src/com/jetbrains/python/validation/CompatibilityVisitor.java @@ -1,8 +1,12 @@ package com.jetbrains.python.validation; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import com.intellij.codeInspection.LocalQuickFix; +import com.intellij.lang.ASTNode; import com.intellij.openapi.roots.ProjectFileIndex; import com.intellij.openapi.roots.ProjectRootManager; +import com.intellij.openapi.util.TextRange; import com.intellij.psi.PsiElement; import com.intellij.psi.PsiFile; import com.intellij.psi.PsiWhiteSpace; @@ -12,12 +16,10 @@ import com.jetbrains.python.PyTokenTypes; import com.jetbrains.python.inspections.quickfix.*; import com.jetbrains.python.psi.*; import com.jetbrains.python.psi.impl.PyQualifiedName; +import com.jetbrains.python.psi.impl.PyStringLiteralExpressionImpl; import org.jetbrains.annotations.Nullable; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.Stack; +import java.util.*; /** * User : catherine @@ -26,6 +28,19 @@ public abstract class CompatibilityVisitor extends PyAnnotator { protected List myVersionsToProcess; private String myCommonMessage = "Python version "; + private static final Map> AVAILABLE_PREFIXES = Maps.newHashMap(); + + static { + AVAILABLE_PREFIXES.put(LanguageLevel.PYTHON24, Sets.newHashSet("R", "U", "UR")); + AVAILABLE_PREFIXES.put(LanguageLevel.PYTHON25, Sets.newHashSet("R", "U", "UR")); + AVAILABLE_PREFIXES.put(LanguageLevel.PYTHON26, Sets.newHashSet("R", "U", "UR", "B", "BR")); + AVAILABLE_PREFIXES.put(LanguageLevel.PYTHON27, Sets.newHashSet("R", "U", "UR", "B", "BR")); + AVAILABLE_PREFIXES.put(LanguageLevel.PYTHON30, Sets.newHashSet("R", "B")); + AVAILABLE_PREFIXES.put(LanguageLevel.PYTHON31, Sets.newHashSet("R", "B", "BR")); + AVAILABLE_PREFIXES.put(LanguageLevel.PYTHON32, Sets.newHashSet("R", "B", "BR")); + AVAILABLE_PREFIXES.put(LanguageLevel.PYTHON33, Sets.newHashSet("R", "U", "B", "BR", "RB")); + } + public CompatibilityVisitor(List versionsToProcess) { myVersionsToProcess = versionsToProcess; } @@ -213,19 +228,25 @@ public abstract class CompatibilityVisitor extends PyAnnotator { @Override public void visitPyStringLiteralExpression(final PyStringLiteralExpression node) { super.visitPyStringLiteralExpression(node); - int len = 0; - StringBuilder message = new StringBuilder(myCommonMessage); - for (int i = 0; i != myVersionsToProcess.size(); ++i) { - LanguageLevel languageLevel = myVersionsToProcess.get(i); + List stringNodes = node.getStringNodes(); - if (languageLevel.isAtLeast(LanguageLevel.PYTHON30) && languageLevel.isOlderThan(LanguageLevel.PYTHON33)) { - final String text = node.getText(); - if (text.startsWith("u") || text.startsWith("U")) { + for (ASTNode stringNode : stringNodes) { + int len = 0; + StringBuilder message = new StringBuilder(myCommonMessage); + String nodeText = stringNode.getText(); + int index = PyStringLiteralExpressionImpl.getPrefixLength(nodeText); + String prefix = nodeText.substring(0, index).toUpperCase(); + final TextRange range = TextRange.create(stringNode.getStartOffset(), stringNode.getStartOffset() + index); + for (int i = 0; i != myVersionsToProcess.size(); ++i) { + LanguageLevel languageLevel = myVersionsToProcess.get(i); + if (prefix.isEmpty()) continue; + + final Set prefixes = AVAILABLE_PREFIXES.get(languageLevel); + if (!prefixes.contains(prefix)) len = appendLanguageLevel(message, len, languageLevel); - } } + commonRegisterProblem(message, " not support a '" + prefix + "' prefix", len, node, range, new RemovePrefixQuickFix(prefix)); } - commonRegisterProblem(message, " not support a leading \'u\' or \'U\'.", len, node, new RemoveLeadingUQuickFix()); } @Override @@ -552,6 +573,7 @@ public abstract class CompatibilityVisitor extends PyAnnotator { } protected abstract void registerProblem(PsiElement node, String s, @Nullable LocalQuickFix localQuickFix, boolean asError); + protected abstract void registerProblem(PsiElement node, TextRange range, String s, @Nullable LocalQuickFix localQuickFix, boolean asError); protected void registerProblem(final PsiElement node, final String s, @Nullable final LocalQuickFix localQuickFix) { registerProblem(node, s, localQuickFix, true); @@ -567,7 +589,22 @@ public abstract class CompatibilityVisitor extends PyAnnotator { protected void commonRegisterProblem(StringBuilder initMessage, String suffix, int len, PyElement node, LocalQuickFix localQuickFix) { - commonRegisterProblem(initMessage, suffix, len, node, localQuickFix, true); + commonRegisterProblem(initMessage, suffix, len, node, node.getTextRange(), localQuickFix, true); + } + + protected void commonRegisterProblem(StringBuilder initMessage, String suffix, + int len, PyElement node, TextRange range, LocalQuickFix localQuickFix) { + commonRegisterProblem(initMessage, suffix, len, node, range, localQuickFix, true); + } + + protected void commonRegisterProblem(StringBuilder initMessage, String suffix, + int len, PyElement node, TextRange range, @Nullable LocalQuickFix localQuickFix, boolean asError) { + initMessage.append(" do"); + if (len == 1) + initMessage.append("es"); + initMessage.append(suffix); + if (len != 0) + registerProblem(node, range, initMessage.toString(), localQuickFix, asError); } protected void commonRegisterProblem(StringBuilder initMessage, String suffix, @@ -577,7 +614,7 @@ public abstract class CompatibilityVisitor extends PyAnnotator { initMessage.append("es"); initMessage.append(suffix); if (len != 0) - registerProblem(node, initMessage.toString(), localQuickFix, asError); + registerProblem(node, node.getTextRange(), initMessage.toString(), localQuickFix, asError); } protected static int appendLanguageLevel(StringBuilder message, int len, LanguageLevel languageLevel) { diff --git a/python/src/com/jetbrains/python/validation/UnsupportedFeatures.java b/python/src/com/jetbrains/python/validation/UnsupportedFeatures.java index 00de71d63e9c..680f0a49c0fb 100644 --- a/python/src/com/jetbrains/python/validation/UnsupportedFeatures.java +++ b/python/src/com/jetbrains/python/validation/UnsupportedFeatures.java @@ -7,11 +7,13 @@ import com.intellij.codeInspection.ProblemDescriptor; import com.intellij.codeInspection.ProblemHighlightType; import com.intellij.codeInspection.ex.ProblemDescriptorImpl; import com.intellij.codeInspection.ex.QuickFixWrapper; +import com.intellij.openapi.util.TextRange; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.psi.PsiElement; import com.jetbrains.python.psi.LanguageLevel; import com.jetbrains.python.psi.PyElement; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import java.util.ArrayList; import java.util.Arrays; @@ -31,20 +33,26 @@ public class UnsupportedFeatures extends CompatibilityVisitor { } @Override - protected void registerProblem(PsiElement node, String message, LocalQuickFix localQuickFix, boolean asError) { - if (node == null || node.getTextLength() == 0){ - return; + protected void registerProblem(@Nullable final PsiElement node, String message, LocalQuickFix localQuickFix, boolean asError) { + if (node == null) return; + registerProblem(node, node.getTextRange(), message, localQuickFix, asError); + } + + @Override + protected void registerProblem(PsiElement node, TextRange range, String message, LocalQuickFix localQuickFix, boolean asError) { + if (range.isEmpty()){ + return; } if (localQuickFix != null) if (asError) - getHolder().createErrorAnnotation(node, message).registerFix(createIntention(node, message, localQuickFix)); + getHolder().createErrorAnnotation(range, message).registerFix(createIntention(node, message, localQuickFix)); else - getHolder().createWarningAnnotation(node, message).registerFix(createIntention(node, message, localQuickFix)); + getHolder().createWarningAnnotation(range, message).registerFix(createIntention(node, message, localQuickFix)); else if (asError) - getHolder().createErrorAnnotation(node, message); + getHolder().createErrorAnnotation(range, message); else - getHolder().createWarningAnnotation(node, message); + getHolder().createWarningAnnotation(range, message); } @NotNull @@ -57,10 +65,14 @@ public class UnsupportedFeatures extends CompatibilityVisitor { } private static IntentionAction createIntention(PsiElement node, String message, LocalQuickFix fix) { + return createIntention(node, node.getTextRange(), message, fix); + } + + private static IntentionAction createIntention(PsiElement node, TextRange range, String message, LocalQuickFix fix) { LocalQuickFix[] quickFixes = {fix}; CommonProblemDescriptorImpl descr = new ProblemDescriptorImpl(node, node, message, quickFixes, ProblemHighlightType.GENERIC_ERROR_OR_WARNING, true, - node.getTextRange(), true); + range, true); return QuickFixWrapper.wrap((ProblemDescriptor)descr, 0); } } diff --git a/python/testData/highlighting/unsupportedFeaturesInPython3.py b/python/testData/highlighting/unsupportedFeaturesInPython3.py index 2286f0d382a8..fce5f970f8ef 100644 --- a/python/testData/highlighting/unsupportedFeaturesInPython3.py +++ b/python/testData/highlighting/unsupportedFeaturesInPython3.py @@ -5,7 +5,7 @@ a = u"text" +s = u"text" raise a, b, c raise a, b diff --git a/python/testData/inspections/PyCompatibilityInspection/stringLiteralExpression.py b/python/testData/inspections/PyCompatibilityInspection/stringLiteralExpression.py index 75e998342171..1f7dd8799901 100644 --- a/python/testData/inspections/PyCompatibilityInspection/stringLiteralExpression.py +++ b/python/testData/inspections/PyCompatibilityInspection/stringLiteralExpression.py @@ -1 +1,42 @@ -a = u"String" +a = u"String" + +# python 3.3 + +a = u"" +a = r"" +a = b"" +a = rb"" +a = br"" + +# python 3.2, 3.1 + +a = r"" +a = b"" +a = br"" + +# python 3.0 + +a = r"" +a = b"" + +# python 2.7, 2.6 + +a = u"" +a = r"" +a = ur"" +a = b"" +a = br"" + +# python 2.5 + +a = u"" +a = r"" +a = ur"" + +# combined +b = u"" b"" + +# never was available +a = rr"" +a = bb"" +a = uu"" \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/python/PyIntentionTest.java b/python/testSrc/com/jetbrains/python/PyIntentionTest.java index 44c0a6902e69..7e81e62d0e27 100644 --- a/python/testSrc/com/jetbrains/python/PyIntentionTest.java +++ b/python/testSrc/com/jetbrains/python/PyIntentionTest.java @@ -69,7 +69,7 @@ public class PyIntentionTest extends PyTestCase { } public void testRemoveLeadingU() { - doTest(PyBundle.message("INTN.remove.leading.u"), LanguageLevel.PYTHON30); + doTest(PyBundle.message("INTN.remove.leading.$0", "U"), LanguageLevel.PYTHON30); } public void testRemoveTrailingL() {