diff --git a/python/python-psi-api/src/com/jetbrains/python/PyTokenTypes.java b/python/python-psi-api/src/com/jetbrains/python/PyTokenTypes.java index efb90730772b..74bcb79d89d6 100644 --- a/python/python-psi-api/src/com/jetbrains/python/PyTokenTypes.java +++ b/python/python-psi-api/src/com/jetbrains/python/PyTokenTypes.java @@ -181,6 +181,7 @@ public class PyTokenTypes { public static final PyElementType DEDENT = new PyElementType("DEDENT"); public static final PyElementType FSTRING_TEXT = new PyElementType("FSTRING_TEXT"); + public static final PyElementType FSTRING_RAW_TEXT = new PyElementType("FSTRING_RAW_TEXT"); public static final PyElementType FSTRING_START = new PyElementType("FSTRING_START"); public static final PyElementType FSTRING_END = new PyElementType("FSTRING_END"); public static final PyElementType FSTRING_FRAGMENT_START = new PyElementType("FSTRING_FRAGMENT_START"); @@ -195,4 +196,6 @@ public class PyTokenTypes { FSTRING_FRAGMENT_END, FSTRING_FRAGMENT_FORMAT_START, FSTRING_FRAGMENT_TYPE_CONVERSION); + + public static final TokenSet FSTRING_TEXT_TOKENS = TokenSet.create(FSTRING_TEXT, FSTRING_RAW_TEXT); } diff --git a/python/python-psi-impl/gen/com/jetbrains/python/lexer/_PythonLexer.java b/python/python-psi-impl/gen/com/jetbrains/python/lexer/_PythonLexer.java index 085c3c7357b4..04af6f56def6 100644 --- a/python/python-psi-impl/gen/com/jetbrains/python/lexer/_PythonLexer.java +++ b/python/python-psi-impl/gen/com/jetbrains/python/lexer/_PythonLexer.java @@ -1271,7 +1271,7 @@ return yylength()-s.length(); // fall through case 160: break; case 40: - { return PyTokenTypes.FSTRING_TEXT; + { return fStringHelper.getTextTokenType(); } // fall through case 161: break; diff --git a/python/python-psi-impl/src/com/jetbrains/python/lexer/PyFStringLiteralLexer.kt b/python/python-psi-impl/src/com/jetbrains/python/lexer/PyFStringLiteralLexer.kt index 2ee57838126b..ead269cd8887 100644 --- a/python/python-psi-impl/src/com/jetbrains/python/lexer/PyFStringLiteralLexer.kt +++ b/python/python-psi-impl/src/com/jetbrains/python/lexer/PyFStringLiteralLexer.kt @@ -1,10 +1,15 @@ // Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. package com.jetbrains.python.lexer +import com.intellij.psi.tree.IElementType import com.intellij.util.text.CharArrayUtil import com.jetbrains.python.PyTokenTypes -class PyFStringLiteralLexer: PyStringLiteralLexerBase(PyTokenTypes.FSTRING_TEXT) { +class PyFStringLiteralLexer(fStringTextToken: IElementType) : PyStringLiteralLexerBase(fStringTextToken) { + init { + assert(PyTokenTypes.FSTRING_TEXT_TOKENS.contains(fStringTextToken)) + } + override fun locateToken(start: Int): Int { if (start >= myBufferEnd) { return myBufferEnd @@ -19,8 +24,7 @@ class PyFStringLiteralLexer: PyStringLiteralLexerBase(PyTokenTypes.FSTRING_TEXT) } } - // TODO actually keep track of "raw" prefixes of f-strings somehow - override fun isRaw(): Boolean = false + override fun isRaw(): Boolean = myOriginalLiteralToken == PyTokenTypes.FSTRING_RAW_TEXT override fun isUnicodeMode(): Boolean = true diff --git a/python/python-psi-impl/src/com/jetbrains/python/lexer/PyLexerFStringHelper.kt b/python/python-psi-impl/src/com/jetbrains/python/lexer/PyLexerFStringHelper.kt index c810353c4e85..4cb89cc95bef 100644 --- a/python/python-psi-impl/src/com/jetbrains/python/lexer/PyLexerFStringHelper.kt +++ b/python/python-psi-impl/src/com/jetbrains/python/lexer/PyLexerFStringHelper.kt @@ -25,15 +25,18 @@ class PyLexerFStringHelper(private val myLexer: FlexLexerEx) { } private fun pushFString(prefixAndQuotes: String): PyElementType { - val openingQuotes = prefixAndQuotes.substring(PyStringLiteralUtil.getPrefixLength(prefixAndQuotes)) - myFStringStates.push(FStringState(myLexer.yystate(), myLexer.tokenStart, openingQuotes)) + val prefixLength = PyStringLiteralUtil.getPrefixLength(prefixAndQuotes) + val openingQuotes = prefixAndQuotes.substring(prefixLength) + val prefix = prefixAndQuotes.substring(0, prefixLength) + myFStringStates.push(FStringState(myLexer.yystate(), myLexer.tokenStart, prefix, openingQuotes)) myLexer.yybegin(_PythonLexer.FSTRING) return PyTokenTypes.FSTRING_START } fun handleFStringEnd(): IElementType { + val textType = getTextTokenType() val (type, offset) = findFStringTerminator(myLexer.yytext().toString()) - return if (offset == 0) type!! else PyTokenTypes.FSTRING_TEXT + return if (offset == 0) type!! else textType } fun handleFragmentStart(): IElementType { @@ -96,7 +99,7 @@ class PyLexerFStringHelper(private val myLexer: FlexLexerEx) { val text = myLexer.yytext().toString() val (_, offset) = findFStringTerminator(text) if (offset == text.length) { - return PyTokenTypes.FSTRING_TEXT + return getTextTokenType() } return PyTokenTypes.LINE_BREAK } @@ -169,7 +172,18 @@ class PyLexerFStringHelper(private val myLexer: FlexLexerEx) { myFStringStates.clear() } - private data class FStringState(val oldState: Int, val offset: Int, val openingQuotes: String) { + fun getTextTokenType(): IElementType { + assert(myFStringStates.isNotEmpty()) + if (PyStringLiteralUtil.isRawPrefix(myFStringStates.peek().prefix)) { + return PyTokenTypes.FSTRING_RAW_TEXT + } + return PyTokenTypes.FSTRING_TEXT + } + + private data class FStringState(val oldState: Int, + val offset: Int, + val prefix: String, + val openingQuotes: String) { val fragmentStates = Stack() } diff --git a/python/python-psi-impl/src/com/jetbrains/python/lexer/Python.flex b/python/python-psi-impl/src/com/jetbrains/python/lexer/Python.flex index 39140e970f36..00ec89f230cf 100644 --- a/python/python-psi-impl/src/com/jetbrains/python/lexer/Python.flex +++ b/python/python-psi-impl/src/com/jetbrains/python/lexer/Python.flex @@ -101,8 +101,8 @@ return yylength()-s.length(); %% { - {FSTRING_TEXT_NO_QUOTES} { return PyTokenTypes.FSTRING_TEXT; } - "\\" { return PyTokenTypes.FSTRING_TEXT; } + {FSTRING_TEXT_NO_QUOTES} { return fStringHelper.getTextTokenType(); } + "\\" { return fStringHelper.getTextTokenType(); } [\n] { return fStringHelper.handleLineBreakInLiteralText(); } {FSTRING_QUOTES} { return fStringHelper.handleFStringEnd(); } "{" { return fStringHelper.handleFragmentStart(); } @@ -136,8 +136,8 @@ return yylength()-s.length(); } { - {FSTRING_FORMAT_TEXT_NO_QUOTES} { return PyTokenTypes.FSTRING_TEXT; } - "\\" { return PyTokenTypes.FSTRING_TEXT; } + {FSTRING_FORMAT_TEXT_NO_QUOTES} { return fStringHelper.getTextTokenType(); } + "\\" { return fStringHelper.getTextTokenType(); } [\n] { return fStringHelper.handleLineBreakInLiteralText(); } {FSTRING_QUOTES} { return fStringHelper.handleFStringEnd(); } "{" { return fStringHelper.handleFragmentStart(); } diff --git a/python/python-psi-impl/src/com/jetbrains/python/parsing/ExpressionParsing.java b/python/python-psi-impl/src/com/jetbrains/python/parsing/ExpressionParsing.java index 5c9d291e472b..152508527a71 100644 --- a/python/python-psi-impl/src/com/jetbrains/python/parsing/ExpressionParsing.java +++ b/python/python-psi-impl/src/com/jetbrains/python/parsing/ExpressionParsing.java @@ -127,7 +127,7 @@ public class ExpressionParsing extends Parsing { final PsiBuilder.Marker marker = builder.mark(); nextToken(); while (true) { - if (atToken(PyTokenTypes.FSTRING_TEXT)) { + if (atAnyOfTokens(PyTokenTypes.FSTRING_TEXT_TOKENS)) { nextToken(); } else if (atToken(PyTokenTypes.FSTRING_FRAGMENT_START)) { @@ -210,7 +210,7 @@ public class ExpressionParsing extends Parsing { final PsiBuilder.Marker marker = myContext.getBuilder().mark(); nextToken(); while (true) { - if (atToken(PyTokenTypes.FSTRING_TEXT)) { + if (atAnyOfTokens(PyTokenTypes.FSTRING_TEXT_TOKENS)) { nextToken(); } else if (atToken(PyTokenTypes.FSTRING_FRAGMENT_START)) { diff --git a/python/python-psi-impl/src/com/jetbrains/python/parsing/Parsing.java b/python/python-psi-impl/src/com/jetbrains/python/parsing/Parsing.java index 67d17b791d9c..c97c469601b0 100644 --- a/python/python-psi-impl/src/com/jetbrains/python/parsing/Parsing.java +++ b/python/python-psi-impl/src/com/jetbrains/python/parsing/Parsing.java @@ -18,6 +18,7 @@ package com.jetbrains.python.parsing; import com.intellij.lang.PsiBuilder; import com.intellij.openapi.diagnostic.Logger; import com.intellij.psi.tree.IElementType; +import com.intellij.psi.tree.TokenSet; import com.jetbrains.python.PyPsiBundle; import com.jetbrains.python.PyElementTypes; import com.jetbrains.python.PyTokenTypes; @@ -98,6 +99,10 @@ public class Parsing { return false; } + protected boolean atAnyOfTokens(@NotNull TokenSet tokenTypes) { + return tokenTypes.contains(myBuilder.getTokenType()); + } + protected boolean matchToken(final IElementType tokenType) { if (myBuilder.getTokenType() == tokenType) { myBuilder.advanceLexer(); diff --git a/python/python-psi-impl/src/com/jetbrains/python/psi/impl/PyFormattedStringElementImpl.java b/python/python-psi-impl/src/com/jetbrains/python/psi/impl/PyFormattedStringElementImpl.java index f7e3d5ee3881..8fde6cf5760a 100644 --- a/python/python-psi-impl/src/com/jetbrains/python/psi/impl/PyFormattedStringElementImpl.java +++ b/python/python-psi-impl/src/com/jetbrains/python/psi/impl/PyFormattedStringElementImpl.java @@ -43,7 +43,7 @@ public class PyFormattedStringElementImpl extends PyElementImpl implements PyFor final TextRange contentRange = getContentRange(); return SyntaxTraverser.psiApi() .children(this) - .filter(child -> child.getNode().getElementType() == PyTokenTypes.FSTRING_TEXT) + .filter(child -> PyTokenTypes.FSTRING_TEXT_TOKENS.contains(child.getNode().getElementType())) .map(PsiElement::getTextRangeInParent) .map(range -> range.intersection(contentRange)) .toList(); @@ -103,7 +103,7 @@ public class PyFormattedStringElementImpl extends PyElementImpl implements PyFor result.add(Pair.create(relChildRange, child.getText())); } } - else if (childType == PyTokenTypes.FSTRING_TEXT) { + else if (PyTokenTypes.FSTRING_TEXT_TOKENS.contains(childType)) { if (continuousTextStart == -1) { continuousTextStart = relChildRange.getStartOffset(); } diff --git a/python/src/com/jetbrains/python/highlighting/PyHighlighter.java b/python/src/com/jetbrains/python/highlighting/PyHighlighter.java index 41c57eb71d56..66ecb2e088fa 100644 --- a/python/src/com/jetbrains/python/highlighting/PyHighlighter.java +++ b/python/src/com/jetbrains/python/highlighting/PyHighlighter.java @@ -49,9 +49,13 @@ public class PyHighlighter extends SyntaxHighlighterBase { PyTokenTypes.TRIPLE_QUOTED_UNICODE ); ret.registerLayer( - new PyFStringLiteralLexer(), + new PyFStringLiteralLexer(PyTokenTypes.FSTRING_TEXT), PyTokenTypes.FSTRING_TEXT ); + ret.registerLayer( + new PyFStringLiteralLexer(PyTokenTypes.FSTRING_RAW_TEXT), + PyTokenTypes.FSTRING_RAW_TEXT + ); return ret; } @@ -136,7 +140,8 @@ public class PyHighlighter extends SyntaxHighlighterBase { keys.put(PyTokenTypes.FSTRING_START, PY_UNICODE_STRING); keys.put(PyTokenTypes.FSTRING_END, PY_UNICODE_STRING); keys.put(PyTokenTypes.FSTRING_TEXT, PY_UNICODE_STRING); - + keys.put(PyTokenTypes.FSTRING_RAW_TEXT, PY_UNICODE_STRING); + keys.put(PyTokenTypes.FSTRING_FRAGMENT_TYPE_CONVERSION, PY_FSTRING_FRAGMENT_TYPE_CONVERSION); keys.put(PyTokenTypes.FSTRING_FRAGMENT_FORMAT_START, PY_FSTRING_FRAGMENT_COLON); keys.put(PyTokenTypes.FSTRING_FRAGMENT_START, PY_FSTRING_FRAGMENT_BRACES); diff --git a/python/testData/psi/FStringEscapeInFormatPartOfPlainLiteral.py b/python/testData/psi/FStringEscapeInFormatPartOfPlainLiteral.py new file mode 100644 index 000000000000..940530f13315 --- /dev/null +++ b/python/testData/psi/FStringEscapeInFormatPartOfPlainLiteral.py @@ -0,0 +1 @@ +s = f'{x:\n}' \ No newline at end of file diff --git a/python/testData/psi/FStringEscapeInFormatPartOfPlainLiteral.txt b/python/testData/psi/FStringEscapeInFormatPartOfPlainLiteral.txt new file mode 100644 index 000000000000..5d6b94fb07ef --- /dev/null +++ b/python/testData/psi/FStringEscapeInFormatPartOfPlainLiteral.txt @@ -0,0 +1,19 @@ +PyFile:FStringEscapeInFormatPartOfPlainLiteral.py + PyAssignmentStatement + PyTargetExpression: s + PsiElement(Py:IDENTIFIER)('s') + PsiWhiteSpace(' ') + PsiElement(Py:EQ)('=') + PsiWhiteSpace(' ') + PyStringLiteralExpression: {x:\n} + PyFormattedStringElement + PsiElement(Py:FSTRING_START)('f'') + PyFStringFragment + PsiElement(Py:FSTRING_FRAGMENT_START)('{') + PyReferenceExpression: x + PsiElement(Py:IDENTIFIER)('x') + PyFStringFragmentFormatPart + PsiElement(Py:FSTRING_FRAGMENT_FORMAT_START)(':') + PsiElement(Py:FSTRING_TEXT)('\n') + PsiElement(Py:FSTRING_FRAGMENT_END)('}') + PsiElement(Py:FSTRING_END)(''') \ No newline at end of file diff --git a/python/testData/psi/FStringEscapeInFormatPartOfRawLiteral.py b/python/testData/psi/FStringEscapeInFormatPartOfRawLiteral.py new file mode 100644 index 000000000000..f149d2a2a4b8 --- /dev/null +++ b/python/testData/psi/FStringEscapeInFormatPartOfRawLiteral.py @@ -0,0 +1 @@ +s = fr'{x:\n}' \ No newline at end of file diff --git a/python/testData/psi/FStringEscapeInFormatPartOfRawLiteral.txt b/python/testData/psi/FStringEscapeInFormatPartOfRawLiteral.txt new file mode 100644 index 000000000000..0b9ecb8085ee --- /dev/null +++ b/python/testData/psi/FStringEscapeInFormatPartOfRawLiteral.txt @@ -0,0 +1,19 @@ +PyFile:FStringEscapeInFormatPartOfRawLiteral.py + PyAssignmentStatement + PyTargetExpression: s + PsiElement(Py:IDENTIFIER)('s') + PsiWhiteSpace(' ') + PsiElement(Py:EQ)('=') + PsiWhiteSpace(' ') + PyStringLiteralExpression: {x:\n} + PyFormattedStringElement + PsiElement(Py:FSTRING_START)('fr'') + PyFStringFragment + PsiElement(Py:FSTRING_FRAGMENT_START)('{') + PyReferenceExpression: x + PsiElement(Py:IDENTIFIER)('x') + PyFStringFragmentFormatPart + PsiElement(Py:FSTRING_FRAGMENT_FORMAT_START)(':') + PsiElement(Py:FSTRING_RAW_TEXT)('\n') + PsiElement(Py:FSTRING_FRAGMENT_END)('}') + PsiElement(Py:FSTRING_END)(''') \ No newline at end of file diff --git a/python/testData/psi/FStringPlainInsideRawFString.py b/python/testData/psi/FStringPlainInsideRawFString.py new file mode 100644 index 000000000000..cd1475015bd8 --- /dev/null +++ b/python/testData/psi/FStringPlainInsideRawFString.py @@ -0,0 +1 @@ +s = rf'foo{f"\n"}bar\n' \ No newline at end of file diff --git a/python/testData/psi/FStringPlainInsideRawFString.txt b/python/testData/psi/FStringPlainInsideRawFString.txt new file mode 100644 index 000000000000..913d1e1a6a49 --- /dev/null +++ b/python/testData/psi/FStringPlainInsideRawFString.txt @@ -0,0 +1,22 @@ +PyFile:FStringPlainInsideRawFString.py + PyAssignmentStatement + PyTargetExpression: s + PsiElement(Py:IDENTIFIER)('s') + PsiWhiteSpace(' ') + PsiElement(Py:EQ)('=') + PsiWhiteSpace(' ') + PyStringLiteralExpression: foo{f"\n"}bar\n + PyFormattedStringElement + PsiElement(Py:FSTRING_START)('rf'') + PsiElement(Py:FSTRING_RAW_TEXT)('foo') + PyFStringFragment + PsiElement(Py:FSTRING_FRAGMENT_START)('{') + PyStringLiteralExpression: + + PyFormattedStringElement + PsiElement(Py:FSTRING_START)('f"') + PsiElement(Py:FSTRING_TEXT)('\n') + PsiElement(Py:FSTRING_END)('"') + PsiElement(Py:FSTRING_FRAGMENT_END)('}') + PsiElement(Py:FSTRING_RAW_TEXT)('bar\n') + PsiElement(Py:FSTRING_END)(''') \ No newline at end of file diff --git a/python/testData/psi/FStringRawFStringInsidePlainFString.py b/python/testData/psi/FStringRawFStringInsidePlainFString.py new file mode 100644 index 000000000000..c5d3f117672d --- /dev/null +++ b/python/testData/psi/FStringRawFStringInsidePlainFString.py @@ -0,0 +1 @@ +s = f'foo{rf"\n"}bar\n' \ No newline at end of file diff --git a/python/testData/psi/FStringRawFStringInsidePlainFString.txt b/python/testData/psi/FStringRawFStringInsidePlainFString.txt new file mode 100644 index 000000000000..9477f363730d --- /dev/null +++ b/python/testData/psi/FStringRawFStringInsidePlainFString.txt @@ -0,0 +1,22 @@ +PyFile:FStringRawFStringInsidePlainFString.py + PyAssignmentStatement + PyTargetExpression: s + PsiElement(Py:IDENTIFIER)('s') + PsiWhiteSpace(' ') + PsiElement(Py:EQ)('=') + PsiWhiteSpace(' ') + PyStringLiteralExpression: foo{rf"\n"}bar + + PyFormattedStringElement + PsiElement(Py:FSTRING_START)('f'') + PsiElement(Py:FSTRING_TEXT)('foo') + PyFStringFragment + PsiElement(Py:FSTRING_FRAGMENT_START)('{') + PyStringLiteralExpression: \n + PyFormattedStringElement + PsiElement(Py:FSTRING_START)('rf"') + PsiElement(Py:FSTRING_RAW_TEXT)('\n') + PsiElement(Py:FSTRING_END)('"') + PsiElement(Py:FSTRING_FRAGMENT_END)('}') + PsiElement(Py:FSTRING_TEXT)('bar\n') + PsiElement(Py:FSTRING_END)(''') \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/python/PythonHighlightingLexerTest.java b/python/testSrc/com/jetbrains/python/PythonHighlightingLexerTest.java index dbb181d83e2a..55d324f9b78e 100644 --- a/python/testSrc/com/jetbrains/python/PythonHighlightingLexerTest.java +++ b/python/testSrc/com/jetbrains/python/PythonHighlightingLexerTest.java @@ -207,40 +207,64 @@ public class PythonHighlightingLexerTest extends PyLexerTestCase { // PY-31758 public void testFStringEscapeSequences() { - doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\nbar'", + doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\nbar'", "Py:FSTRING_START", "Py:FSTRING_TEXT", "VALID_STRING_ESCAPE_TOKEN", "Py:FSTRING_TEXT", "Py:FSTRING_END"); - doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\\nbar'", + doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\\nbar'", "Py:FSTRING_START", "Py:FSTRING_TEXT", "VALID_STRING_ESCAPE_TOKEN", "Py:FSTRING_TEXT", "Py:FSTRING_END"); - doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\u0041bar'", + doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\u0041bar'", "Py:FSTRING_START", "Py:FSTRING_TEXT", "VALID_STRING_ESCAPE_TOKEN", "Py:FSTRING_TEXT", "Py:FSTRING_END"); - doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\x41bar'", + doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\x41bar'", "Py:FSTRING_START", "Py:FSTRING_TEXT", "VALID_STRING_ESCAPE_TOKEN", "Py:FSTRING_TEXT", "Py:FSTRING_END"); - doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\101bar'", + doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\101bar'", "Py:FSTRING_START", "Py:FSTRING_TEXT", "VALID_STRING_ESCAPE_TOKEN", "Py:FSTRING_TEXT", "Py:FSTRING_END"); - doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\N{GREEK SMALL LETTER ALPHA}bar'", + doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\N{GREEK SMALL LETTER ALPHA}bar'", "Py:FSTRING_START", "Py:FSTRING_TEXT", "VALID_STRING_ESCAPE_TOKEN", "Py:FSTRING_TEXT", "Py:FSTRING_END"); doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\", "Py:FSTRING_START", "Py:FSTRING_TEXT", "INVALID_CHARACTER_ESCAPE_TOKEN"); - doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\u00'", + doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\u00'", "Py:FSTRING_START", "Py:FSTRING_TEXT", "INVALID_UNICODE_ESCAPE_TOKEN", "Py:FSTRING_END"); - doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\uZZZZbar'", + doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\uZZZZbar'", "Py:FSTRING_START", "Py:FSTRING_TEXT", "INVALID_UNICODE_ESCAPE_TOKEN", "Py:FSTRING_TEXT", "Py:FSTRING_END"); - doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\x0'", + doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\x0'", "Py:FSTRING_START", "Py:FSTRING_TEXT", "INVALID_UNICODE_ESCAPE_TOKEN", "Py:FSTRING_END"); - doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\xZZbar'", + doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\xZZbar'", "Py:FSTRING_START", "Py:FSTRING_TEXT", "INVALID_UNICODE_ESCAPE_TOKEN", "Py:FSTRING_TEXT", "Py:FSTRING_END"); doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\10'", "Py:FSTRING_START", "Py:FSTRING_TEXT", "VALID_STRING_ESCAPE_TOKEN", "Py:FSTRING_END"); doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\777'", "Py:FSTRING_START", "Py:FSTRING_TEXT", "VALID_STRING_ESCAPE_TOKEN", "Py:FSTRING_TEXT", "Py:FSTRING_END"); - - doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\N{GREEK SMALL LETTER ALPHA'", + + doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\N{GREEK SMALL LETTER ALPHA'", "Py:FSTRING_START", "Py:FSTRING_TEXT", "INVALID_UNICODE_ESCAPE_TOKEN", "Py:FSTRING_END"); + doTestStringHighlighting(LanguageLevel.PYTHON36, "f'{x:\\n}'", + "Py:FSTRING_START", "Py:FSTRING_FRAGMENT_START", "Py:IDENTIFIER", "Py:FSTRING_FRAGMENT_FORMAT_START", + "VALID_STRING_ESCAPE_TOKEN", "Py:FSTRING_FRAGMENT_END", "Py:FSTRING_END"); + } + + // PY-32123 + public void testRawFStringEscapeSequences() { + doTestStringHighlighting(LanguageLevel.PYTHON36, "rf'foo\\nbar'", + "Py:FSTRING_START", "Py:FSTRING_RAW_TEXT", "Py:FSTRING_RAW_TEXT", "Py:FSTRING_RAW_TEXT", "Py:FSTRING_END"); + doTestStringHighlighting(LanguageLevel.PYTHON36, "rf'foo\\\nbar'", + "Py:FSTRING_START", "Py:FSTRING_RAW_TEXT", "Py:FSTRING_RAW_TEXT", "Py:FSTRING_RAW_TEXT", "Py:FSTRING_END"); + doTestStringHighlighting(LanguageLevel.PYTHON36, "rf'foo\\", + "Py:FSTRING_START", "Py:FSTRING_RAW_TEXT", "Py:FSTRING_RAW_TEXT"); + doTestStringHighlighting(LanguageLevel.PYTHON36, "rf'{x:\\n}'", + "Py:FSTRING_START", "Py:FSTRING_FRAGMENT_START", "Py:IDENTIFIER", "Py:FSTRING_FRAGMENT_FORMAT_START", + "Py:FSTRING_RAW_TEXT", "Py:FSTRING_RAW_TEXT", "Py:FSTRING_FRAGMENT_END", "Py:FSTRING_END"); + doTestStringHighlighting(LanguageLevel.PYTHON36, "rf'{f\"\\n\"}'", + "Py:FSTRING_START", "Py:FSTRING_FRAGMENT_START", + "Py:FSTRING_START", "VALID_STRING_ESCAPE_TOKEN", "Py:FSTRING_END", + "Py:FSTRING_FRAGMENT_END", "Py:FSTRING_END"); + doTestStringHighlighting(LanguageLevel.PYTHON36, "f'{rf\"\\n\"}'", + "Py:FSTRING_START", "Py:FSTRING_FRAGMENT_START", + "Py:FSTRING_START", "Py:FSTRING_RAW_TEXT", "Py:FSTRING_RAW_TEXT", "Py:FSTRING_END", + "Py:FSTRING_FRAGMENT_END", "Py:FSTRING_END"); } private static void doTest(LanguageLevel languageLevel, String text, String... expectedTokens) { diff --git a/python/testSrc/com/jetbrains/python/parsing/PythonParsingTest.java b/python/testSrc/com/jetbrains/python/parsing/PythonParsingTest.java index b7fdf9cd40a0..da660f35b45f 100644 --- a/python/testSrc/com/jetbrains/python/parsing/PythonParsingTest.java +++ b/python/testSrc/com/jetbrains/python/parsing/PythonParsingTest.java @@ -848,6 +848,26 @@ public class PythonParsingTest extends ParsingTestCase { doTest(LanguageLevel.PYTHON36); } + // PY-32123 + public void testFStringRawFStringInsidePlainFString() { + doTest(LanguageLevel.PYTHON36); + } + + // PY-32123 + public void testFStringPlainInsideRawFString() { + doTest(LanguageLevel.PYTHON36); + } + + // PY-32123 + public void testFStringEscapeInFormatPartOfRawLiteral() { + doTest(LanguageLevel.PYTHON36); + } + + // PY-32123 + public void testFStringEscapeInFormatPartOfPlainLiteral() { + doTest(LanguageLevel.PYTHON36); + } + // PY-19036 public void testAwaitInNonAsyncNestedFunction() { doTest(LanguageLevel.PYTHON35);