From b3351b941e57c51eca220e5fc0def6cfc848d318 Mon Sep 17 00:00:00 2001 From: Mikhail Golubev Date: Mon, 25 Sep 2023 23:39:24 +0300 Subject: [PATCH] PY-59594 PEP 701: Stop recovery of incomplete f-string fragments at statement-only keywords (cherry picked from commit 7aa912a43d63f6c9867d772e120c5b45adb10be3) IJ-MR-115763 GitOrigin-RevId: ca6909a8e7a59e06eae69b631160e4d48541ce58 --- .../lexer/PythonIndentingProcessor.java | 6 ++- ...ntRecoveryStoppedAtStatementOnlyKeyword.py | 3 ++ ...tRecoveryStoppedAtStatementOnlyKeyword.txt | 39 +++++++++++++++++ ...ntRecoveryStoppedAtStatementOnlyKeyword.py | 3 ++ ...tRecoveryStoppedAtStatementOnlyKeyword.txt | 33 +++++++++++++++ ...ntRecoveryStoppedAtStatementOnlyKeyword.py | 3 ++ ...tRecoveryStoppedAtStatementOnlyKeyword.txt | 42 +++++++++++++++++++ .../python/parsing/PythonParsingTest.java | 12 ++++++ 8 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 python/testData/psi/FormatPartFStringFragmentRecoveryStoppedAtStatementOnlyKeyword.py create mode 100644 python/testData/psi/FormatPartFStringFragmentRecoveryStoppedAtStatementOnlyKeyword.txt create mode 100644 python/testData/psi/IncompleteFStringFragmentRecoveryStoppedAtStatementOnlyKeyword.py create mode 100644 python/testData/psi/IncompleteFStringFragmentRecoveryStoppedAtStatementOnlyKeyword.txt create mode 100644 python/testData/psi/NestedIncompleteFStringFragmentRecoveryStoppedAtStatementOnlyKeyword.py create mode 100644 python/testData/psi/NestedIncompleteFStringFragmentRecoveryStoppedAtStatementOnlyKeyword.txt diff --git a/python/python-psi-impl/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java b/python/python-psi-impl/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java index a8d1c8a10e59..57d86d3f830b 100644 --- a/python/python-psi-impl/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java +++ b/python/python-psi-impl/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java @@ -259,6 +259,7 @@ public class PythonIndentingProcessor extends MergingLexerAdapter { } private void adjustBraceLevel() { + boolean insideFStringFragment = !myFStringStack.isEmpty() && !myFStringStack.peek().fragments.isEmpty(); final IElementType tokenType = getTokenType(); if (PyTokenTypes.OPEN_BRACES.contains(tokenType)) { myBraceLevel++; @@ -266,8 +267,11 @@ public class PythonIndentingProcessor extends MergingLexerAdapter { else if (PyTokenTypes.CLOSE_BRACES.contains(tokenType)) { myBraceLevel--; } - else if (myBraceLevel != 0 && RECOVERY_TOKENS.contains(tokenType)) { + else if ((myBraceLevel != 0 || insideFStringFragment) && RECOVERY_TOKENS.contains(tokenType)) { myBraceLevel = 0; + if (insideFStringFragment) { + myFStringStack.clear(); + } final int pos = getTokenStart(); pushToken(PyTokenTypes.STATEMENT_BREAK, pos, pos); final int indents = myIndentStack.size(); diff --git a/python/testData/psi/FormatPartFStringFragmentRecoveryStoppedAtStatementOnlyKeyword.py b/python/testData/psi/FormatPartFStringFragmentRecoveryStoppedAtStatementOnlyKeyword.py new file mode 100644 index 000000000000..7ac45e68d171 --- /dev/null +++ b/python/testData/psi/FormatPartFStringFragmentRecoveryStoppedAtStatementOnlyKeyword.py @@ -0,0 +1,3 @@ +s = f'{foo:{bar' +print(42) +raise Exception diff --git a/python/testData/psi/FormatPartFStringFragmentRecoveryStoppedAtStatementOnlyKeyword.txt b/python/testData/psi/FormatPartFStringFragmentRecoveryStoppedAtStatementOnlyKeyword.txt new file mode 100644 index 000000000000..075c31d01f94 --- /dev/null +++ b/python/testData/psi/FormatPartFStringFragmentRecoveryStoppedAtStatementOnlyKeyword.txt @@ -0,0 +1,39 @@ +PyFile:FormatPartFStringFragmentRecoveryStoppedAtStatementOnlyKeyword.py + PyAssignmentStatement + PyTargetExpression: s + PsiElement(Py:IDENTIFIER)('s') + PsiWhiteSpace(' ') + PsiElement(Py:EQ)('=') + PsiWhiteSpace(' ') + PyStringLiteralExpression: {foo:{bar' +print(42) + + PyFormattedStringElement + PsiElement(Py:FSTRING_START)('f'') + PyFStringFragment + PsiElement(Py:FSTRING_FRAGMENT_START)('{') + PyReferenceExpression: foo + PsiElement(Py:IDENTIFIER)('foo') + PyFStringFragmentFormatPart + PsiElement(Py:FSTRING_FRAGMENT_FORMAT_START)(':') + PyFStringFragment + PsiElement(Py:FSTRING_FRAGMENT_START)('{') + PyReferenceExpression: bar + PsiElement(Py:IDENTIFIER)('bar') + PsiErrorElement:Unexpected expression part + PsiElement(Py:SINGLE_QUOTED_STRING)(''') + PsiWhiteSpace('\n') + PsiElement(Py:IDENTIFIER)('print') + PsiElement(Py:LPAR)('(') + PsiElement(Py:INTEGER_LITERAL)('42') + PsiElement(Py:RPAR)(')') + PsiWhiteSpace('\n') + PsiErrorElement:Type conversion, ':' or '}' expected + + PsiErrorElement:' expected + + PyRaiseStatement + PsiElement(Py:RAISE_KEYWORD)('raise') + PsiWhiteSpace(' ') + PyReferenceExpression: Exception + PsiElement(Py:IDENTIFIER)('Exception') \ No newline at end of file diff --git a/python/testData/psi/IncompleteFStringFragmentRecoveryStoppedAtStatementOnlyKeyword.py b/python/testData/psi/IncompleteFStringFragmentRecoveryStoppedAtStatementOnlyKeyword.py new file mode 100644 index 000000000000..b6cddc269bb6 --- /dev/null +++ b/python/testData/psi/IncompleteFStringFragmentRecoveryStoppedAtStatementOnlyKeyword.py @@ -0,0 +1,3 @@ +s = f'{foo' +print(42) +raise Exception \ No newline at end of file diff --git a/python/testData/psi/IncompleteFStringFragmentRecoveryStoppedAtStatementOnlyKeyword.txt b/python/testData/psi/IncompleteFStringFragmentRecoveryStoppedAtStatementOnlyKeyword.txt new file mode 100644 index 000000000000..24a40777a14c --- /dev/null +++ b/python/testData/psi/IncompleteFStringFragmentRecoveryStoppedAtStatementOnlyKeyword.txt @@ -0,0 +1,33 @@ +PyFile:IncompleteFStringFragmentRecoveryStoppedAtStatementOnlyKeyword.py + PyAssignmentStatement + PyTargetExpression: s + PsiElement(Py:IDENTIFIER)('s') + PsiWhiteSpace(' ') + PsiElement(Py:EQ)('=') + PsiWhiteSpace(' ') + PyStringLiteralExpression: {foo' +print(42) + + PyFormattedStringElement + PsiElement(Py:FSTRING_START)('f'') + PyFStringFragment + PsiElement(Py:FSTRING_FRAGMENT_START)('{') + PyReferenceExpression: foo + PsiElement(Py:IDENTIFIER)('foo') + PsiErrorElement:Unexpected expression part + PsiElement(Py:SINGLE_QUOTED_STRING)(''') + PsiWhiteSpace('\n') + PsiElement(Py:IDENTIFIER)('print') + PsiElement(Py:LPAR)('(') + PsiElement(Py:INTEGER_LITERAL)('42') + PsiElement(Py:RPAR)(')') + PsiWhiteSpace('\n') + PsiErrorElement:Type conversion, ':' or '}' expected + + PsiErrorElement:' expected + + PyRaiseStatement + PsiElement(Py:RAISE_KEYWORD)('raise') + PsiWhiteSpace(' ') + PyReferenceExpression: Exception + PsiElement(Py:IDENTIFIER)('Exception') \ No newline at end of file diff --git a/python/testData/psi/NestedIncompleteFStringFragmentRecoveryStoppedAtStatementOnlyKeyword.py b/python/testData/psi/NestedIncompleteFStringFragmentRecoveryStoppedAtStatementOnlyKeyword.py new file mode 100644 index 000000000000..10c444941f94 --- /dev/null +++ b/python/testData/psi/NestedIncompleteFStringFragmentRecoveryStoppedAtStatementOnlyKeyword.py @@ -0,0 +1,3 @@ +s = f'{f"{foo"' +print(42) +raise Exception \ No newline at end of file diff --git a/python/testData/psi/NestedIncompleteFStringFragmentRecoveryStoppedAtStatementOnlyKeyword.txt b/python/testData/psi/NestedIncompleteFStringFragmentRecoveryStoppedAtStatementOnlyKeyword.txt new file mode 100644 index 000000000000..f19dd4924288 --- /dev/null +++ b/python/testData/psi/NestedIncompleteFStringFragmentRecoveryStoppedAtStatementOnlyKeyword.txt @@ -0,0 +1,42 @@ +PyFile:NestedIncompleteFStringFragmentRecoveryStoppedAtStatementOnlyKeyword.py + PyAssignmentStatement + PyTargetExpression: s + PsiElement(Py:IDENTIFIER)('s') + PsiWhiteSpace(' ') + PsiElement(Py:EQ)('=') + PsiWhiteSpace(' ') + PyStringLiteralExpression: {f"{foo"' +print(42) + + PyFormattedStringElement + PsiElement(Py:FSTRING_START)('f'') + PyFStringFragment + PsiElement(Py:FSTRING_FRAGMENT_START)('{') + PyStringLiteralExpression: {foo"' +print(42) + + PyFormattedStringElement + PsiElement(Py:FSTRING_START)('f"') + PyFStringFragment + PsiElement(Py:FSTRING_FRAGMENT_START)('{') + PyReferenceExpression: foo + PsiElement(Py:IDENTIFIER)('foo') + PsiErrorElement:Unexpected expression part + PsiElement(Py:SINGLE_QUOTED_STRING)('"'') + PsiWhiteSpace('\n') + PsiElement(Py:IDENTIFIER)('print') + PsiElement(Py:LPAR)('(') + PsiElement(Py:INTEGER_LITERAL)('42') + PsiElement(Py:RPAR)(')') + PsiWhiteSpace('\n') + PsiErrorElement:Type conversion, ':' or '}' expected + + PsiErrorElement:" expected + + PsiErrorElement:' expected + + PyRaiseStatement + PsiElement(Py:RAISE_KEYWORD)('raise') + PsiWhiteSpace(' ') + PyReferenceExpression: Exception + PsiElement(Py:IDENTIFIER)('Exception') \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/python/parsing/PythonParsingTest.java b/python/testSrc/com/jetbrains/python/parsing/PythonParsingTest.java index c9481c327ffd..674fa0f5956b 100644 --- a/python/testSrc/com/jetbrains/python/parsing/PythonParsingTest.java +++ b/python/testSrc/com/jetbrains/python/parsing/PythonParsingTest.java @@ -861,6 +861,18 @@ public class PythonParsingTest extends ParsingTestCase { doTest(LanguageLevel.PYTHON36); } + public void testIncompleteFStringFragmentRecoveryStoppedAtStatementOnlyKeyword() { + doTest(LanguageLevel.PYTHON36); + } + + public void testNestedIncompleteFStringFragmentRecoveryStoppedAtStatementOnlyKeyword() { + doTest(LanguageLevel.PYTHON36); + } + + public void testFormatPartFStringFragmentRecoveryStoppedAtStatementOnlyKeyword() { + doTest(LanguageLevel.PYTHON36); + } + // PY-19036 public void testAwaitInNonAsyncNestedFunction() { doTest(LanguageLevel.PYTHON35);