MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-01-08 15:09:39 +07:00
Code Issues Packages Projects Releases Wiki Activity

PY-26110: Syntax highlighting changes from docstring to string if comment is added to the last line

Browse Source 
- change lexer to support docstrings with trailing comments
- small syntax clean-up
- prevent NPE in PyTestCase#tearDown()
- add some tests for PY-40634

GitOrigin-RevId: 949617a518f8938f557106bd2a4a589bdbb1f542
This commit is contained in:
Marcus Mews
2025-06-26 21:59:49 +00:00
committed by intellij-monorepo-bot
parent 2b12a054a2
commit c4394a0e40
6 changed files with 908 additions and 583 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1304
python/python-parser/gen/com/jetbrains/python/lexer/_PythonLexer.java
View File

File diff suppressed because it is too large Load Diff

47
python/python-parser/src/com/jetbrains/python/lexer/Python.flex
View File

@@ -105,14 +105,6 @@ FSTRING_FRAGMENT_TYPE_CONVERSION = "!" [^=:'\"} \t\r\n]*
%xstate FSTRING_FRAGMENT_FORMAT
%{
private final PyLexerFStringHelper fStringHelper = new PyLexerFStringHelper(this);
private int getSpaceLength(CharSequence string) {
String string1 = string.toString();
string1 = StringUtil.trimEnd(string1, "\\");
string1 = StringUtil.trimEnd(string1, ";");
final String s = StringUtil.trimTrailing(string1);
return yylength() - s.length();
}
%}
%%
@@ -182,28 +174,27 @@ private int getSpaceLength(CharSequence string) {
[\f] { return PyTokenTypes.FORMFEED; }
"\\" { return PyTokenTypes.BACKSLASH; }
<YYINITIAL> {
[\n] { if (zzCurrentPos == 0 && !isConsole()) yybegin(PENDING_DOCSTRING); return PyTokenTypes.LINE_BREAK; }
{END_OF_LINE_COMMENT} { if (zzCurrentPos == 0 && !isConsole()) yybegin(PENDING_DOCSTRING); return PyTokenTypes.END_OF_LINE_COMMENT; }
{SINGLE_QUOTED_STRING} { if (zzInput == YYEOF && zzStartRead == 0 && !isConsole()) return PyTokenTypes.DOCSTRING;
else return PyTokenTypes.SINGLE_QUOTED_STRING; }
else return PyTokenTypes.SINGLE_QUOTED_STRING; }
{TRIPLE_QUOTED_STRING} { if (zzInput == YYEOF && zzStartRead == 0 && !isConsole()) return PyTokenTypes.DOCSTRING;
else return PyTokenTypes.TRIPLE_QUOTED_STRING; }
else return PyTokenTypes.TRIPLE_QUOTED_STRING; }
{SINGLE_QUOTED_STRING}[\ \t]*[\n;] { yypushback(getSpaceLength(yytext())); if (zzCurrentPos != 0 || isConsole()) return PyTokenTypes.SINGLE_QUOTED_STRING;
return PyTokenTypes.DOCSTRING; }
{SINGLE_QUOTED_STRING} / [\ \t]*[\n;#] { if (zzCurrentPos != 0 || isConsole()) return PyTokenTypes.SINGLE_QUOTED_STRING;
return PyTokenTypes.DOCSTRING; }
{TRIPLE_QUOTED_STRING}[\ \t]*[\n;] { yypushback(getSpaceLength(yytext())); if (zzCurrentPos != 0 || isConsole()) return PyTokenTypes.TRIPLE_QUOTED_STRING;
return PyTokenTypes.DOCSTRING; }
{TRIPLE_QUOTED_STRING} / [\ \t]*[\n;#] { if (zzCurrentPos != 0 || isConsole()) return PyTokenTypes.TRIPLE_QUOTED_STRING;
return PyTokenTypes.DOCSTRING; }
{SINGLE_QUOTED_STRING}[\ \t]*"\\" {
yypushback(getSpaceLength(yytext())); if (zzCurrentPos != 0 || isConsole()) return PyTokenTypes.SINGLE_QUOTED_STRING;
yybegin(PENDING_DOCSTRING); return PyTokenTypes.DOCSTRING; }
{SINGLE_QUOTED_STRING} / [\ \t]*"\\" { if (zzCurrentPos != 0 || isConsole()) return PyTokenTypes.SINGLE_QUOTED_STRING;
yybegin(PENDING_DOCSTRING); return PyTokenTypes.DOCSTRING; }
{TRIPLE_QUOTED_STRING}[\ \t]*"\\" {
yypushback(getSpaceLength(yytext())); if (zzCurrentPos != 0 || isConsole()) return PyTokenTypes.TRIPLE_QUOTED_STRING;
yybegin(PENDING_DOCSTRING); return PyTokenTypes.DOCSTRING; }
{TRIPLE_QUOTED_STRING} / [\ \t]*"\\" { if (zzCurrentPos != 0 || isConsole()) return PyTokenTypes.TRIPLE_QUOTED_STRING;
yybegin(PENDING_DOCSTRING); return PyTokenTypes.DOCSTRING; }
}
@@ -305,15 +296,15 @@ return PyTokenTypes.DOCSTRING; }
}
<IN_DOCSTRING_OWNER> {
":"(\ )*{END_OF_LINE_COMMENT}?"\n" { yypushback(yylength()-1); yybegin(PENDING_DOCSTRING); return PyTokenTypes.COLON; }
":"(\ )*{END_OF_LINE_COMMENT}?"\n" { yypushback(yylength()-1); yybegin(PENDING_DOCSTRING); return PyTokenTypes.COLON; }
}
<PENDING_DOCSTRING> {
{SINGLE_QUOTED_STRING} { if (zzInput == YYEOF) return PyTokenTypes.DOCSTRING;
else yybegin(YYINITIAL); return PyTokenTypes.SINGLE_QUOTED_STRING; }
{TRIPLE_QUOTED_STRING} { if (zzInput == YYEOF) return PyTokenTypes.DOCSTRING;
else yybegin(YYINITIAL); return PyTokenTypes.TRIPLE_QUOTED_STRING; }
{DOCSTRING_LITERAL}[\ \t]*[\n;] { yypushback(getSpaceLength(yytext())); yybegin(YYINITIAL); return PyTokenTypes.DOCSTRING; }
{DOCSTRING_LITERAL}[\ \t]*"\\" { yypushback(getSpaceLength(yytext())); return PyTokenTypes.DOCSTRING; }
. { yypushback(1); yybegin(YYINITIAL); }
{SINGLE_QUOTED_STRING} { if (zzInput == YYEOF) return PyTokenTypes.DOCSTRING;
else yybegin(YYINITIAL); return PyTokenTypes.SINGLE_QUOTED_STRING; }
{TRIPLE_QUOTED_STRING} { if (zzInput == YYEOF) return PyTokenTypes.DOCSTRING;
else yybegin(YYINITIAL); return PyTokenTypes.TRIPLE_QUOTED_STRING; }
{DOCSTRING_LITERAL} / [\ \t]*[\n;#] { yybegin(YYINITIAL); return PyTokenTypes.DOCSTRING; }
{DOCSTRING_LITERAL} / [\ \t]*"\\" { return PyTokenTypes.DOCSTRING; }
. { yypushback(1); yybegin(YYINITIAL); }
}

106
python/testSrc/com/jetbrains/python/PythonHighlightingLexerTest.java
View File

@@ -189,6 +189,14 @@ public class PythonHighlightingLexerTest extends PyLexerTestCase {
"Py:DOCSTRING", "Py:LINE_BREAK");
}
public void testMetaClass() {
doTest(LanguageLevel.getLatest(), """
class IOBase(metaclass=abc.ABCMeta):
pass""",
"Py:CLASS_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:LPAR", "Py:IDENTIFIER", "Py:EQ", "Py:IDENTIFIER", "Py:DOT", "Py:IDENTIFIER", "Py:RPAR", "Py:COLON", "Py:LINE_BREAK",
"Py:SPACE", "Py:SPACE", "Py:PASS_KEYWORD");
}
public void testSingleDocStringWithBackslash() {
doTest(LanguageLevel.PYTHON27, "\"one docstring \" \\\n\"new line of docstring\"\n",
"Py:DOCSTRING", "Py:SPACE", "Py:BACKSLASH", "Py:LINE_BREAK", "Py:DOCSTRING", "Py:LINE_BREAK");
@@ -215,6 +223,104 @@ public class PythonHighlightingLexerTest extends PyLexerTestCase {
"Py:COLON", "Py:SPACE", "Py:SINGLE_QUOTED_STRING", "Py:LINE_BREAK", "Py:SPACE", "Py:RBRACE");
}
public void testDocstringAtModule() {
doTest(LanguageLevel.getLatest(), """
""\" module docstring ""\"
""",
"Py:DOCSTRING", "Py:LINE_BREAK");
}
public void testDocstringAtModuleWithTrailingComment() {
doTest(LanguageLevel.getLatest(), """
""\" module docstring ""\" # trailing comment
""",
"Py:DOCSTRING", "Py:SPACE", "Py:END_OF_LINE_COMMENT", "Py:LINE_BREAK");
}
public void testDocstringAtClass() {
doTest(LanguageLevel.getLatest(), """
class C:
""\" class docstring ""\"
""",
"Py:CLASS_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:COLON", "Py:LINE_BREAK",
"Py:SPACE", "Py:SPACE", "Py:SPACE", "Py:SPACE", "Py:DOCSTRING", "Py:LINE_BREAK");
}
public void testDocstringAtClassWithTrailingComment() {
doTest(LanguageLevel.getLatest(), """
class C:
""\" class docstring ""\" # trailing comment
""",
"Py:CLASS_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:COLON", "Py:LINE_BREAK",
"Py:SPACE", "Py:SPACE", "Py:SPACE", "Py:SPACE", "Py:DOCSTRING", "Py:SPACE", "Py:END_OF_LINE_COMMENT", "Py:LINE_BREAK");
}
public void testDocstringAtFunction() {
doTest(LanguageLevel.getLatest(), """
def fun():
""\" function docstring ""\"
""",
"Py:DEF_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:LPAR", "Py:RPAR", "Py:COLON", "Py:LINE_BREAK",
"Py:SPACE", "Py:SPACE", "Py:SPACE", "Py:SPACE", "Py:DOCSTRING", "Py:LINE_BREAK");
}
public void testDocstringAtFunctionWithTrailingComment() {
doTest(LanguageLevel.getLatest(), """
def fun():
""\" function docstring ""\" # trailing comment
""",
"Py:DEF_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:LPAR", "Py:RPAR", "Py:COLON", "Py:LINE_BREAK",
"Py:SPACE", "Py:SPACE", "Py:SPACE", "Py:SPACE", "Py:DOCSTRING", "Py:SPACE", "Py:END_OF_LINE_COMMENT", "Py:LINE_BREAK");
}
// PY-40634
public void testDocstringAtVariableDeclaration() {
fixme("PY-40634", () -> doTest(LanguageLevel.getLatest(), """
VAR = 2
""\" variable declaration docstring ""\"
""",
"Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:INTEGER_LITERAL", "Py:LINE_BREAK",
"Py:SPACE", "Py:SPACE", "Py:SPACE", "Py:SPACE", "Py:DOCSTRING", "Py:LINE_BREAK"));
}
// PY-40634
public void testDocstringAtVariableDeclarationWithTrailingComment() {
fixme("PY-40634", () -> doTest(LanguageLevel.getLatest(), """
VAR = 2
""\" variable declaration docstring ""\" # trailing comment
""",
"Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:INTEGER_LITERAL", "Py:LINE_BREAK",
"Py:SPACE", "Py:SPACE", "Py:SPACE", "Py:SPACE", "Py:DOCSTRING", "Py:SPACE", "Py:END_OF_LINE_COMMENT", "Py:LINE_BREAK"));
}
// PY-40634
public void testDocstringAtClassVariableDeclaration() {
fixme("PY-40634", () -> doTest(LanguageLevel.getLatest(), """
class C:
def __init__(self):
self.thing = 42
""\" class variable declaration docstring ""\"
""",
"Py:CLASS_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:COLON", "Py:LINE_BREAK",
"Py:SPACE", "Py:SPACE", "Py:DEF_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:LPAR", "Py:IDENTIFIER", "Py:RPAR", "Py:COLON", "Py:LINE_BREAK",
"Py:SPACE", "Py:SPACE", "Py:SPACE", "Py:SPACE", "Py:SPACE", "Py:SPACE", "Py:IDENTIFIER", "Py:DOT", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:INTEGER_LITERAL", "Py:LINE_BREAK",
"Py:SPACE", "Py:SPACE", "Py:SPACE", "Py:SPACE", "Py:SPACE", "Py:SPACE", "Py:DOCSTRING", "Py:LINE_BREAK"));
}
// PY-40634
public void testDocstringAtClassVariableDeclarationWithTrailingComment() {
fixme("PY-40634", () -> doTest(LanguageLevel.getLatest(), """
class C:
def __init__(self):
self.thing = 42
""\" class variable declaration docstring ""\" # trailing comment
""",
"Py:CLASS_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:COLON", "Py:LINE_BREAK",
"Py:SPACE", "Py:SPACE", "Py:DEF_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:LPAR", "Py:IDENTIFIER", "Py:RPAR", "Py:COLON", "Py:LINE_BREAK",
"Py:SPACE", "Py:SPACE", "Py:SPACE", "Py:SPACE", "Py:SPACE", "Py:SPACE", "Py:IDENTIFIER", "Py:DOT", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:INTEGER_LITERAL", "Py:LINE_BREAK",
"Py:SPACE", "Py:SPACE", "Py:SPACE", "Py:SPACE", "Py:SPACE", "Py:SPACE", "Py:DOCSTRING", "Py:SPACE", "Py:END_OF_LINE_COMMENT", "Py:LINE_BREAK"));
}
// PY-29665
public void testRawBytesLiteral() {
doTest(LanguageLevel.PYTHON27, "expr = br'raw bytes'", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_STRING");

16
python/testSrc/com/jetbrains/python/PythonLexerTest.java
View File

@@ -466,44 +466,44 @@ public class PythonLexerTest extends PyLexerTestCase {
public void testTripleSingleQuotedStringWithEscapedSlashAfterOneQuote() {
doTest("""
s = '''
'\\\\'''
'\\'''
'''
""",
"Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:TRIPLE_QUOTED_STRING",
"Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:TRIPLE_QUOTED_STRING", "Py:STATEMENT_BREAK");
"Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:STATEMENT_BREAK");
}
// PY-21697
public void testTripleSingleQuotedStringWithEscapedSlashAfterTwoQuotes() {
doTest("""
s = '''
''\\\\'''
''\\'''
'''
""",
"Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:TRIPLE_QUOTED_STRING",
"Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:TRIPLE_QUOTED_STRING", "Py:STATEMENT_BREAK");
"Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:STATEMENT_BREAK");
}
// PY-21697
public void testTripleDoubleQuotedStringWithEscapedSlashAfterOneQuote() {
doTest("""
s = ""\"
"\\\\""\"
"\\""\"
""\"
""",
"Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:TRIPLE_QUOTED_STRING",
"Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:TRIPLE_QUOTED_STRING", "Py:STATEMENT_BREAK");
"Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:STATEMENT_BREAK");
}
// PY-21697
public void testTripleDoubleQuotedStringWithEscapedSlashAfterTwoQuotes() {
doTest("""
s = ""\"
""\\\\""\"
""\\""\"
""\"
""",
"Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:TRIPLE_QUOTED_STRING",
"Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:TRIPLE_QUOTED_STRING", "Py:STATEMENT_BREAK");
"Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:STATEMENT_BREAK");
}
// PY-40757

14
python/testSrc/com/jetbrains/python/fixtures/PyLexerTestCase.java
View File

@@ -7,7 +7,9 @@ import com.intellij.psi.tree.IElementType;
import com.intellij.testFramework.PlatformLiteFixture;
import com.jetbrains.python.PythonDialectsTokenSetContributor;
import com.jetbrains.python.PythonTokenSetContributor;
import junit.framework.AssertionFailedError;
import one.util.streamex.StreamEx;
import org.jetbrains.annotations.NotNull;
import java.util.List;
import java.util.Objects;
@@ -40,4 +42,16 @@ public abstract class PyLexerTestCase extends PlatformLiteFixture {
String expectedTokensInCode = StringUtil.join(actualTokens, t -> '"' + t + '"', ", ");
assertEquals("Token mismatch. Actual values: " + expectedTokensInCode, List.of(expectedTokens), actualTokens);
}
public static void fixme(@NotNull String comment, @NotNull Runnable test) {
try {
test.run();
}
catch (AssertionFailedError failedError) {
// fix-me tests are supposed to fail
return;
}
// the fix-me test passed -> the bug/feature was fixed!
fail("Test (" + comment + ") FIXED!");
}
}

4
python/testSrc/com/jetbrains/python/fixtures/PyTestCase.java
View File

@@ -91,7 +91,9 @@ public abstract class PyTestCase extends UsefulTestCase {
protected void tearDown() throws Exception {
try {
if (myFixture != null) {
PyNamespacePackagesService.getInstance(myFixture.getModule()).resetAllNamespacePackages();
if (myFixture.getModule() != null) {
PyNamespacePackagesService.getInstance(myFixture.getModule()).resetAllNamespacePackages();
}
PyModuleNameCompletionContributor.ENABLED = true;
setLanguageLevel(null);