From 87968ab1a476a6f1edced8833115aae94569dc8c Mon Sep 17 00:00:00 2001 From: Anton Bragin Date: Mon, 3 Dec 2018 11:21:22 +0300 Subject: [PATCH] PY-31274 IPython magic is parsed and highlighted with no errors in Jupyter cells Separate dialect for IPython is created to redefine parser and lexer in Jupyter cells. Dollar sign added to console symbols since it can be used in shell commands. Additional cell type for cell magic introduced. --- .../parsing/PyConsoleParsingContext.java | 14 ++++++++++--- .../console/parsing/PyConsoleTokenTypes.java | 1 + .../console/parsing/PythonConsoleLexer.java | 9 ++++++--- .../PySyntaxHighlighterFactory.java | 20 ++++++++++++++++--- .../console/ipython/psi/magicMultiline.txt | 2 +- .../console/ipython/psi/shellExpansion.txt | 2 +- 6 files changed, 37 insertions(+), 11 deletions(-) diff --git a/python/src/com/jetbrains/python/console/parsing/PyConsoleParsingContext.java b/python/src/com/jetbrains/python/console/parsing/PyConsoleParsingContext.java index 78c9d24052d4..4a592836b393 100644 --- a/python/src/com/jetbrains/python/console/parsing/PyConsoleParsingContext.java +++ b/python/src/com/jetbrains/python/console/parsing/PyConsoleParsingContext.java @@ -62,7 +62,7 @@ public class PyConsoleParsingContext extends ParsingContext { if (parseIPythonHelp()) { return; } - if (myStartsWithIPythonSymbol) { + if (shouldParseIPythonCommand()) { parseIPythonCommand(); } else { @@ -95,8 +95,8 @@ public class PyConsoleParsingContext extends ParsingContext { if (myBuilder.getTokenType() == PyConsoleTokenTypes.QUESTION_MARK) { myBuilder.advanceLexer(); if (myBuilder.getTokenType() == PyTokenTypes.STATEMENT_BREAK || myBuilder.eof()) { - ipythonHelp.done(PyElementTypes.EMPTY_EXPRESSION); myBuilder.advanceLexer(); + ipythonHelp.done(PyElementTypes.EMPTY_EXPRESSION); return true; } } @@ -136,9 +136,17 @@ public class PyConsoleParsingContext extends ParsingContext { return true; } + protected boolean shouldParseIPythonCommand() { + return myStartsWithIPythonSymbol; + } + + protected boolean continueParseIPythonCommand() { + return !myBuilder.eof(); + } + private void parseIPythonCommand() { PsiBuilder.Marker ipythonCommand = myBuilder.mark(); - while (!myBuilder.eof()) { + while (continueParseIPythonCommand()) { myBuilder.advanceLexer(); } ipythonCommand.done(PyElementTypes.EMPTY_EXPRESSION); diff --git a/python/src/com/jetbrains/python/console/parsing/PyConsoleTokenTypes.java b/python/src/com/jetbrains/python/console/parsing/PyConsoleTokenTypes.java index 9af8b479fd97..7e8cbb6db8b4 100644 --- a/python/src/com/jetbrains/python/console/parsing/PyConsoleTokenTypes.java +++ b/python/src/com/jetbrains/python/console/parsing/PyConsoleTokenTypes.java @@ -23,6 +23,7 @@ import com.jetbrains.python.psi.PyElementType; public class PyConsoleTokenTypes { public static final PyElementType QUESTION_MARK = new PyElementType("QUESTION_MARK"); //? public static final PyElementType PLING = new PyElementType("PLING"); //! + public static final PyElementType DOLLAR = new PyElementType("DOLLAR"); //$ private PyConsoleTokenTypes() { } diff --git a/python/src/com/jetbrains/python/console/parsing/PythonConsoleLexer.java b/python/src/com/jetbrains/python/console/parsing/PythonConsoleLexer.java index 7269945afc0d..65effc067edb 100644 --- a/python/src/com/jetbrains/python/console/parsing/PythonConsoleLexer.java +++ b/python/src/com/jetbrains/python/console/parsing/PythonConsoleLexer.java @@ -27,13 +27,16 @@ import java.util.Map; * @author traff */ public class PythonConsoleLexer extends PythonIndentingLexer { - private final static Map SPECIAL_IPYTHON_SYMBOLS = ImmutableMap.of("?", PyConsoleTokenTypes - .QUESTION_MARK, "!", PyConsoleTokenTypes.PLING); + private final static Map SPECIAL_IPYTHON_SYMBOLS = ImmutableMap.of( + "?", PyConsoleTokenTypes.QUESTION_MARK, + "!", PyConsoleTokenTypes.PLING, + "$", PyConsoleTokenTypes.DOLLAR + ); - @Override /** * Treats special symbols used in IPython console */ + @Override public IElementType getTokenType() { IElementType type = super.getTokenType(); if (type == PyTokenTypes.BAD_CHARACTER && isSpecialSymbols(getTokenText())) { diff --git a/python/src/com/jetbrains/python/highlighting/PySyntaxHighlighterFactory.java b/python/src/com/jetbrains/python/highlighting/PySyntaxHighlighterFactory.java index 0d80d8a5f0e7..e3abd93c574b 100644 --- a/python/src/com/jetbrains/python/highlighting/PySyntaxHighlighterFactory.java +++ b/python/src/com/jetbrains/python/highlighting/PySyntaxHighlighterFactory.java @@ -35,15 +35,22 @@ public class PySyntaxHighlighterFactory extends SyntaxHighlighterFactory { @Override @NotNull public SyntaxHighlighter getSyntaxHighlighter(@Nullable final Project project, @Nullable final VirtualFile virtualFile) { - final LanguageLevel level = project != null && virtualFile != null ? - PyUtil.getLanguageLevelForVirtualFile(project, virtualFile) : - LanguageLevel.getDefault(); + final LanguageLevel level = getLanguageLevel(project, virtualFile); if (useConsoleLexer(project, virtualFile)) { return myConsoleMap.get(level); } return getSyntaxHighlighterForLanguageLevel(level); } + /** + * Returns a syntax highlighter for Python console. + */ + @NotNull + public SyntaxHighlighter getConsoleSyntaxHighlighter(@Nullable final Project project, @Nullable final VirtualFile virtualFile) { + final LanguageLevel level = getLanguageLevel(project, virtualFile); + return myConsoleMap.get(level); + } + /** * Returns a syntax highlighter targeting the specified version of Python. */ @@ -52,6 +59,13 @@ public class PySyntaxHighlighterFactory extends SyntaxHighlighterFactory { return myMap.get(level); } + @NotNull + private static LanguageLevel getLanguageLevel(@Nullable final Project project, @Nullable final VirtualFile virtualFile) { + return project != null && virtualFile != null ? + PyUtil.getLanguageLevelForVirtualFile(project, virtualFile) : + LanguageLevel.getDefault(); + } + private static boolean useConsoleLexer(@Nullable final Project project, @Nullable final VirtualFile virtualFile) { if (virtualFile == null || project == null || virtualFile instanceof VirtualFileWindow) { return false; diff --git a/python/testData/console/ipython/psi/magicMultiline.txt b/python/testData/console/ipython/psi/magicMultiline.txt index 0d36d47727c9..3f7d8a56364c 100644 --- a/python/testData/console/ipython/psi/magicMultiline.txt +++ b/python/testData/console/ipython/psi/magicMultiline.txt @@ -8,7 +8,7 @@ PyFile:magicMultiline.py PsiWhiteSpace(' ') PsiElement(Py:SINGLE_QUOTED_STRING)('"My shell is:"') PsiWhiteSpace(' ') - PsiElement(BAD_CHARACTER)('$') + PsiElement(Py:DOLLAR)('$') PsiElement(Py:IDENTIFIER)('SHELL') PsiWhiteSpace('\n') PsiElement(Py:IDENTIFIER)('echo') diff --git a/python/testData/console/ipython/psi/shellExpansion.txt b/python/testData/console/ipython/psi/shellExpansion.txt index 7254faacfae1..ef5a962c82b0 100644 --- a/python/testData/console/ipython/psi/shellExpansion.txt +++ b/python/testData/console/ipython/psi/shellExpansion.txt @@ -3,7 +3,7 @@ PyFile:shellExpansion.py PsiElement(Py:PLING)('!') PsiElement(Py:IDENTIFIER)('mv') PsiWhiteSpace(' ') - PsiElement(BAD_CHARACTER)('$') + PsiElement(Py:DOLLAR)('$') PsiElement(Py:IDENTIFIER)('file') PsiWhiteSpace(' ') PsiElement(Py:LBRACE)('{')