PY-31274 IPython magic is parsed and highlighted with no errors in Jupyter cells

Separate dialect for IPython is created to redefine parser and lexer in Jupyter cells. Dollar sign added to console
symbols since it can be used in shell commands. Additional cell type for cell magic introduced.

python/src/com/jetbrains/python/console/parsing/PyConsoleParsingContext.java

@@ -62,7 +62,7 @@ public class PyConsoleParsingContext extends ParsingContext {
       if (parseIPythonHelp()) {
         return;
       }
-      if (myStartsWithIPythonSymbol) {
+      if (shouldParseIPythonCommand()) {
         parseIPythonCommand();
       }
       else {
@@ -95,8 +95,8 @@ public class PyConsoleParsingContext extends ParsingContext {
       if (myBuilder.getTokenType() == PyConsoleTokenTypes.QUESTION_MARK) {
         myBuilder.advanceLexer();
         if (myBuilder.getTokenType() == PyTokenTypes.STATEMENT_BREAK || myBuilder.eof()) {
-          ipythonHelp.done(PyElementTypes.EMPTY_EXPRESSION);
           myBuilder.advanceLexer();
+          ipythonHelp.done(PyElementTypes.EMPTY_EXPRESSION);
           return true;
         }
       }
@@ -136,9 +136,17 @@ public class PyConsoleParsingContext extends ParsingContext {
       return true;
     }
 
+    protected boolean shouldParseIPythonCommand() {
+      return myStartsWithIPythonSymbol;
+    }
+
+    protected boolean continueParseIPythonCommand() {
+      return !myBuilder.eof();
+    }
+
     private void parseIPythonCommand() {
       PsiBuilder.Marker ipythonCommand = myBuilder.mark();
-      while (!myBuilder.eof()) {
+      while (continueParseIPythonCommand()) {
         myBuilder.advanceLexer();
       }
       ipythonCommand.done(PyElementTypes.EMPTY_EXPRESSION);

python/src/com/jetbrains/python/console/parsing/PyConsoleTokenTypes.java

@@ -23,6 +23,7 @@ import com.jetbrains.python.psi.PyElementType;
 public class PyConsoleTokenTypes {
   public static final PyElementType QUESTION_MARK = new PyElementType("QUESTION_MARK"); //?
   public static final PyElementType PLING = new PyElementType("PLING"); //!
+  public static final PyElementType DOLLAR = new PyElementType("DOLLAR"); //$
 
   private PyConsoleTokenTypes() {
   }

python/src/com/jetbrains/python/console/parsing/PythonConsoleLexer.java

@@ -27,13 +27,16 @@ import java.util.Map;
  * @author traff
  */
 public class PythonConsoleLexer extends PythonIndentingLexer {
-  private final static Map<String, PyElementType> SPECIAL_IPYTHON_SYMBOLS = ImmutableMap.of("?", PyConsoleTokenTypes
-    .QUESTION_MARK, "!", PyConsoleTokenTypes.PLING);
+  private final static Map<String, PyElementType> SPECIAL_IPYTHON_SYMBOLS = ImmutableMap.of(
+    "?", PyConsoleTokenTypes.QUESTION_MARK,
+    "!", PyConsoleTokenTypes.PLING,
+    "$", PyConsoleTokenTypes.DOLLAR
+  );
 
-  @Override
   /**
    * Treats special symbols used in IPython console
    */
+  @Override
   public IElementType getTokenType() {
     IElementType type = super.getTokenType();
     if (type == PyTokenTypes.BAD_CHARACTER && isSpecialSymbols(getTokenText())) {

python/src/com/jetbrains/python/highlighting/PySyntaxHighlighterFactory.java

@@ -35,15 +35,22 @@ public class PySyntaxHighlighterFactory extends SyntaxHighlighterFactory {
   @Override
   @NotNull
   public SyntaxHighlighter getSyntaxHighlighter(@Nullable final Project project, @Nullable final VirtualFile virtualFile) {
-    final LanguageLevel level = project != null && virtualFile != null ?
-                                PyUtil.getLanguageLevelForVirtualFile(project, virtualFile) :
-                                LanguageLevel.getDefault();
+    final LanguageLevel level = getLanguageLevel(project, virtualFile);
     if (useConsoleLexer(project, virtualFile)) {
       return myConsoleMap.get(level);
     }
     return getSyntaxHighlighterForLanguageLevel(level);
   }
 
+  /**
+   * Returns a syntax highlighter for Python console.
+   */
+  @NotNull
+  public SyntaxHighlighter getConsoleSyntaxHighlighter(@Nullable final Project project, @Nullable final VirtualFile virtualFile) {
+    final LanguageLevel level = getLanguageLevel(project, virtualFile);
+    return myConsoleMap.get(level);
+  }
+
   /**
    * Returns a syntax highlighter targeting the specified version of Python.
    */
@@ -52,6 +59,13 @@ public class PySyntaxHighlighterFactory extends SyntaxHighlighterFactory {
     return myMap.get(level);
   }
 
+  @NotNull
+  private static LanguageLevel getLanguageLevel(@Nullable final Project project, @Nullable final VirtualFile virtualFile) {
+    return project != null && virtualFile != null ?
+           PyUtil.getLanguageLevelForVirtualFile(project, virtualFile) :
+           LanguageLevel.getDefault();
+  }
+
   private static boolean useConsoleLexer(@Nullable final Project project, @Nullable final VirtualFile virtualFile) {
     if (virtualFile == null || project == null || virtualFile instanceof VirtualFileWindow) {
       return false;

python/testData/console/ipython/psi/magicMultiline.txt

@@ -8,7 +8,7 @@ PyFile:magicMultiline.py
     PsiWhiteSpace(' ')
     PsiElement(Py:SINGLE_QUOTED_STRING)('"My shell is:"')
     PsiWhiteSpace(' ')
-    PsiElement(BAD_CHARACTER)('$')
+    PsiElement(Py:DOLLAR)('$')
     PsiElement(Py:IDENTIFIER)('SHELL')
     PsiWhiteSpace('\n')
     PsiElement(Py:IDENTIFIER)('echo')

python/testData/console/ipython/psi/shellExpansion.txt

@@ -3,7 +3,7 @@ PyFile:shellExpansion.py
     PsiElement(Py:PLING)('!')
     PsiElement(Py:IDENTIFIER)('mv')
     PsiWhiteSpace(' ')
-    PsiElement(BAD_CHARACTER)('$')
+    PsiElement(Py:DOLLAR)('$')
     PsiElement(Py:IDENTIFIER)('file')
     PsiWhiteSpace(' ')
     PsiElement(Py:LBRACE)('{')