PY-32597 Console accommodates IPython syntax without generating parsing errors

IPython specific syntax is captured by console parser, including:
* Multiline magic
* Exclamation in suffixes
* Double exclamation
* Help syntax

Test classes for console moved to separate package.
This commit is contained in:
Anton Bragin
2018-11-30 12:06:13 +03:00
parent 2e1a0ff7f2
commit 3e3e9bb82e
55 changed files with 458 additions and 41 deletions
@@ -15,6 +15,7 @@
*/
package com.jetbrains.python.console.parsing;
import com.google.common.collect.ImmutableSet;
import com.intellij.lang.ASTNode;
import com.intellij.lang.PsiBuilder;
import com.intellij.psi.tree.IElementType;
@@ -28,11 +29,20 @@ import org.jetbrains.annotations.NotNull;
/**
* @author traff
*/
public class PyConsoleParser extends PyParser{
public class PyConsoleParser extends PyParser {
private StatementParsing.FUTURE myFutureFlag;
private final PythonConsoleData myPythonConsoleData;
private boolean myIPythonStartSymbol;
private static final ImmutableSet<IElementType> IPYTHON_START_SYMBOLS = new ImmutableSet.Builder<IElementType>().add(
PyConsoleTokenTypes.QUESTION_MARK,
PyConsoleTokenTypes.PLING,
PyTokenTypes.PERC,
PyTokenTypes.COMMA,
PyTokenTypes.SEMICOLON,
PyTokenTypes.DIV
).build();
public PyConsoleParser(PythonConsoleData pythonConsoleData, LanguageLevel languageLevel) {
myPythonConsoleData = pythonConsoleData;
myLanguageLevel = languageLevel;
@@ -59,11 +69,9 @@ public class PyConsoleParser extends PyParser{
public static boolean startsWithIPythonSpecialSymbol(PsiBuilder builder) {
IElementType tokenType = builder.getTokenType();
return builder.getTokenType() == PyConsoleTokenTypes.QUESTION_MARK || tokenType == PyTokenTypes.PERC || tokenType == PyTokenTypes.COMMA || tokenType == PyTokenTypes.SEMICOLON ||
"/".equals(builder.getTokenText());
return IPYTHON_START_SYMBOLS.contains(tokenType);
}
@Override
protected ParsingContext createParsingContext(PsiBuilder builder, LanguageLevel languageLevel, StatementParsing.FUTURE futureFlag) {
return new PyConsoleParsingContext(builder, languageLevel, futureFlag, myPythonConsoleData, myIPythonStartSymbol);
@@ -59,6 +59,9 @@ public class PyConsoleParsingContext extends ParsingContext {
@Override
public void parseStatement() {
if (parseIPythonHelp()) {
return;
}
if (myStartsWithIPythonSymbol) {
parseIPythonCommand();
}
@@ -79,6 +82,60 @@ public class PyConsoleParsingContext extends ParsingContext {
}
}
private boolean parseIPythonHelp() {
return parseIPythonGlobalHelp() ||
parseIPythonSuffixHelp();
}
/**
* Parse statements consisting of the single question mark.
*/
private boolean parseIPythonGlobalHelp() {
PsiBuilder.Marker ipythonHelp = myBuilder.mark();
if (myBuilder.getTokenType() == PyConsoleTokenTypes.QUESTION_MARK) {
myBuilder.advanceLexer();
if (myBuilder.getTokenType() == PyTokenTypes.STATEMENT_BREAK || myBuilder.eof()) {
ipythonHelp.done(PyElementTypes.EMPTY_EXPRESSION);
myBuilder.advanceLexer();
return true;
}
}
ipythonHelp.rollbackTo();
return false;
}
/**
* Parse statements ending with a question mark.
*/
private boolean parseIPythonSuffixHelp() {
PsiBuilder.Marker ipythonHelp = myBuilder.mark();
while (myBuilder.getTokenType() != PyTokenTypes.STATEMENT_BREAK &&
myBuilder.getTokenType() != PyTokenTypes.LINE_BREAK &&
!myBuilder.eof()
) {
myBuilder.advanceLexer();
}
if (myBuilder.rawLookup(-1) != PyConsoleTokenTypes.QUESTION_MARK) {
ipythonHelp.rollbackTo();
return false;
}
int lookupIndex = -2;
if (myBuilder.rawLookup(lookupIndex) == PyConsoleTokenTypes.QUESTION_MARK) {
--lookupIndex;
}
if (myBuilder.rawLookup(lookupIndex) == PyTokenTypes.MULT) {
ipythonHelp.rollbackTo();
parseIPythonCommand();
return true;
}
if (myBuilder.rawLookup(lookupIndex) != PyTokenTypes.IDENTIFIER) {
myBuilder.error("Help request must follow the name");
}
ipythonHelp.done(PyElementTypes.EMPTY_EXPRESSION);
myBuilder.advanceLexer();
return true;
}
private void parseIPythonCommand() {
PsiBuilder.Marker ipythonCommand = myBuilder.mark();
while (!myBuilder.eof()) {
@@ -107,12 +164,11 @@ public class PyConsoleParsingContext extends ParsingContext {
return;
}
else {
if (builder.getTokenType() == PyConsoleTokenTypes.PLING || builder.getTokenType() == PyConsoleTokenTypes.QUESTION_MARK) {
if (builder.getTokenType() == PyConsoleTokenTypes.QUESTION_MARK && builder.rawLookup(-1) == PyTokenTypes.IDENTIFIER) {
builder.advanceLexer();
if (builder.getTokenType() == PyConsoleTokenTypes.PLING || builder.getTokenType() == PyConsoleTokenTypes.QUESTION_MARK) {
if (builder.getTokenType() == PyConsoleTokenTypes.QUESTION_MARK) {
builder.advanceLexer();
}
return;
}
builder.error("End of statement expected");
@@ -139,6 +195,10 @@ public class PyConsoleParsingContext extends ParsingContext {
myBuilder.advanceLexer();
if (myBuilder.getTokenType() == PyConsoleTokenTypes.PLING) {
myBuilder.advanceLexer();
}
if (myBuilder.getTokenType() == PyTokenTypes.IDENTIFIER) {
myBuilder.advanceLexer();
command.done(getReferenceType());
@@ -159,5 +219,38 @@ public class PyConsoleParsingContext extends ParsingContext {
return super.parseExpressionOptional();
}
}
@Override
public boolean parseYieldOrTupleExpression(boolean isTargetExpression) {
if (parseIPythonCaptureExpression()) {
return true;
}
else {
return super.parseYieldOrTupleExpression(isTargetExpression);
}
}
private boolean parseIPythonCaptureExpression() {
if (myBuilder.getTokenType() == PyConsoleTokenTypes.PLING) {
captureIPythonExpression();
return true;
}
if (myBuilder.getTokenType() == PyTokenTypes.PERC) {
if (myBuilder.lookAhead(1) == PyTokenTypes.PERC) {
myBuilder.error("Multiline magic can't be used as an expression");
}
captureIPythonExpression();
return true;
}
return false;
}
private void captureIPythonExpression() {
PsiBuilder.Marker mark = myBuilder.mark();
while (myBuilder.getTokenType() != PyTokenTypes.STATEMENT_BREAK) {
myBuilder.advanceLexer();
}
mark.done(PyElementTypes.EMPTY_EXPRESSION);
}
}
}
@@ -0,0 +1 @@
?
@@ -0,0 +1,3 @@
PyFile:help1.py
PyEmptyExpression
PsiElement(Py:QUESTION_MARK)('?')
@@ -0,0 +1 @@
?
@@ -0,0 +1,3 @@
PyFile:help2.py
PyEmptyExpression
PsiElement(Py:QUESTION_MARK)('?')
@@ -0,0 +1,2 @@
class A:
?
@@ -0,0 +1,12 @@
PyFile:help3.py
PyClass: A
PsiElement(Py:CLASS_KEYWORD)('class')
PsiWhiteSpace(' ')
PsiElement(Py:IDENTIFIER)('A')
PyArgumentList
<empty list>
PsiElement(Py:COLON)(':')
PsiWhiteSpace('\n ')
PyStatementList
PyEmptyExpression
PsiElement(Py:QUESTION_MARK)('?')
@@ -0,0 +1 @@
int ?
@@ -0,0 +1,7 @@
PyFile:helpError.py
PyEmptyExpression
PsiElement(Py:IDENTIFIER)('int')
PsiWhiteSpace(' ')
PsiElement(Py:QUESTION_MARK)('?')
PsiErrorElement:Help request must follow the name
<empty list>
@@ -0,0 +1 @@
?object
@@ -0,0 +1,4 @@
PyFile:helpObjectPrefix.py
PyEmptyExpression
PsiElement(Py:QUESTION_MARK)('?')
PsiElement(Py:IDENTIFIER)('object')
@@ -0,0 +1 @@
object?
@@ -0,0 +1,4 @@
PyFile:helpObjectSuffix.py
PyEmptyExpression
PsiElement(Py:IDENTIFIER)('object')
PsiElement(Py:QUESTION_MARK)('?')
@@ -0,0 +1 @@
??object
@@ -0,0 +1,5 @@
PyFile:helpObjectVerbosePrefix.py
PyEmptyExpression
PsiElement(Py:QUESTION_MARK)('?')
PsiElement(Py:QUESTION_MARK)('?')
PsiElement(Py:IDENTIFIER)('object')
@@ -0,0 +1 @@
object??
@@ -0,0 +1,5 @@
PyFile:helpObjectVerboseSuffix.py
PyEmptyExpression
PsiElement(Py:IDENTIFIER)('object')
PsiElement(Py:QUESTION_MARK)('?')
PsiElement(Py:QUESTION_MARK)('?')
@@ -0,0 +1 @@
*int*?
@@ -0,0 +1,6 @@
PyFile:helpWildcards.py
PyEmptyExpression
PsiElement(Py:MULT)('*')
PsiElement(Py:IDENTIFIER)('int')
PsiElement(Py:MULT)('*')
PsiElement(Py:QUESTION_MARK)('?')
@@ -0,0 +1 @@
%xmode
@@ -0,0 +1,4 @@
PyFile:magic1.py
PyEmptyExpression
PsiElement(Py:PERC)('%')
PsiElement(Py:IDENTIFIER)('xmode')
@@ -0,0 +1 @@
%alias_magic t timeit
@@ -0,0 +1,8 @@
PyFile:magic2.py
PyEmptyExpression
PsiElement(Py:PERC)('%')
PsiElement(Py:IDENTIFIER)('alias_magic')
PsiWhiteSpace(' ')
PsiElement(Py:IDENTIFIER)('t')
PsiWhiteSpace(' ')
PsiElement(Py:IDENTIFIER)('timeit')
@@ -0,0 +1 @@
%config Class.trait=value
@@ -0,0 +1,10 @@
PyFile:magic3.py
PyEmptyExpression
PsiElement(Py:PERC)('%')
PsiElement(Py:IDENTIFIER)('config')
PsiWhiteSpace(' ')
PsiElement(Py:IDENTIFIER)('Class')
PsiElement(Py:DOT)('.')
PsiElement(Py:IDENTIFIER)('trait')
PsiElement(Py:EQ)('=')
PsiElement(Py:IDENTIFIER)('value')
@@ -0,0 +1 @@
results = %timeit -r1 -n1 -o list(range(1000))
@@ -0,0 +1,27 @@
PyFile:magicAssignment.py
PyAssignmentStatement
PyTargetExpression: results
PsiElement(Py:IDENTIFIER)('results')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyEmptyExpression
PsiElement(Py:PERC)('%')
PsiElement(Py:IDENTIFIER)('timeit')
PsiWhiteSpace(' ')
PsiElement(Py:MINUS)('-')
PsiElement(Py:IDENTIFIER)('r1')
PsiWhiteSpace(' ')
PsiElement(Py:MINUS)('-')
PsiElement(Py:IDENTIFIER)('n1')
PsiWhiteSpace(' ')
PsiElement(Py:MINUS)('-')
PsiElement(Py:IDENTIFIER)('o')
PsiWhiteSpace(' ')
PsiElement(Py:IDENTIFIER)('list')
PsiElement(Py:LPAR)('(')
PsiElement(Py:IDENTIFIER)('range')
PsiElement(Py:LPAR)('(')
PsiElement(Py:INTEGER_LITERAL)('1000')
PsiElement(Py:RPAR)(')')
PsiElement(Py:RPAR)(')')
@@ -0,0 +1 @@
b = %%bach
@@ -0,0 +1,13 @@
PyFile:magicError.py
PyAssignmentStatement
PyTargetExpression: b
PsiElement(Py:IDENTIFIER)('b')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiErrorElement:Multiline magic can't be used as an expression
<empty list>
PsiWhiteSpace(' ')
PyEmptyExpression
PsiElement(Py:PERC)('%')
PsiElement(Py:PERC)('%')
PsiElement(Py:IDENTIFIER)('bach')
@@ -0,0 +1,4 @@
%%bash
echo "My shell is:" $SHELL
echo "My disk usage is:"
df -h
@@ -0,0 +1,21 @@
PyFile:magicMultiline.py
PyEmptyExpression
PsiElement(Py:PERC)('%')
PsiElement(Py:PERC)('%')
PsiElement(Py:IDENTIFIER)('bash')
PsiWhiteSpace('\n')
PsiElement(Py:IDENTIFIER)('echo')
PsiWhiteSpace(' ')
PsiElement(Py:SINGLE_QUOTED_STRING)('"My shell is:"')
PsiWhiteSpace(' ')
PsiElement(BAD_CHARACTER)('$')
PsiElement(Py:IDENTIFIER)('SHELL')
PsiWhiteSpace('\n')
PsiElement(Py:IDENTIFIER)('echo')
PsiWhiteSpace(' ')
PsiElement(Py:SINGLE_QUOTED_STRING)('"My disk usage is:"')
PsiWhiteSpace('\n')
PsiElement(Py:IDENTIFIER)('df')
PsiWhiteSpace(' ')
PsiElement(Py:MINUS)('-')
PsiElement(Py:IDENTIFIER)('h')
@@ -0,0 +1 @@
!pwd
@@ -0,0 +1,4 @@
PyFile:shell1.py
PyEmptyExpression
PsiElement(Py:PLING)('!')
PsiElement(Py:IDENTIFIER)('pwd')
@@ -0,0 +1 @@
!cd /var/etc
@@ -0,0 +1,9 @@
PyFile:shell2.py
PyEmptyExpression
PsiElement(Py:PLING)('!')
PsiElement(Py:IDENTIFIER)('cd')
PsiWhiteSpace(' ')
PsiElement(Py:DIV)('/')
PsiElement(Py:IDENTIFIER)('var')
PsiElement(Py:DIV)('/')
PsiElement(Py:IDENTIFIER)('etc')
@@ -0,0 +1 @@
!mvim myfile.txt
@@ -0,0 +1,8 @@
PyFile:shell3.py
PyEmptyExpression
PsiElement(Py:PLING)('!')
PsiElement(Py:IDENTIFIER)('mvim')
PsiWhiteSpace(' ')
PsiElement(Py:IDENTIFIER)('myfile')
PsiElement(Py:DOT)('.')
PsiElement(Py:IDENTIFIER)('txt')
@@ -0,0 +1 @@
!!dir
@@ -0,0 +1,5 @@
PyFile:shell4.py
PyEmptyExpression
PsiElement(Py:PLING)('!')
PsiElement(Py:PLING)('!')
PsiElement(Py:IDENTIFIER)('dir')
@@ -0,0 +1 @@
!!
@@ -0,0 +1,4 @@
PyFile:shell5.py
PyEmptyExpression
PsiElement(Py:PLING)('!')
PsiElement(Py:PLING)('!')
@@ -0,0 +1 @@
a = !!dir
@@ -0,0 +1,11 @@
PyFile:shell6.py
PyAssignmentStatement
PyTargetExpression: a
PsiElement(Py:IDENTIFIER)('a')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyEmptyExpression
PsiElement(Py:PLING)('!')
PsiElement(Py:PLING)('!')
PsiElement(Py:IDENTIFIER)('dir')
@@ -0,0 +1 @@
my_files = !ls
@@ -0,0 +1,10 @@
PyFile:shellAssignment1.py
PyAssignmentStatement
PyTargetExpression: my_files
PsiElement(Py:IDENTIFIER)('my_files')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyEmptyExpression
PsiElement(Py:PLING)('!')
PsiElement(Py:IDENTIFIER)('ls')
@@ -0,0 +1 @@
my_files_1 = !ls -la
@@ -0,0 +1,13 @@
PyFile:shellAssignment2.py
PyAssignmentStatement
PyTargetExpression: my_files_1
PsiElement(Py:IDENTIFIER)('my_files_1')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyEmptyExpression
PsiElement(Py:PLING)('!')
PsiElement(Py:IDENTIFIER)('ls')
PsiWhiteSpace(' ')
PsiElement(Py:MINUS)('-')
PsiElement(Py:IDENTIFIER)('la')
@@ -0,0 +1 @@
int!
@@ -0,0 +1,11 @@
PyFile:shellError.py
PyExpressionStatement
PyReferenceExpression: int
PsiElement(Py:IDENTIFIER)('int')
PsiErrorElement:End of statement expected
<empty list>
PsiElement(Py:PLING)('!')
PsiErrorElement:Identifier expected.
<empty list>
PsiErrorElement:Statement expected, found Py:PLING
<empty list>
@@ -0,0 +1 @@
!mv $file {file.upper()}
@@ -0,0 +1,15 @@
PyFile:shellExpansion.py
PyEmptyExpression
PsiElement(Py:PLING)('!')
PsiElement(Py:IDENTIFIER)('mv')
PsiWhiteSpace(' ')
PsiElement(BAD_CHARACTER)('$')
PsiElement(Py:IDENTIFIER)('file')
PsiWhiteSpace(' ')
PsiElement(Py:LBRACE)('{')
PsiElement(Py:IDENTIFIER)('file')
PsiElement(Py:DOT)('.')
PsiElement(Py:IDENTIFIER)('upper')
PsiElement(Py:LPAR)('(')
PsiElement(Py:RPAR)(')')
PsiElement(Py:RBRACE)('}')
@@ -0,0 +1,93 @@
// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.jetbrains.python.console
import com.intellij.lang.ASTFactory
import com.intellij.lang.LanguageASTFactory
import com.intellij.openapi.application.PathManager
import com.intellij.openapi.project.Project
import com.intellij.testFramework.ParsingTestCase
import com.jetbrains.python.*
import com.jetbrains.python.console.parsing.PyConsoleParser
import com.jetbrains.python.console.parsing.PythonConsoleData
import com.jetbrains.python.console.parsing.PythonConsoleLexer
import com.jetbrains.python.psi.LanguageLevel
import com.jetbrains.python.psi.impl.PythonASTFactory
class IPythonConsoleParsingTest : ParsingTestCase(
"psi",
"py",
true,
PyConsoleParsingDefinition()
) {
override fun setUp() {
super.setUp()
registerExtensionPoint(PythonDialectsTokenSetContributor.EP_NAME, PythonDialectsTokenSetContributor::class.java)
registerExtension(PythonDialectsTokenSetContributor.EP_NAME, PythonTokenSetContributor())
addExplicitExtension<ASTFactory>(LanguageASTFactory.INSTANCE, PythonLanguage.getInstance(), PythonASTFactory())
PythonDialectsTokenSetProvider.reset()
}
override fun getTestDataPath() = "${PathManager.getHomePath()}/community/python/testData/console/ipython"
fun testHelp1() = doTestInternal()
fun testHelp2() = doTestInternal()
fun testHelp3() = doTestInternal()
fun testHelpObjectPrefix() = doTestInternal()
fun testHelpObjectSuffix() = doTestInternal()
fun testHelpObjectVerbosePrefix() = doTestInternal()
fun testHelpObjectVerboseSuffix() = doTestInternal()
fun testHelpWildcards() = doTestInternal()
fun testHelpError() = doTestInternal(false)
fun testShell1() = doTestInternal()
fun testShell2() = doTestInternal()
fun testShell3() = doTestInternal()
fun testShell4() = doTestInternal()
fun testShell5() = doTestInternal()
fun testShell6() = doTestInternal()
fun testShellAssignment1() = doTestInternal()
fun testShellAssignment2() = doTestInternal()
fun testShellExpansion() = doTestInternal()
fun testShellError() = doTestInternal(false)
fun testMagic1() = doTestInternal()
fun testMagic2() = doTestInternal()
fun testMagic3() = doTestInternal()
fun testMagicAssignment() = doTestInternal()
fun testMagicError() = doTestInternal(false)
fun testMagicMultiline() = doTestInternal()
private fun doTestInternal(ensureNoErrorElements: Boolean = true) = doTest(true, ensureNoErrorElements)
}
private class PyConsoleParsingDefinition : PythonParserDefinition() {
override fun createLexer(project: Project) = PythonConsoleLexer()
override fun createParser(project: Project) = PyConsoleParser(
PythonConsoleData().apply { isIPythonEnabled = true },
LanguageLevel.getDefault()
)
}
@@ -1,26 +1,12 @@
/*
* Copyright 2000-2016 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.jetbrains.python
// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.jetbrains.python.console
import com.intellij.openapi.command.WriteCommandAction
import com.intellij.openapi.editor.Editor
import com.intellij.openapi.editor.ex.EditorEx
import com.intellij.openapi.util.Disposer
import com.intellij.testFramework.EditorTestUtil
import com.jetbrains.python.console.PyConsoleEnterHandler
import com.jetbrains.python.PythonFileType
import com.jetbrains.python.fixtures.PyTestCase
import kotlin.math.max
@@ -1,24 +1,10 @@
/*
* Copyright 2000-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.jetbrains.python;
// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.jetbrains.python.console;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.testFramework.UsefulTestCase;
import com.jetbrains.python.console.PyConsoleIndentUtil;
import com.jetbrains.python.PythonTestUtil;
import java.io.File;
import java.io.IOException;