MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

separated token types for single/double and triple quoted strings

Browse Source 
added lexer-level bytes/unicode detection
added lexer-level docstring detection
This commit is contained in:
Ekaterina Tuzova
2011-07-10 17:59:57 +04:00
parent c9a2641059
commit ef175a3436
42 changed files with 950 additions and 523 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/src/com/jetbrains/python/PyNames.java
+1
View File
@@ -36,6 +36,7 @@ public class PyNames {
public static final String FAKE_OLD_BASE = "___Classobj";
public static final String FUTURE_MODULE = "__future__";
public static final String UNICODE_LITERALS = "unicode_literals";
public static final String CLASSMETHOD = "classmethod";
public static final String STATICMETHOD = "staticmethod";
python/src/com/jetbrains/python/PyTokenTypes.java
+11 -1
View File
@@ -72,8 +72,18 @@ public class PyTokenTypes {
public static final PyElementType INTEGER_LITERAL = new PyElementType("INTEGER_LITERAL");
public static final PyElementType FLOAT_LITERAL = new PyElementType("FLOAT_LITERAL");
public static final PyElementType IMAGINARY_LITERAL = new PyElementType("IMAGINARY_LITERAL");
public static final PyElementType STRING_LITERAL = new PyElementType("STRING_LITERAL");
public static final PyElementType SINGLE_QUOTED_STRING = new PyElementType("SINGLE_QUOTED_STRING");
public static final PyElementType TRIPLE_QUOTED_STRING = new PyElementType("TRIPLE_QUOTED_STRING");
public static final PyElementType SINGLE_QUOTED_UNICODE = new PyElementType("SINGLE_QUOTED_UNICODE");
public static final PyElementType TRIPLE_QUOTED_UNICODE = new PyElementType("TRIPLE_QUOTED_UNICODE");
public static final PyElementType DOCSTRING = new PyElementType("DOCSTRING");
public static final TokenSet UNICODE_NODES = TokenSet.create(TRIPLE_QUOTED_UNICODE, SINGLE_QUOTED_UNICODE);
public static final TokenSet STRING_NODES = TokenSet.orSet(UNICODE_NODES, TokenSet.create(SINGLE_QUOTED_STRING,
TRIPLE_QUOTED_STRING, DOCSTRING));
// Operators
public static final PyElementType PLUS = new PyElementType("PLUS", "__add__");// +
public static final PyElementType MINUS = new PyElementType("MINUS", "__sub__");// -
python/src/com/jetbrains/python/PythonFileType.java
+15
View File
@@ -1,11 +1,17 @@
package com.jetbrains.python;
import com.intellij.openapi.editor.colors.EditorColorsScheme;
import com.intellij.openapi.editor.highlighter.EditorHighlighter;
import com.intellij.openapi.fileTypes.EditorHighlighterProvider;
import com.intellij.openapi.fileTypes.FileType;
import com.intellij.openapi.fileTypes.FileTypeEditorHighlighterProviders;
import com.intellij.openapi.fileTypes.LanguageFileType;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.util.IconLoader;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.openapi.vfs.CharsetToolkit;
import com.intellij.openapi.vfs.VirtualFile;
import com.jetbrains.python.lexer.PythonEditorHighlighter;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -30,6 +36,15 @@ public class PythonFileType extends LanguageFileType {
public PythonFileType() {
super(new PythonLanguage());
_icon = IconLoader.getIcon("/com/jetbrains/python/icons/pythonFile.png");
FileTypeEditorHighlighterProviders.INSTANCE.addExplicitExtension(this, new EditorHighlighterProvider() {
@Override
public EditorHighlighter getEditorHighlighter(@Nullable Project project,
@NotNull FileType fileType, @Nullable VirtualFile virtualFile,
@NotNull EditorColorsScheme colors) {
return new PythonEditorHighlighter(colors, project, virtualFile);
}
});
}
@NotNull
python/src/com/jetbrains/python/PythonParserDefinition.java
+1 -1
View File
@@ -35,7 +35,7 @@ public class PythonParserDefinition implements ParserDefinition {
myWhitespaceTokens = TokenSet.create(PyTokenTypes.LINE_BREAK, PyTokenTypes.SPACE, PyTokenTypes.TAB, PyTokenTypes.FORMFEED);
myCommentTokens = TokenSet.create(PyTokenTypes.END_OF_LINE_COMMENT);
myStringLiteralTokens = TokenSet.create(PyTokenTypes.STRING_LITERAL, PyElementTypes.STRING_LITERAL_EXPRESSION);
myStringLiteralTokens = TokenSet.orSet(PyTokenTypes.STRING_NODES, TokenSet.create(PyElementTypes.STRING_LITERAL_EXPRESSION));
}
@NotNull
python/src/com/jetbrains/python/actions/ConvertDocstringQuickFix.java
+1 -2
View File
@@ -42,8 +42,7 @@ public class ConvertDocstringQuickFix implements LocalQuickFix {
content = content.length() == 1 ? "" : content.substring(1, content.length()-1);
}
PyExpression newString = elementGenerator.createFromText(LanguageLevel.forElement(expression),
PyExpressionStatement.class, myModificator+"\"\"\"" + content + "\"\"\"").getExpression();
PyExpression newString = elementGenerator.createDocstring(myModificator+"\"\"\"" + content + "\"\"\"").getExpression();
expression.replace(newString);
}
}
python/src/com/jetbrains/python/actions/DocstringQuickFix.java
+1 -2
View File
@@ -66,8 +66,7 @@ public class DocstringQuickFix implements LocalQuickFix {
replacement = createUnexpectedReplacement(replacement);
}
if (!replacement.equals(element.getText())) {
PyStringLiteralExpression str = (PyStringLiteralExpression)elementGenerator.createFromText(LanguageLevel.forElement(element),
PyExpressionStatement.class, replacement).getExpression();
PyExpression str = elementGenerator.createDocstring(replacement).getExpression();
element.replace(str);
}
}
python/src/com/jetbrains/python/codeInsight/PyKeywordCompletionContributor.java
+1 -1
View File
@@ -90,7 +90,7 @@ public class PyKeywordCompletionContributor extends PySeeingOriginalCompletionCo
public boolean isAcceptable(Object element, PsiElement context) {
if (element instanceof PsiElement) {
final ASTNode ctxNode = context.getNode();
if (ctxNode != null && ctxNode.getElementType() == PyTokenTypes.STRING_LITERAL) return false; // no sense inside string
if (ctxNode != null && PyTokenTypes.STRING_NODES.contains(ctxNode.getElementType())) return false; // no sense inside string
PsiElement p = (PsiElement)element;
//int org_offset = p.getUserData(ORG_OFFSET); // saved by fillCompletionVariants()
p = p.getUserData(ORG_ELT); // saved by fillCompletionVariants().
python/src/com/jetbrains/python/codeInsight/intentions/PyDocStubIntention.java
+1 -2
View File
@@ -59,8 +59,7 @@ public class PyDocStubIntention extends BaseIntentionAction {
ws = ws + whitespace.getText().split("\n")[1];
}
String docContent = ws + documentationProvider.generateDocumentationContentStub(function, ws, true);
PyExpressionStatement string = elementGenerator.createFromText(LanguageLevel.forElement(function), PyExpressionStatement.class,
"\"\"\"" + docContent + "\"\"\"");
PyExpressionStatement string = elementGenerator.createDocstring("\"\"\"" + docContent + "\"\"\"");
if (list.getStatements().length != 0)
list.addBefore(string, list.getStatements()[0]);
python/src/com/jetbrains/python/editor/PythonQuoteHandler.java
+1 -2
View File
@@ -1,6 +1,5 @@
package com.jetbrains.python.editor;
import com.intellij.psi.tree.TokenSet;
import com.jetbrains.python.PyTokenTypes;
/**
@@ -8,6 +7,6 @@ import com.jetbrains.python.PyTokenTypes;
*/
public class PythonQuoteHandler extends BaseQuoteHandler {
public PythonQuoteHandler() {
super(TokenSet.create(PyTokenTypes.STRING_LITERAL), new char[]{'}', ']', ')', ',', ':', ';', ' ', '\t', '\n'});
super(PyTokenTypes.STRING_NODES, new char[]{'}', ']', ')', ',', ':', ';', ' ', '\t', '\n'});
}
}
python/src/com/jetbrains/python/editor/selectWord/PyLiteralSelectionHandler.java
+1 -1
View File
@@ -19,7 +19,7 @@ public class PyLiteralSelectionHandler implements ExtendWordSelectionHandler {
@Override
public boolean canSelect(PsiElement e) {
final ASTNode node = e.getNode();
return node != null && node.getElementType() == PyTokenTypes.STRING_LITERAL;
return node != null && PyTokenTypes.STRING_NODES.contains(node.getElementType());
}
@Override
python/src/com/jetbrains/python/findUsages/PyWordsScanner.java
+1 -1
View File
@@ -13,6 +13,6 @@ public class PyWordsScanner extends DefaultWordsScanner {
super(new PythonLexer(),
TokenSet.create(PyTokenTypes.IDENTIFIER),
TokenSet.create(PyTokenTypes.END_OF_LINE_COMMENT),
TokenSet.create(PyTokenTypes.STRING_LITERAL));
PyTokenTypes.STRING_NODES);
}
}
python/src/com/jetbrains/python/formatter/PyBlock.java
+1 -1
View File
@@ -161,7 +161,7 @@ public class PyBlock implements ASTBlock {
}
}
else if (parentType == PyElementTypes.STRING_LITERAL_EXPRESSION) {
if (childType == PyTokenTypes.STRING_LITERAL) {
if (PyTokenTypes.STRING_NODES.contains(childType)) {
childAlignment = getAlignmentForChildren();
}
}
python/src/com/jetbrains/python/highlighting/PyHighlighter.java
+7 -3
View File
@@ -1,6 +1,5 @@
package com.jetbrains.python.highlighting;
import com.intellij.lexer.LayeredLexer;
import com.intellij.lexer.Lexer;
import com.intellij.openapi.editor.HighlighterColors;
import com.intellij.openapi.editor.colors.TextAttributesKey;
@@ -27,7 +26,7 @@ public class PyHighlighter extends SyntaxHighlighterBase {
@NotNull
public Lexer getHighlightingLexer() {
return new LayeredLexer(new PythonHighlightingLexer(myLanguageLevel));
return new PythonHighlightingLexer(myLanguageLevel);
}
private static TextAttributesKey _copy(String name, TextAttributesKey src) {
@@ -102,7 +101,12 @@ public class PyHighlighter extends SyntaxHighlighterBase {
keys.put(PyTokenTypes.INTEGER_LITERAL, PY_NUMBER);
keys.put(PyTokenTypes.FLOAT_LITERAL, PY_NUMBER);
keys.put(PyTokenTypes.IMAGINARY_LITERAL, PY_NUMBER);
keys.put(PyTokenTypes.STRING_LITERAL, PY_BYTE_STRING);
keys.put(PyTokenTypes.SINGLE_QUOTED_STRING, PY_BYTE_STRING);
keys.put(PyTokenTypes.TRIPLE_QUOTED_STRING, PY_BYTE_STRING);
keys.put(PyTokenTypes.SINGLE_QUOTED_UNICODE, PY_UNICODE_STRING);
keys.put(PyTokenTypes.TRIPLE_QUOTED_UNICODE, PY_UNICODE_STRING);
keys.put(PyTokenTypes.DOCSTRING, PY_DOC_COMMENT);
keys.put(PyTokenTypes.LPAR, PY_PARENTHS);
keys.put(PyTokenTypes.RPAR, PY_PARENTHS);
python/src/com/jetbrains/python/lexer/Python.flex
+35 -11
View File
@@ -43,9 +43,15 @@ EXPONENT = [eE][+\-]?({DIGIT})+
IMAGNUMBER=(({FLOATNUMBER})|({INTPART}))[Jj]
STRING_LITERAL=[UuBb]?({RAW_STRING}|{QUOTED_STRING})
RAW_STRING=[Rr]{QUOTED_STRING}
QUOTED_STRING=({TRIPLE_APOS_LITERAL})|({QUOTED_LITERAL})|({DOUBLE_QUOTED_LITERAL})|({TRIPLE_QUOTED_LITERAL})
//STRING_LITERAL=[UuBb]?({RAW_STRING}|{QUOTED_STRING})
//RAW_STRING=[Rr]{QUOTED_STRING}
//QUOTED_STRING=({TRIPLE_APOS_LITERAL})|({QUOTED_LITERAL})|({DOUBLE_QUOTED_LITERAL})|({TRIPLE_QUOTED_LITERAL})
SINGLE_QUOTED_STRING=[UuBb]?[Rr]?({QUOTED_LITERAL} | {DOUBLE_QUOTED_LITERAL})
TRIPLE_QUOTED_STRING=[UuBb]?[Rr]?({TRIPLE_QUOTED_LITERAL}|{TRIPLE_APOS_LITERAL})
DOCSTRING_LITERAL=({SINGLE_QUOTED_STRING}|{TRIPLE_QUOTED_STRING})
QUOTED_LITERAL="'" ([^\\\'\r\n] | {ESCAPE_SEQUENCE} | (\\[\r\n]))* ("'"|\\)?
DOUBLE_QUOTED_LITERAL=\"([^\\\"\r\n]|{ESCAPE_SEQUENCE}|(\\[\r\n]))*?(\"|\\)?
ESCAPE_SEQUENCE=\\[^\r\n]
@@ -62,22 +68,30 @@ ONE_TWO_APOS = ('[^']) | ('\\[^]) | (''[^']) | (''\\[^])
APOS_STRING_CHAR = [^\\'] | {ANY_ESCAPE_SEQUENCE} | {ONE_TWO_APOS}
TRIPLE_APOS_LITERAL = {THREE_APOS} {APOS_STRING_CHAR}* {THREE_APOS}?
%state PENDING_DOCSTRING
%state USUAL
%%
[\n] { return PyTokenTypes.LINE_BREAK; }
[\ ] { return PyTokenTypes.SPACE; }
[\t] { return PyTokenTypes.TAB; }
[\f] { return PyTokenTypes.FORMFEED; }
[\n] { return PyTokenTypes.LINE_BREAK; }
[\ ] { return PyTokenTypes.SPACE; }
[\t] { return PyTokenTypes.TAB; }
[\f] { return PyTokenTypes.FORMFEED; }
{END_OF_LINE_COMMENT} { return PyTokenTypes.END_OF_LINE_COMMENT; }
"\\" { return PyTokenTypes.BACKSLASH; }
{END_OF_LINE_COMMENT} { return PyTokenTypes.END_OF_LINE_COMMENT; }
<YYINITIAL> {
. { yypushback(1); yybegin(PENDING_DOCSTRING);}
}
<USUAL> {
{LONGINTEGER} { return PyTokenTypes.INTEGER_LITERAL; }
{INTEGER} { return PyTokenTypes.INTEGER_LITERAL; }
{FLOATNUMBER} { return PyTokenTypes.FLOAT_LITERAL; }
{IMAGNUMBER} { return PyTokenTypes.IMAGINARY_LITERAL; }
{STRING_LITERAL} { return PyTokenTypes.STRING_LITERAL; }
{SINGLE_QUOTED_STRING} { return PyTokenTypes.SINGLE_QUOTED_STRING; }
{TRIPLE_QUOTED_STRING} { return PyTokenTypes.TRIPLE_QUOTED_STRING; }
"and" { return PyTokenTypes.AND_KEYWORD; }
"assert" { return PyTokenTypes.ASSERT_KEYWORD; }
@@ -149,12 +163,22 @@ TRIPLE_APOS_LITERAL = {THREE_APOS} {APOS_STRING_CHAR}* {THREE_APOS}?
"}" { return PyTokenTypes.RBRACE; }
"@" { return PyTokenTypes.AT; }
"," { return PyTokenTypes.COMMA; }
":" { return PyTokenTypes.COLON; }
":" { yybegin(PENDING_DOCSTRING); return PyTokenTypes.COLON; }
"." { return PyTokenTypes.DOT; }
"`" { return PyTokenTypes.TICK; }
"=" { return PyTokenTypes.EQ; }
";" { return PyTokenTypes.SEMICOLON; }
"\\" { return PyTokenTypes.BACKSLASH; }
. { return PyTokenTypes.BAD_CHARACTER; }
}
<PENDING_DOCSTRING> {
{SINGLE_QUOTED_STRING} { yybegin(USUAL); return PyTokenTypes.SINGLE_QUOTED_STRING; }
{TRIPLE_QUOTED_STRING} { yybegin(USUAL); return PyTokenTypes.TRIPLE_QUOTED_STRING; }
{DOCSTRING_LITERAL}[\n\t\f; ] { yypushback(1); yybegin(USUAL); return PyTokenTypes.DOCSTRING; }
{DOCSTRING_LITERAL}(\ )*[\\] { yypushback(1); return PyTokenTypes.DOCSTRING; }
. { yypushback(1); yybegin(USUAL); }
}
python/src/com/jetbrains/python/lexer/PythonEditorHighlighter.java
+38
View File
@@ -0,0 +1,38 @@
package com.jetbrains.python.lexer;
import com.intellij.lexer.Lexer;
import com.intellij.openapi.editor.Document;
import com.intellij.openapi.editor.colors.EditorColorsScheme;
import com.intellij.openapi.editor.event.DocumentEvent;
import com.intellij.openapi.editor.ex.util.LexerEditorHighlighter;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.vfs.VirtualFile;
import com.jetbrains.python.PythonLanguage;
import com.jetbrains.python.highlighting.PySyntaxHighlighterFactory;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
/**
* User : catherine
*/
public class PythonEditorHighlighter extends LexerEditorHighlighter {
public PythonEditorHighlighter(@NotNull EditorColorsScheme scheme, @Nullable Project project, @Nullable VirtualFile file) {
super(PySyntaxHighlighterFactory.getSyntaxHighlighter(PythonLanguage.getInstance(), project, file), scheme);
}
public void documentChanged(DocumentEvent e) {
synchronized (this) {
final Document document = e.getDocument();
Lexer l = getLexer();
if (l instanceof PythonHighlightingLexer &&
(((PythonHighlightingLexer)l).getImportOffset() > e.getOffset()
|| ((PythonHighlightingLexer)l).getImportOffset() == -1)) {
((PythonHighlightingLexer)l).clearState(e.getDocument().getTextLength());
setText(document.getCharsSequence());
}
else super.documentChanged(e);
}
}
}
python/src/com/jetbrains/python/lexer/PythonHighlightingLexer.java
+112
View File
@@ -1,6 +1,7 @@
package com.jetbrains.python.lexer;
import com.intellij.psi.tree.IElementType;
import com.jetbrains.python.PyNames;
import com.jetbrains.python.PyTokenTypes;
import com.jetbrains.python.psi.LanguageLevel;
@@ -12,11 +13,38 @@ public class PythonHighlightingLexer extends PythonLexer {
public PythonHighlightingLexer(LanguageLevel languageLevel) {
myLanguageLevel = languageLevel;
hasUnicodeImport = false;
}
public IElementType convertStringType(IElementType tokenType, String tokenText) {
if (tokenType == PyTokenTypes.SINGLE_QUOTED_STRING) {
if (myLanguageLevel.isPy3K()) {
if (!tokenText.toLowerCase().startsWith("b")) return PyTokenTypes.SINGLE_QUOTED_UNICODE;
}
else {
if ((hasUnicodeImport && !tokenText.toLowerCase().startsWith("b"))
|| tokenText.toLowerCase().startsWith("u")) return PyTokenTypes.SINGLE_QUOTED_UNICODE;
}
}
if (tokenType == PyTokenTypes.TRIPLE_QUOTED_STRING) {
if (myLanguageLevel.isPy3K()) {
if (!tokenText.toLowerCase().startsWith("b")) return PyTokenTypes.TRIPLE_QUOTED_UNICODE;
}
else {
if ((hasUnicodeImport && !tokenText.toLowerCase().startsWith("b"))
|| tokenText.toLowerCase().startsWith("u")) return PyTokenTypes.TRIPLE_QUOTED_UNICODE;
}
}
return tokenType;
}
@Override
public IElementType getTokenType() {
final String tokenText = getTokenText();
IElementType tokenType = super.getTokenType();
if (PyTokenTypes.STRING_NODES.contains(tokenType))
return convertStringType(tokenType, tokenText);
if (myLanguageLevel.hasWithStatement()) {
if (tokenText.equals("with")) return PyTokenTypes.WITH_KEYWORD;
if (tokenText.equals("as")) return PyTokenTypes.AS_KEYWORD;
@@ -35,4 +63,88 @@ public class PythonHighlightingLexer extends PythonLexer {
}
return super.getTokenType();
}
private enum state {
init,
pending_future,
pending_import,
pending_lpar,
pending_id,
pending_comma,
stop
} ;
private state myState = state.init;
private boolean hasUnicodeImport = false;
private int myImportOffset = -1;
@Override
public void advance() {
IElementType type = super.getTokenType();
String tokenText = super.getTokenText();
switch (myState) {
case init:
if (type == PyTokenTypes.BACKSLASH) break;
if (PyTokenTypes.WHITESPACE_OR_LINEBREAK.contains(type)) break;
if (PyTokenTypes.END_OF_LINE_COMMENT == type) break;
if (PyTokenTypes.DOCSTRING == type) break;
if (type == PyTokenTypes.FROM_KEYWORD)
myState = state.pending_future;
else myState = state.stop;
break;
case pending_future:
if (type == PyTokenTypes.BACKSLASH) break;
if (PyTokenTypes.WHITESPACE_OR_LINEBREAK.contains(type)) break;
if (type == PyTokenTypes.IDENTIFIER && PyNames.FUTURE_MODULE.equals(tokenText))
myState = state.pending_import;
else myState = state.stop;
break;
case pending_import:
if (type == PyTokenTypes.BACKSLASH) break;
if (PyTokenTypes.WHITESPACE_OR_LINEBREAK.contains(type)) break;
if (type == PyTokenTypes.IMPORT_KEYWORD)
myState = state.pending_lpar;
else myState = state.stop;
break;
case pending_lpar:
if (type == PyTokenTypes.LPAR) {
myState = state.pending_id;
break;
}
case pending_id:
if (type == PyTokenTypes.BACKSLASH) break;
if (PyTokenTypes.WHITESPACE_OR_LINEBREAK.contains(type)) break;
if (type == PyTokenTypes.IDENTIFIER) {
myState = state.pending_comma;
if (PyNames.UNICODE_LITERALS.equals(tokenText)) {
hasUnicodeImport = true;
myImportOffset = getTokenEnd();
}
}
else myState = state.init;
break;
case pending_comma:
if (type == PyTokenTypes.RPAR) break;
if (type == PyTokenTypes.BACKSLASH) break;
if (PyTokenTypes.LINE_BREAK == type) myState = state.init;
if (PyTokenTypes.WHITESPACE_OR_LINEBREAK.contains(type)) break;
if (type == PyTokenTypes.COMMA)
myState = state.pending_id;
break;
}
super.advance();
}
public int getImportOffset() {
return myImportOffset;
}
public void clearState(int position) {
myState = state.init;
myImportOffset = position;
hasUnicodeImport = false;
}
}
python/src/com/jetbrains/python/lexer/_PythonLexer.java
+474 -416
View File
File diff suppressed because it is too large Load Diff
python/src/com/jetbrains/python/parsing/ExpressionParsing.java
+2 -2
View File
@@ -50,9 +50,9 @@ public class ExpressionParsing extends Parsing {
buildTokenElement(PyElementTypes.BOOL_LITERAL_EXPRESSION, myBuilder);
return true;
}
else if (firstToken == PyTokenTypes.STRING_LITERAL) {
else if (PyTokenTypes.STRING_NODES.contains(firstToken)) {
final PsiBuilder.Marker marker = myBuilder.mark();
while (myBuilder.getTokenType() == PyTokenTypes.STRING_LITERAL) {
while (PyTokenTypes.STRING_NODES.contains(myBuilder.getTokenType())) {
myBuilder.advanceLexer();
}
marker.done(PyElementTypes.STRING_LITERAL_EXPRESSION);
python/src/com/jetbrains/python/psi/PyElementGenerator.java
+2
View File
@@ -68,4 +68,6 @@ public abstract class PyElementGenerator {
public abstract PyNamedParameter createParameter(@NotNull String name);
public abstract PsiFile createDummyFile(LanguageLevel langLevel, String contents);
public abstract PyExpressionStatement createDocstring(String content);
}
python/src/com/jetbrains/python/psi/impl/PyBuiltinCache.java
+10
View File
@@ -263,6 +263,16 @@ public class PyBuiltinCache {
}
}
@Nullable
public PyClassType getUnicodeType(LanguageLevel level) {
if (level.isPy3K()) {
return getObjectType("str");
}
else {
return getObjectType("unicode");
}
}
@Nullable
public PyType getStringType(LanguageLevel level) {
if (level.isPy3K()) {
python/src/com/jetbrains/python/psi/impl/PyElementGeneratorImpl.java
+5
View File
@@ -237,4 +237,9 @@ public class PyElementGeneratorImpl extends PyElementGenerator {
}
return (T)ret;
}
public PyExpressionStatement createDocstring(String content){
return createFromText(LanguageLevel.getDefault(),
PyExpressionStatement.class, content + "\n");
}
}
python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java
+15 -11
View File
@@ -1,6 +1,8 @@
package com.jetbrains.python.psi.impl;
import com.intellij.lang.ASTNode;
import com.intellij.lexer.Lexer;
import com.intellij.openapi.fileTypes.SyntaxHighlighter;
import com.intellij.openapi.util.Pair;
import com.intellij.openapi.util.Ref;
import com.intellij.openapi.util.TextRange;
@@ -8,8 +10,11 @@ import com.intellij.psi.*;
import com.intellij.psi.impl.source.resolve.reference.ReferenceProvidersRegistry;
import com.intellij.psi.impl.source.tree.LeafElement;
import com.intellij.psi.impl.source.tree.injected.InjectedLanguageUtil;
import com.intellij.psi.tree.TokenSet;
import com.intellij.psi.tree.IElementType;
import com.jetbrains.python.PyTokenTypes;
import com.jetbrains.python.PythonLanguage;
import com.jetbrains.python.highlighting.PySyntaxHighlighterFactory;
import com.jetbrains.python.lexer.PythonHighlightingLexer;
import com.jetbrains.python.psi.LanguageLevel;
import com.jetbrains.python.psi.PyElementVisitor;
import com.jetbrains.python.psi.PyStringLiteralExpression;
@@ -142,7 +147,7 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt
public List<ASTNode> getStringNodes() {
return Arrays.asList(getNode().getChildren(TokenSet.create(PyTokenTypes.STRING_LITERAL)));
return Arrays.asList(getNode().getChildren(PyTokenTypes.STRING_NODES));
}
public String getStringValue() {
@@ -276,18 +281,17 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt
final List<ASTNode> nodes = getStringNodes();
if (nodes.size() > 0) {
String text = getStringNodes().get(0).getText();
if (LanguageLevel.forElement(this).isPy3K()) {
if (isBytes(text)) {
return PyBuiltinCache.getInstance(this).getObjectType("bytes");
}
}
else {
if (isUnicode(text)) {
return PyBuiltinCache.getInstance(this).getObjectType("unicode");
SyntaxHighlighter highlighter = PySyntaxHighlighterFactory.getSyntaxHighlighter(PythonLanguage.getInstance(), getProject(),
getContainingFile().getVirtualFile());
Lexer lexer = highlighter.getHighlightingLexer();
if (lexer instanceof PythonHighlightingLexer) {
IElementType type = ((PythonHighlightingLexer)lexer).convertStringType(getStringNodes().get(0).getElementType(), text);
if (PyTokenTypes.UNICODE_NODES.contains(type)) {
return PyBuiltinCache.getInstance(this).getUnicodeType(LanguageLevel.forElement(this));
}
}
}
return PyBuiltinCache.getInstance(this).getStrType();
return PyBuiltinCache.getInstance(this).getBytesType(LanguageLevel.forElement(this));
}
@NotNull
python/src/com/jetbrains/python/validation/DocStringAnnotator.java
-2
View File
@@ -52,8 +52,6 @@ public class DocStringAnnotator extends PyAnnotator {
if (PydevConsoleRunner.isInPydevConsole(stmt)){
return;
}
Annotation ann = getHolder().createInfoAnnotation(stmt, null);
ann.setTextAttributes(PyHighlighter.PY_DOC_COMMENT);
final PyDocumentationSettings settings = PyDocumentationSettings.getInstance(stmt.getProject());
if (settings.isEpydocFormat(stmt.getContainingFile()) || settings.isReSTFormat(stmt.getContainingFile())) {
python/src/com/jetbrains/python/validation/UnicodeOrByteLiteralAnnotator.java
-3
View File
@@ -74,9 +74,6 @@ public class UnicodeOrByteLiteralAnnotator extends PyAnnotator {
boolean is_unicode = isDefaultUnicode(expr);
is_unicode |= (first_char == 'u');
is_unicode &= (first_char != 'b');
if (is_unicode) {
getHolder().createInfoAnnotation(node, null).setTextAttributes(PyHighlighter.PY_UNICODE_STRING);
}
final boolean is_raw = isRaw(text);
// highlight escapes
Matcher n_matcher = N_ESC_PATTERN.matcher(text);
python/testData/psi/CommentAtBeginningOfStatementList.txt
+1 -1
View File
@@ -32,7 +32,7 @@ PyFile:CommentAtBeginningOfStatementList.py
PsiElement(Py:PRINT_KEYWORD)('print')
PsiWhiteSpace(' ')
PyStringLiteralExpression: no
PsiElement(Py:STRING_LITERAL)(''no'')
PsiElement(Py:SINGLE_QUOTED_STRING)(''no'')
PsiWhiteSpace('\n\n')
PyExpressionStatement
PyCallExpression: foo
python/testData/psi/DecoratedFunction.txt
+2 -2
View File
@@ -18,7 +18,7 @@ PyFile:DecoratedFunction.py
PsiElement(Py:IDENTIFIER)('node')
PsiElement(Py:EQ)('=')
PyStringLiteralExpression: foo
PsiElement(Py:STRING_LITERAL)('"foo"')
PsiElement(Py:SINGLE_QUOTED_STRING)('"foo"')
PsiElement(Py:RPAR)(')')
PsiWhiteSpace('\n')
PsiElement(Py:DEF_KEYWORD)('def')
@@ -31,4 +31,4 @@ PyFile:DecoratedFunction.py
PsiWhiteSpace('\n ')
PyStatementList
PyPassStatement
PsiElement(Py:PASS_KEYWORD)('pass')
PsiElement(Py:PASS_KEYWORD)('pass')
python/testData/psi/DictLiteral.txt
+2 -2
View File
@@ -4,7 +4,7 @@ PyFile:DictLiteral.py
PsiElement(Py:LBRACE)('{')
PyKeyValueExpression
PyStringLiteralExpression: a
PsiElement(Py:STRING_LITERAL)(''a'')
PsiElement(Py:SINGLE_QUOTED_STRING)(''a'')
PsiElement(Py:COLON)(':')
PsiWhiteSpace(' ')
PyNumericLiteralExpression
@@ -13,7 +13,7 @@ PyFile:DictLiteral.py
PsiWhiteSpace(' ')
PyKeyValueExpression
PyStringLiteralExpression: b
PsiElement(Py:STRING_LITERAL)(''b'')
PsiElement(Py:SINGLE_QUOTED_STRING)(''b'')
PsiElement(Py:COLON)(':')
PsiWhiteSpace(' ')
PyNumericLiteralExpression
python/testData/psi/DictMissingComma.txt
+2 -2
View File
@@ -9,7 +9,7 @@ PyFile:DictMissingComma.py
PsiElement(Py:LBRACE)('{')
PyKeyValueExpression
PyStringLiteralExpression: key1
PsiElement(Py:STRING_LITERAL)(''key1'')
PsiElement(Py:SINGLE_QUOTED_STRING)(''key1'')
PsiElement(Py:COLON)(':')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('1')
@@ -18,7 +18,7 @@ PyFile:DictMissingComma.py
PsiWhiteSpace(' ')
PyKeyValueExpression
PyStringLiteralExpression: key2
PsiElement(Py:STRING_LITERAL)(''key2'')
PsiElement(Py:SINGLE_QUOTED_STRING)(''key2'')
PsiElement(Py:COLON)(':')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('2')
python/testData/psi/ErrorInParameterList.txt
+1 -1
View File
@@ -19,7 +19,7 @@ PyFile:ErrorInParameterList.py
PsiErrorElement:formal parameter name expected
PsiElement(Py:FROM_KEYWORD)('from')
PsiElement(Py:EQ)('=')
PsiElement(Py:STRING_LITERAL)('''')
PsiElement(Py:SINGLE_QUOTED_STRING)('''')
PsiElement(Py:RPAR)(')')
PsiElement(Py:COLON)(':')
PsiWhiteSpace('\n ')
python/testData/psi/ExecPy2.txt
+2 -2
View File
@@ -3,7 +3,7 @@ PyFile:ExecPy2.py
PsiElement(Py:EXEC_KEYWORD)('exec')
PsiWhiteSpace(' ')
PyStringLiteralExpression: print 1
PsiElement(Py:STRING_LITERAL)(''print 1'')
PsiElement(Py:SINGLE_QUOTED_STRING)(''print 1'')
PsiWhiteSpace(' ')
PsiElement(Py:IN_KEYWORD)('in')
PsiWhiteSpace(' ')
@@ -14,4 +14,4 @@ PyFile:ExecPy2.py
PsiWhiteSpace(' ')
PyDictLiteralExpression
PsiElement(Py:LBRACE)('{')
PsiElement(Py:RBRACE)('}')
PsiElement(Py:RBRACE)('}')
python/testData/psi/GeneratorInArgumentList.txt
+2 -2
View File
@@ -3,14 +3,14 @@ PyFile:GeneratorInArgumentList.py
PyCallExpression: "".join
PyReferenceExpression: join
PyStringLiteralExpression:
PsiElement(Py:STRING_LITERAL)('""')
PsiElement(Py:SINGLE_QUOTED_STRING)('""')
PsiElement(Py:DOT)('.')
PsiElement(Py:IDENTIFIER)('join')
PyArgumentList
PsiElement(Py:LPAR)('(')
PyGeneratorExpression
PyStringLiteralExpression:
PsiElement(Py:STRING_LITERAL)('' '')
PsiElement(Py:SINGLE_QUOTED_STRING)('' '')
PsiWhiteSpace(' ')
PsiElement(Py:FOR_KEYWORD)('for')
PsiWhiteSpace(' ')
python/testData/psi/HelloWorld.txt
+1 -1
View File
@@ -3,4 +3,4 @@ PyFile:HelloWorld.py
PsiElement(Py:PRINT_KEYWORD)('print')
PsiWhiteSpace(' ')
PyStringLiteralExpression: hello, world
PsiElement(Py:STRING_LITERAL)('"hello, world"')
PsiElement(Py:SINGLE_QUOTED_STRING)('"hello, world"')
python/testData/psi/IncompleteDict.txt
+1 -1
View File
@@ -10,7 +10,7 @@ PyFile:IncompleteDict.py
PsiWhiteSpace(' ')
PyKeyValueExpression
PyStringLiteralExpression: key
PsiElement(Py:STRING_LITERAL)(''key'')
PsiElement(Py:SINGLE_QUOTED_STRING)(''key'')
PsiElement(Py:COLON)(':')
PsiErrorElement:expression expected
<empty list>
python/testData/psi/LongString.txt
+4 -4
View File
@@ -6,7 +6,7 @@ PyFile:LongString.py
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression:
PsiElement(Py:STRING_LITERAL)('""""""')
PsiElement(Py:TRIPLE_QUOTED_STRING)('""""""')
PsiWhiteSpace('\n')
PyAssignmentStatement
PyTargetExpression: b
@@ -17,7 +17,7 @@ PyFile:LongString.py
PyStringLiteralExpression:
"1.0" encoding="ascii"?
PsiElement(Py:STRING_LITERAL)('"""\n"1.0" encoding="ascii"?\n"""')
PsiElement(Py:TRIPLE_QUOTED_STRING)('"""\n"1.0" encoding="ascii"?\n"""')
PsiWhiteSpace('\n')
PyAssignmentStatement
PyTargetExpression: str
@@ -34,11 +34,11 @@ PyFile:LongString.py
PyArgumentList
PsiElement(Py:LPAR)('(')
PyStringLiteralExpression: \\\\
PsiElement(Py:STRING_LITERAL)('r"""\\\\"""')
PsiElement(Py:TRIPLE_QUOTED_STRING)('r"""\\\\"""')
PsiElement(Py:COMMA)(',')
PsiWhiteSpace(' ')
PyStringLiteralExpression: &#92;
PsiElement(Py:STRING_LITERAL)('r"""&#92;"""')
PsiElement(Py:TRIPLE_QUOTED_STRING)('r"""&#92;"""')
PsiElement(Py:COMMA)(',')
PsiWhiteSpace(' ')
PyReferenceExpression: str
python/testData/psi/PrintAsFunction26.txt
+1 -1
View File
@@ -18,5 +18,5 @@ PyFile:PrintAsFunction26.py
PyArgumentList
PsiElement(Py:LPAR)('(')
PyStringLiteralExpression: a
PsiElement(Py:STRING_LITERAL)(''a'')
PsiElement(Py:SINGLE_QUOTED_STRING)(''a'')
PsiElement(Py:RPAR)(')')
python/testData/psi/TrailingCommaInList.txt
+2 -2
View File
@@ -8,9 +8,9 @@ PyFile:TrailingCommaInList.py
PyListLiteralExpression
PsiElement(Py:LBRACKET)('[')
PyStringLiteralExpression: a
PsiElement(Py:STRING_LITERAL)(''a'')
PsiElement(Py:SINGLE_QUOTED_STRING)(''a'')
PsiElement(Py:COMMA)(',')
PyStringLiteralExpression: b
PsiElement(Py:STRING_LITERAL)(''b'')
PsiElement(Py:SINGLE_QUOTED_STRING)(''b'')
PsiElement(Py:COMMA)(',')
PsiElement(Py:RBRACKET)(']')
python/testData/psi/TryExceptAs.txt
+1 -1
View File
@@ -17,7 +17,7 @@ PyFile:TryExceptAs.py
PyArgumentList
PsiElement(Py:LPAR)('(')
PyStringLiteralExpression: myfile.txt
PsiElement(Py:STRING_LITERAL)(''myfile.txt'')
PsiElement(Py:SINGLE_QUOTED_STRING)(''myfile.txt'')
PsiElement(Py:RPAR)(')')
PsiWhiteSpace('\n')
PyExceptPart
python/testData/wrap/WrapInDocstring.after.py
+2 -1
View File
@@ -2,4 +2,5 @@ def foo():
"""
Gallia est omnis divisa in partes tres, quarum unam incolumnt Belgae,
aliam Aquitani
"""
"""
pass
python/testData/wrap/WrapInDocstring.py
+2 -1
View File
@@ -1,4 +1,5 @@
def foo():
"""
Gallia est omnis divisa in partes tres, quarum unam incolumnt Belgae, aliam <caret>
"""
"""
pass
python/testSrc/com/jetbrains/python/PythonHighlightingLexerTest.java
+151
View File
@@ -0,0 +1,151 @@
package com.jetbrains.python;
import com.jetbrains.python.fixtures.PyLexerTestCase;
import com.jetbrains.python.lexer.PythonHighlightingLexer;
import com.jetbrains.python.psi.LanguageLevel;
/**
* user : catherine
*/
public class PythonHighlightingLexerTest extends PyLexerTestCase {
public void testFromFutureUnicode() {
doTest(LanguageLevel.PYTHON26, "from __future__ import unicode_literals\n\n" +
"s = \"some string\"",
"Py:FROM_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:SPACE", "Py:IMPORT_KEYWORD", "Py:SPACE", "Py:IDENTIFIER",
"Py:LINE_BREAK", "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_UNICODE");
}
public void testFromFutureUnicodeDocstring() {
doTest(LanguageLevel.PYTHON26, "\"\"\"docstring\"\"\"\n" +
"from __future__ import unicode_literals\n\n" +
"s = \"some string\"",
"Py:DOCSTRING", "Py:LINE_BREAK",
"Py:FROM_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:SPACE", "Py:IMPORT_KEYWORD", "Py:SPACE", "Py:IDENTIFIER",
"Py:LINE_BREAK", "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_UNICODE");
}
public void testFromFutureUnicodeWithBackslash() {
doTest(LanguageLevel.PYTHON26, "from __future__ \\\nimport unicode_literals\n\n" +
"s = \"some string\"",
"Py:FROM_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:SPACE", "Py:BACKSLASH", "Py:LINE_BREAK",
"Py:IMPORT_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:LINE_BREAK",
"Py:LINE_BREAK", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_UNICODE");
}
public void testFromFutureUnicodeWithBrace() {
doTest(LanguageLevel.PYTHON26, "from __future__ import (unicode_literals)\n\n" +
"s = \"some string\"",
"Py:FROM_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:SPACE",
"Py:IMPORT_KEYWORD", "Py:SPACE", "Py:LPAR", "Py:IDENTIFIER", "Py:RPAR", "Py:LINE_BREAK",
"Py:LINE_BREAK", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_UNICODE");
}
public void testFromFutureUnicodeComment() {
doTest(LanguageLevel.PYTHON26, "#one comment\n" +
"from __future__ import ((unicode_literals))\n" +
"s = \"some string\"",
"Py:END_OF_LINE_COMMENT", "Py:LINE_BREAK", "Py:FROM_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:SPACE",
"Py:IMPORT_KEYWORD", "Py:SPACE", "Py:LPAR", "Py:LPAR", "Py:IDENTIFIER", "Py:RPAR", "Py:RPAR",
"Py:LINE_BREAK", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_STRING");
}
public void testFromFutureBytes() {
doTest(LanguageLevel.PYTHON26, "from __future__ import unicode_literals\n" +
"s = b\"some string\"",
"Py:FROM_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:SPACE",
"Py:IMPORT_KEYWORD", "Py:SPACE", "Py:IDENTIFIER",
"Py:LINE_BREAK", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_STRING");
}
public void testBytes30() {
doTest(LanguageLevel.PYTHON30, "s = b\"some string\"",
"Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_STRING");
}
public void testBytes() {
doTest(LanguageLevel.PYTHON26, "s = b\"some string\"",
"Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_STRING");
}
public void testFromFutureUnicodeWithBraceFail() {
doTest(LanguageLevel.PYTHON26, "from __future__ import ((unicode_literals))\n" +
"s = \"some string\"",
"Py:FROM_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:SPACE", "Py:IMPORT_KEYWORD", "Py:SPACE",
"Py:LPAR", "Py:LPAR", "Py:IDENTIFIER", "Py:RPAR", "Py:RPAR",
"Py:LINE_BREAK", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_STRING");
}
public void testDoubleFromFutureUnicode() {
doTest(LanguageLevel.PYTHON26, "from __future__ import absolute_import\n" +
"from __future__ import (unicode_literals)\n\n" +
"s = \"some string\"",
"Py:FROM_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:SPACE", "Py:IMPORT_KEYWORD", "Py:SPACE", "Py:IDENTIFIER",
"Py:LINE_BREAK", "Py:FROM_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:SPACE",
"Py:IMPORT_KEYWORD", "Py:SPACE", "Py:LPAR", "Py:IDENTIFIER", "Py:RPAR", "Py:LINE_BREAK",
"Py:LINE_BREAK", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_UNICODE");
}
public void testDoubleFromFutureUnicodeWithComma() {
doTest(LanguageLevel.PYTHON26, "from __future__ import absolute_import, unicode_literals\n" +
"s = \"some string\"",
"Py:FROM_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:SPACE", "Py:IMPORT_KEYWORD", "Py:SPACE", "Py:IDENTIFIER",
"Py:COMMA", "Py:SPACE", "Py:IDENTIFIER",
"Py:LINE_BREAK", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_UNICODE");
}
public void testDoubleFromFutureUnicodeWithCommaFail() {
doTest(LanguageLevel.PYTHON26, "from __future__ import absolute_import, (unicode_literals)\n" +
"s = \"some string\"",
"Py:FROM_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:SPACE", "Py:IMPORT_KEYWORD", "Py:SPACE", "Py:IDENTIFIER",
"Py:COMMA", "Py:SPACE", "Py:LPAR", "Py:IDENTIFIER", "Py:RPAR",
"Py:LINE_BREAK", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_STRING");
}
public void testFromFutureUnicodeFail() {
doTest(LanguageLevel.PYTHON26, "a = 2\n" +
"from __future__ import unicode_literals\n\n" +
"s = \"some string\"",
"Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE","Py:INTEGER_LITERAL", "Py:LINE_BREAK",
"Py:FROM_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:SPACE", "Py:IMPORT_KEYWORD", "Py:SPACE", "Py:IDENTIFIER",
"Py:LINE_BREAK", "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_STRING");
}
public void testUnicode30() {
doTest(LanguageLevel.PYTHON30, "s = \"some string\"",
"Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_UNICODE");
}
public void testString() {
doTest(LanguageLevel.PYTHON27, "s = \"some string\"",
"Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_STRING");
}
public void testUnicode() {
doTest(LanguageLevel.PYTHON27, "s = u\"some string\"",
"Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_UNICODE");
}
public void testDocString() {
doTest(LanguageLevel.PYTHON27, "\"\"\"one docstrings \"\"\"\n",
"Py:DOCSTRING", "Py:LINE_BREAK");
}
public void testSingleDocStringWithBackslash() {
doTest(LanguageLevel.PYTHON27, "\"one docstring \" \\\n\"new line of docstring\"\n",
"Py:DOCSTRING", "Py:BACKSLASH", "Py:LINE_BREAK", "Py:DOCSTRING", "Py:LINE_BREAK");
}
public void testSingleDocstringFunction() {
doTest(LanguageLevel.PYTHON27, "def foo():\n" +
" \"\"\"function foo\"\"\"\n" +
" a = \"string\"",
"Py:DEF_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:LPAR", "Py:RPAR", "Py:COLON", "Py:LINE_BREAK",
"Py:SPACE", "Py:SPACE", "Py:DOCSTRING", "Py:LINE_BREAK", "Py:SPACE", "Py:SPACE", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE",
"Py:SINGLE_QUOTED_STRING");
}
private static void doTest(LanguageLevel languageLevel, String text, String... expectedTokens) {
doLexerTest(text, new PythonHighlightingLexer(languageLevel), expectedTokens);
}
}
python/testSrc/com/jetbrains/python/PythonHighlightingTest.java
+27 -27
View File
@@ -60,18 +60,18 @@ public class PythonHighlightingTest extends PyLightFixtureTestCase {
doTest();
}
public void testDocStrings() {
EditorColorsManager manager = EditorColorsManager.getInstance();
EditorColorsScheme scheme = (EditorColorsScheme)manager.getGlobalScheme().clone();
manager.addColorsScheme(scheme);
EditorColorsManager.getInstance().setGlobalScheme(scheme);
TextAttributesKey xKey = TextAttributesKey.find("PY.DOC_COMMENT");
TextAttributes xAttributes = new TextAttributes(Color.blue, Color.black, Color.white, EffectType.BOXED, Font.BOLD);
scheme.setAttributes(xKey, xAttributes);
doTest();
}
//public void testDocStrings() {
// EditorColorsManager manager = EditorColorsManager.getInstance();
// EditorColorsScheme scheme = (EditorColorsScheme)manager.getGlobalScheme().clone();
// manager.addColorsScheme(scheme);
// EditorColorsManager.getInstance().setGlobalScheme(scheme);
//
// TextAttributesKey xKey = TextAttributesKey.find("PY.DOC_COMMENT");
// TextAttributes xAttributes = new TextAttributes(Color.blue, Color.black, Color.white, EffectType.BOXED, Font.BOLD);
// scheme.setAttributes(xKey, xAttributes);
//
// doTest();
//}
public void testAssignmentTargets() {
setLanguageLevel(LanguageLevel.PYTHON26);
@@ -166,21 +166,21 @@ public class PythonHighlightingTest extends PyLightFixtureTestCase {
doTest();
}
public void testUnicodeOrByteRaw() {
doTest(LanguageLevel.PYTHON25, true, true);
}
public void testUnicodeOrByte25() {
doTest(LanguageLevel.PYTHON25, true, true);
}
public void testUnicodeOrByte26future() {
doTest(LanguageLevel.PYTHON26, true, true);
}
public void testUnicodeOrByte30() {
doTest(LanguageLevel.PYTHON30, true, true);
}
//public void testUnicodeOrByteRaw() {
// doTest(LanguageLevel.PYTHON25, true, true);
//}
//
//public void testUnicodeOrByte25() {
// doTest(LanguageLevel.PYTHON25, true, true);
//}
//
//public void testUnicodeOrByte26future() {
// doTest(LanguageLevel.PYTHON26, true, true);
//}
//
//public void testUnicodeOrByte30() {
// doTest(LanguageLevel.PYTHON30, true, true);
//}
// ---
python/testSrc/com/jetbrains/python/PythonLexerTest.java
+10 -10
View File
@@ -113,7 +113,7 @@ public class PythonLexerTest extends PyLexerTestCase {
}
public void testBytesLiteral() {
doTest("b'ABC'", "Py:STRING_LITERAL");
doTest("b'ABC'", "Py:SINGLE_QUOTED_STRING");
}
public void testOctalLiteral() {
@@ -128,15 +128,15 @@ public class PythonLexerTest extends PyLexerTestCase {
public void testLongString() {
doTest(THREE_QUOTES + THREE_QUOTES + "\nb=" + THREE_QUOTES + THREE_QUOTES,
"Py:STRING_LITERAL", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:EQ", "Py:STRING_LITERAL");
"Py:DOCSTRING", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:EQ", "Py:TRIPLE_QUOTED_STRING");
}
public void testLongStringEscaped() {
doTest("''' \\'''foo\\''' ''';", "Py:STRING_LITERAL", "Py:SEMICOLON");
doTest("''' \\'''foo\\''' ''';", "Py:DOCSTRING", "Py:SEMICOLON");
}
public void testLongStringWithLineBreak() {
doTest(THREE_QUOTES + "\\\na\n\n" + THREE_QUOTES + ";", "Py:STRING_LITERAL", "Py:SEMICOLON");
doTest(THREE_QUOTES + "\\\na\n\n" + THREE_QUOTES + ";", "Py:DOCSTRING", "Py:SEMICOLON");
}
@@ -149,25 +149,25 @@ public class PythonLexerTest extends PyLexerTestCase {
}
public void testBackslashBeforeEmptyLine() {
doTest("x=\"a\" + \\\n \"b\" \\\n\nprint x", "Py:IDENTIFIER", "Py:EQ", "Py:STRING_LITERAL", "Py:SPACE", "Py:PLUS", "Py:SPACE", "Py:LINE_BREAK",
"Py:STRING_LITERAL", "Py:SPACE", "Py:STATEMENT_BREAK", "Py:LINE_BREAK",
doTest("x=\"a\" + \\\n \"b\" \\\n\nprint x", "Py:IDENTIFIER", "Py:EQ", "Py:SINGLE_QUOTED_STRING", "Py:SPACE", "Py:PLUS", "Py:SPACE", "Py:LINE_BREAK",
"Py:SINGLE_QUOTED_STRING", "Py:SPACE", "Py:STATEMENT_BREAK", "Py:LINE_BREAK",
"Py:IDENTIFIER" , "Py:SPACE", "Py:IDENTIFIER");
}
public void testIncompleteTripleQuotedString() { // PY-1768
doTest("'''abc\nd", "Py:STRING_LITERAL");
doTest("tmp = '''abc\nd", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:TRIPLE_QUOTED_STRING");
}
public void testEscapedClosingTripleApos() { // PY-1777
doTest("''' foo '\\''' bar '''", "Py:STRING_LITERAL");
doTest("''' foo '\\''' bar '''", "Py:TRIPLE_QUOTED_STRING");
}
public void testEscapedClosingTripleQuote() { // PY-1777
doTest(THREE_QUOTES + " foo \"\\" + THREE_QUOTES + " bar " + THREE_QUOTES, "Py:STRING_LITERAL");
doTest(THREE_QUOTES + " foo \"\\" + THREE_QUOTES + " bar " + THREE_QUOTES, "Py:TRIPLE_QUOTED_STRING");
}
public void testOddNumberOfQuotes() { // PY-2802
doTest("'''foo''''", "Py:STRING_LITERAL", "Py:STRING_LITERAL");
doTest("'''foo''''", "Py:TRIPLE_QUOTED_STRING", "Py:SINGLE_QUOTED_STRING");
}
public void testDedentBeforeComment() { // PY-2209 & friends