fixed PY-9274 Broken raw bytes literal parsing for Python 3.3

refactored lexer, added compatibility warnings, quickfix to remove not only U prefix
This commit is contained in:
Ekaterina Tuzova
2013-03-29 16:44:08 +04:00
parent ade6f46162
commit ab9622ffc6
12 changed files with 745 additions and 624 deletions
@@ -140,7 +140,8 @@ INTN.convert.dict.comp.to=Convert dictionary comprehension to 'dict' method call
INTN.replace.noteq.operator=Replace not equal operator
INTN.remove.leading.u=Remove leading U
INTN.remove.leading.$0=Remove leading {0}
INTN.remove.leading.prefix=Remove prefix
INTN.remove.trailing.l=Remove trailing L
@@ -7,6 +7,7 @@ import com.intellij.openapi.application.ApplicationManager;
import com.intellij.openapi.roots.ProjectFileIndex;
import com.intellij.openapi.roots.ProjectRootManager;
import com.intellij.openapi.util.JDOMExternalizableStringList;
import com.intellij.openapi.util.TextRange;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.psi.PsiElement;
import com.intellij.psi.PsiElementVisitor;
@@ -128,14 +129,22 @@ public class PyCompatibilityInspection extends PyInspection {
@Override
protected final void registerProblem(@Nullable final PsiElement element,
@NotNull final String message,
@Nullable final LocalQuickFix quickFix, final boolean asError){
if (element == null || element.getTextLength() == 0){
return;
@Nullable final LocalQuickFix quickFix, final boolean asError) {
if (element == null) return;
registerProblem(element, element.getTextRange(), message, quickFix, asError);
}
@Override
protected void registerProblem(@NotNull final PsiElement element, @NotNull TextRange range, String message,
@Nullable LocalQuickFix quickFix, boolean asError) {
if (element.getTextLength() == 0) {
return;
}
range = TextRange.create(range.getStartOffset() - element.getTextOffset(), range.getEndOffset() - element.getTextOffset());
if (quickFix != null)
myHolder.registerProblem(element, message, quickFix);
myHolder.registerProblem(element, range, message, quickFix);
else
myHolder.registerProblem(element, message);
myHolder.registerProblem(element, range, message);
}
@Override
@@ -7,6 +7,7 @@ import com.intellij.psi.PsiElement;
import com.jetbrains.python.PyBundle;
import com.jetbrains.python.psi.PyElementGenerator;
import com.jetbrains.python.psi.PyStringLiteralExpression;
import com.jetbrains.python.psi.impl.PyStringLiteralExpressionImpl;
import org.jetbrains.annotations.NotNull;
/**
@@ -15,16 +16,23 @@ import org.jetbrains.annotations.NotNull;
* Date: 06.03.2010
* Time: 16:50:53
*/
public class RemoveLeadingUQuickFix implements LocalQuickFix {
public class RemovePrefixQuickFix implements LocalQuickFix {
private final String myPrefix;
public RemovePrefixQuickFix(String prefix) {
myPrefix = prefix;
}
@NotNull
@Override
public String getName() {
return PyBundle.message("INTN.remove.leading.u");
return PyBundle.message("INTN.remove.leading.$0", myPrefix);
}
@NotNull
public String getFamilyName() {
return getName();
return PyBundle.message("INTN.remove.leading.prefix");
}
@Override
@@ -32,7 +40,8 @@ public class RemoveLeadingUQuickFix implements LocalQuickFix {
PsiElement stringLiteralExpression = descriptor.getPsiElement();
if (stringLiteralExpression instanceof PyStringLiteralExpression) {
PyElementGenerator elementGenerator = PyElementGenerator.getInstance(project);
stringLiteralExpression.replace(elementGenerator.createExpressionFromText(stringLiteralExpression.getText().substring(1)));
final int length = PyStringLiteralExpressionImpl.getPrefixLength(stringLiteralExpression.getText());
stringLiteralExpression.replace(elementGenerator.createExpressionFromText(stringLiteralExpression.getText().substring(length)));
}
}
}
@@ -57,22 +57,18 @@ public class PyStringLiteralLexer extends LexerBase {
// the following could be parsing steps if we wanted this info as tokens
int i = myStart;
// unicode flag
char c = buffer.charAt(i);
if (c == 'u' || c == 'U' || c == 'b' || c == 'B' || c == 'c' || c == 'C')
i += 1;
// raw flag
c = buffer.charAt(i);
if (c == 'r' || c == 'R') {
myIsRaw = true;
i += 1;
}
else myIsRaw = false;
i = skipEncodingPrefix(buffer, i);
int offset = skipRawPrefix(buffer, i);
if (offset > i) myIsRaw = true;
i = offset;
i = skipEncodingPrefix(buffer, i);
offset = skipRawPrefix(buffer, i);
if (offset > i) myIsRaw = true;
i = offset;
// which quote char?
c = buffer.charAt(i);
char c = buffer.charAt(i);
assert (c == '"') || (c == '\'') : "String must be quoted by single or double quote";
myQuoteChar = c;
@@ -80,6 +76,22 @@ public class PyStringLiteralLexer extends LexerBase {
myEnd = locateToken(myStart);
}
public static int skipRawPrefix(CharSequence text, int startOffset) {
char c = Character.toUpperCase(text.charAt(startOffset));
if (c == 'R') {
startOffset++;
}
return startOffset;
}
public static int skipEncodingPrefix(CharSequence text, int startOffset) {
char c = Character.toUpperCase(text.charAt(startOffset));
if (c == 'U' || c == 'B' || c == 'C') {
startOffset++;
}
return startOffset;
}
public int getState() {
return myLastState;
}
@@ -48,8 +48,8 @@ IMAGNUMBER=(({FLOATNUMBER})|({INTPART}))[Jj]
//RAW_STRING=[Rr]{QUOTED_STRING}
//QUOTED_STRING=({TRIPLE_APOS_LITERAL})|({QUOTED_LITERAL})|({DOUBLE_QUOTED_LITERAL})|({TRIPLE_QUOTED_LITERAL})
SINGLE_QUOTED_STRING=[UuBbCc]?[Rr]?({QUOTED_LITERAL} | {DOUBLE_QUOTED_LITERAL})
TRIPLE_QUOTED_STRING=[UuBbCc]?[Rr]?({TRIPLE_QUOTED_LITERAL}|{TRIPLE_APOS_LITERAL})
SINGLE_QUOTED_STRING=[UuBbCcRr]{0,2}({QUOTED_LITERAL} | {DOUBLE_QUOTED_LITERAL})
TRIPLE_QUOTED_STRING=[UuBbCcRr]{0,2}[UuBbCcRr]?({TRIPLE_QUOTED_LITERAL}|{TRIPLE_APOS_LITERAL})
DOCSTRING_LITERAL=({SINGLE_QUOTED_STRING}|{TRIPLE_QUOTED_STRING})
File diff suppressed because it is too large Load Diff
@@ -13,6 +13,7 @@ import com.intellij.psi.tree.IElementType;
import com.intellij.psi.util.PsiTreeUtil;
import com.jetbrains.python.PyTokenTypes;
import com.jetbrains.python.codeInsight.regexp.PythonVerboseRegexpLanguage;
import com.jetbrains.python.lexer.PyStringLiteralLexer;
import com.jetbrains.python.lexer.PythonHighlightingLexer;
import com.jetbrains.python.psi.*;
import com.jetbrains.python.psi.types.PyType;
@@ -100,34 +101,20 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt
public static int getPrefixLength(String text) {
int startOffset = 0;
startOffset = skipEncodingPrefix(text, startOffset);
startOffset = skipRawPrefix(text, startOffset);
return startOffset;
}
private static int skipRawPrefix(String text, int startOffset) {
char c = Character.toUpperCase(text.charAt(startOffset));
if (c == 'R') {
startOffset++;
}
return startOffset;
}
private static int skipEncodingPrefix(String text, int startOffset) {
char c = Character.toUpperCase(text.charAt(startOffset));
if (c == 'U' || c == 'B' || c == 'C') {
startOffset++;
}
startOffset = PyStringLiteralLexer.skipEncodingPrefix(text, startOffset);
startOffset = PyStringLiteralLexer.skipRawPrefix(text, startOffset);
startOffset = PyStringLiteralLexer.skipEncodingPrefix(text, startOffset);
startOffset = PyStringLiteralLexer.skipRawPrefix(text, startOffset);
return startOffset;
}
private static boolean isRaw(String text) {
int startOffset = skipEncodingPrefix(text, 0);
return skipRawPrefix(text, startOffset) > startOffset;
int startOffset = PyStringLiteralLexer.skipEncodingPrefix(text, 0);
return PyStringLiteralLexer.skipRawPrefix(text, startOffset) > startOffset;
}
private static boolean isUnicode(String text) {
return text.length() > 0 && Character.toUpperCase(text.charAt(0)) == 'U';
return text.length() > 0 && Character.toUpperCase(text.charAt(0)) == 'U'; //TODO[ktisha]
}
private static boolean isBytes(String text) {
@@ -1,8 +1,12 @@
package com.jetbrains.python.validation;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.intellij.codeInspection.LocalQuickFix;
import com.intellij.lang.ASTNode;
import com.intellij.openapi.roots.ProjectFileIndex;
import com.intellij.openapi.roots.ProjectRootManager;
import com.intellij.openapi.util.TextRange;
import com.intellij.psi.PsiElement;
import com.intellij.psi.PsiFile;
import com.intellij.psi.PsiWhiteSpace;
@@ -12,12 +16,10 @@ import com.jetbrains.python.PyTokenTypes;
import com.jetbrains.python.inspections.quickfix.*;
import com.jetbrains.python.psi.*;
import com.jetbrains.python.psi.impl.PyQualifiedName;
import com.jetbrains.python.psi.impl.PyStringLiteralExpressionImpl;
import org.jetbrains.annotations.Nullable;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.Stack;
import java.util.*;
/**
* User : catherine
@@ -26,6 +28,19 @@ public abstract class CompatibilityVisitor extends PyAnnotator {
protected List<LanguageLevel> myVersionsToProcess;
private String myCommonMessage = "Python version ";
private static final Map<LanguageLevel, Set<String>> AVAILABLE_PREFIXES = Maps.newHashMap();
static {
AVAILABLE_PREFIXES.put(LanguageLevel.PYTHON24, Sets.newHashSet("R", "U", "UR"));
AVAILABLE_PREFIXES.put(LanguageLevel.PYTHON25, Sets.newHashSet("R", "U", "UR"));
AVAILABLE_PREFIXES.put(LanguageLevel.PYTHON26, Sets.newHashSet("R", "U", "UR", "B", "BR"));
AVAILABLE_PREFIXES.put(LanguageLevel.PYTHON27, Sets.newHashSet("R", "U", "UR", "B", "BR"));
AVAILABLE_PREFIXES.put(LanguageLevel.PYTHON30, Sets.newHashSet("R", "B"));
AVAILABLE_PREFIXES.put(LanguageLevel.PYTHON31, Sets.newHashSet("R", "B", "BR"));
AVAILABLE_PREFIXES.put(LanguageLevel.PYTHON32, Sets.newHashSet("R", "B", "BR"));
AVAILABLE_PREFIXES.put(LanguageLevel.PYTHON33, Sets.newHashSet("R", "U", "B", "BR", "RB"));
}
public CompatibilityVisitor(List<LanguageLevel> versionsToProcess) {
myVersionsToProcess = versionsToProcess;
}
@@ -213,19 +228,25 @@ public abstract class CompatibilityVisitor extends PyAnnotator {
@Override
public void visitPyStringLiteralExpression(final PyStringLiteralExpression node) {
super.visitPyStringLiteralExpression(node);
int len = 0;
StringBuilder message = new StringBuilder(myCommonMessage);
for (int i = 0; i != myVersionsToProcess.size(); ++i) {
LanguageLevel languageLevel = myVersionsToProcess.get(i);
List<ASTNode> stringNodes = node.getStringNodes();
if (languageLevel.isAtLeast(LanguageLevel.PYTHON30) && languageLevel.isOlderThan(LanguageLevel.PYTHON33)) {
final String text = node.getText();
if (text.startsWith("u") || text.startsWith("U")) {
for (ASTNode stringNode : stringNodes) {
int len = 0;
StringBuilder message = new StringBuilder(myCommonMessage);
String nodeText = stringNode.getText();
int index = PyStringLiteralExpressionImpl.getPrefixLength(nodeText);
String prefix = nodeText.substring(0, index).toUpperCase();
final TextRange range = TextRange.create(stringNode.getStartOffset(), stringNode.getStartOffset() + index);
for (int i = 0; i != myVersionsToProcess.size(); ++i) {
LanguageLevel languageLevel = myVersionsToProcess.get(i);
if (prefix.isEmpty()) continue;
final Set<String> prefixes = AVAILABLE_PREFIXES.get(languageLevel);
if (!prefixes.contains(prefix))
len = appendLanguageLevel(message, len, languageLevel);
}
}
commonRegisterProblem(message, " not support a '" + prefix + "' prefix", len, node, range, new RemovePrefixQuickFix(prefix));
}
commonRegisterProblem(message, " not support a leading \'u\' or \'U\'.", len, node, new RemoveLeadingUQuickFix());
}
@Override
@@ -552,6 +573,7 @@ public abstract class CompatibilityVisitor extends PyAnnotator {
}
protected abstract void registerProblem(PsiElement node, String s, @Nullable LocalQuickFix localQuickFix, boolean asError);
protected abstract void registerProblem(PsiElement node, TextRange range, String s, @Nullable LocalQuickFix localQuickFix, boolean asError);
protected void registerProblem(final PsiElement node, final String s, @Nullable final LocalQuickFix localQuickFix) {
registerProblem(node, s, localQuickFix, true);
@@ -567,7 +589,22 @@ public abstract class CompatibilityVisitor extends PyAnnotator {
protected void commonRegisterProblem(StringBuilder initMessage, String suffix,
int len, PyElement node, LocalQuickFix localQuickFix) {
commonRegisterProblem(initMessage, suffix, len, node, localQuickFix, true);
commonRegisterProblem(initMessage, suffix, len, node, node.getTextRange(), localQuickFix, true);
}
protected void commonRegisterProblem(StringBuilder initMessage, String suffix,
int len, PyElement node, TextRange range, LocalQuickFix localQuickFix) {
commonRegisterProblem(initMessage, suffix, len, node, range, localQuickFix, true);
}
protected void commonRegisterProblem(StringBuilder initMessage, String suffix,
int len, PyElement node, TextRange range, @Nullable LocalQuickFix localQuickFix, boolean asError) {
initMessage.append(" do");
if (len == 1)
initMessage.append("es");
initMessage.append(suffix);
if (len != 0)
registerProblem(node, range, initMessage.toString(), localQuickFix, asError);
}
protected void commonRegisterProblem(StringBuilder initMessage, String suffix,
@@ -577,7 +614,7 @@ public abstract class CompatibilityVisitor extends PyAnnotator {
initMessage.append("es");
initMessage.append(suffix);
if (len != 0)
registerProblem(node, initMessage.toString(), localQuickFix, asError);
registerProblem(node, node.getTextRange(), initMessage.toString(), localQuickFix, asError);
}
protected static int appendLanguageLevel(StringBuilder message, int len, LanguageLevel languageLevel) {
@@ -7,11 +7,13 @@ import com.intellij.codeInspection.ProblemDescriptor;
import com.intellij.codeInspection.ProblemHighlightType;
import com.intellij.codeInspection.ex.ProblemDescriptorImpl;
import com.intellij.codeInspection.ex.QuickFixWrapper;
import com.intellij.openapi.util.TextRange;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.psi.PsiElement;
import com.jetbrains.python.psi.LanguageLevel;
import com.jetbrains.python.psi.PyElement;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.util.ArrayList;
import java.util.Arrays;
@@ -31,20 +33,26 @@ public class UnsupportedFeatures extends CompatibilityVisitor {
}
@Override
protected void registerProblem(PsiElement node, String message, LocalQuickFix localQuickFix, boolean asError) {
if (node == null || node.getTextLength() == 0){
return;
protected void registerProblem(@Nullable final PsiElement node, String message, LocalQuickFix localQuickFix, boolean asError) {
if (node == null) return;
registerProblem(node, node.getTextRange(), message, localQuickFix, asError);
}
@Override
protected void registerProblem(PsiElement node, TextRange range, String message, LocalQuickFix localQuickFix, boolean asError) {
if (range.isEmpty()){
return;
}
if (localQuickFix != null)
if (asError)
getHolder().createErrorAnnotation(node, message).registerFix(createIntention(node, message, localQuickFix));
getHolder().createErrorAnnotation(range, message).registerFix(createIntention(node, message, localQuickFix));
else
getHolder().createWarningAnnotation(node, message).registerFix(createIntention(node, message, localQuickFix));
getHolder().createWarningAnnotation(range, message).registerFix(createIntention(node, message, localQuickFix));
else
if (asError)
getHolder().createErrorAnnotation(node, message);
getHolder().createErrorAnnotation(range, message);
else
getHolder().createWarningAnnotation(node, message);
getHolder().createWarningAnnotation(range, message);
}
@NotNull
@@ -57,10 +65,14 @@ public class UnsupportedFeatures extends CompatibilityVisitor {
}
private static IntentionAction createIntention(PsiElement node, String message, LocalQuickFix fix) {
return createIntention(node, node.getTextRange(), message, fix);
}
private static IntentionAction createIntention(PsiElement node, TextRange range, String message, LocalQuickFix fix) {
LocalQuickFix[] quickFixes = {fix};
CommonProblemDescriptorImpl descr = new ProblemDescriptorImpl(node, node, message,
quickFixes, ProblemHighlightType.GENERIC_ERROR_OR_WARNING, true,
node.getTextRange(), true);
range, true);
return QuickFixWrapper.wrap((ProblemDescriptor)descr, 0);
}
}
@@ -5,7 +5,7 @@ a = <error descr="Python version 3.0 does not support this syntax. It requires '
a = 0X43
a = 0b1
a = 0.0
s = <error descr="Python version 3.0 does not support a leading 'u' or 'U'.">u"text"</error>
s = <error descr="Python version 3.0 does not support a 'U' prefix">u</error>"text"
<error descr="Python version 3.0 does not support this syntax.">raise a, b, c</error>
<error descr="Python version 3.0 does not support this syntax.">raise a, b</error>
@@ -1 +1,42 @@
a = <warning descr="Python version 3.0, 3.1, 3.2 do not support a leading 'u' or 'U'.">u"String"</warning>
a = <warning descr="Python version 3.0, 3.1, 3.2 do not support a 'U' prefix">u</warning>"String"
# python 3.3
a = <warning descr="Python version 3.0, 3.1, 3.2 do not support a 'U' prefix">u</warning>""
a = r""
a = <warning descr="Python version 2.4, 2.5 do not support a 'B' prefix">b</warning>""
a = <error descr="Python version 2.7 does not support a 'RB' prefix"><warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2 do not support a 'RB' prefix">rb</warning></error>""
a = <warning descr="Python version 2.4, 2.5, 3.0 do not support a 'BR' prefix">br</warning>""
# python 3.2, 3.1
a = r""
a = <warning descr="Python version 2.4, 2.5 do not support a 'B' prefix">b</warning>""
a = <warning descr="Python version 2.4, 2.5, 3.0 do not support a 'BR' prefix">br</warning>""
# python 3.0
a = r""
a = <warning descr="Python version 2.4, 2.5 do not support a 'B' prefix">b</warning>""
# python 2.7, 2.6
a = <warning descr="Python version 3.0, 3.1, 3.2 do not support a 'U' prefix">u</warning>""
a = r""
a = <warning descr="Python version 3.0, 3.1, 3.2, 3.3 do not support a 'UR' prefix">ur</warning>""
a = <warning descr="Python version 2.4, 2.5 do not support a 'B' prefix">b</warning>""
a = <warning descr="Python version 2.4, 2.5, 3.0 do not support a 'BR' prefix">br</warning>""
# python 2.5
a = <warning descr="Python version 3.0, 3.1, 3.2 do not support a 'U' prefix">u</warning>""
a = r""
a = <warning descr="Python version 3.0, 3.1, 3.2, 3.3 do not support a 'UR' prefix">ur</warning>""
# combined
b = <warning descr="Python version 3.0, 3.1, 3.2 do not support a 'U' prefix">u</warning>"" <warning descr="Python version 2.4, 2.5 do not support a 'B' prefix">b</warning>""
# never was available
a = <error descr="Python version 2.7 does not support a 'RR' prefix"><warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3 do not support a 'RR' prefix">rr</warning></error>""
a = <error descr="Python version 2.7 does not support a 'BB' prefix"><warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3 do not support a 'BB' prefix">bb</warning></error>""
a = <error descr="Python version 2.7 does not support a 'UU' prefix"><warning descr="Python version 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3 do not support a 'UU' prefix">uu</warning></error>""
@@ -69,7 +69,7 @@ public class PyIntentionTest extends PyTestCase {
}
public void testRemoveLeadingU() {
doTest(PyBundle.message("INTN.remove.leading.u"), LanguageLevel.PYTHON30);
doTest(PyBundle.message("INTN.remove.leading.$0", "U"), LanguageLevel.PYTHON30);
}
public void testRemoveTrailingL() {