MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

Introduce substring for new-style formatted strings (PY-8372)

Browse Source
This commit is contained in:
Andrey Vlasovskikh
2013-01-10 21:20:51 +04:00
parent 3e0784b9a9
commit b5874c65f6
12 changed files with 307 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/src/com/jetbrains/python/inspections/PyStringFormatParser.java
+25 -2
View File
@@ -1,6 +1,7 @@
package com.jetbrains.python.inspections;
import com.intellij.openapi.util.TextRange;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.psi.PsiElement;
import com.intellij.util.containers.HashMap;
import com.jetbrains.python.psi.*;
@@ -52,6 +53,7 @@ public class PyStringFormatParser {
@Nullable private String myConversionFlags;
@Nullable private String myWidth;
@Nullable private String myPrecision;
@Nullable private Integer myPosition;
private char myLengthModifier;
private char myConversionType;
private boolean myUnclosedMapping;
@@ -123,6 +125,15 @@ public class PyStringFormatParser {
private void setUnclosedMapping(boolean unclosedMapping) {
myUnclosedMapping = unclosedMapping;
}
@Nullable
public Integer getPosition() {
return myPosition;
}
private void setPosition(@Nullable Integer position) {
myPosition = position;
}
}
@NotNull private final String myLiteral;
@@ -160,9 +171,21 @@ public class PyStringFormatParser {
next = s.indexOf('}', next + 2);
}
if (next > pos) {
// TODO: Parse substitution details
final SubstitutionChunk chunk = new SubstitutionChunk(pos);
chunk.setEndIndex(next + 1);
final int nameStart = pos + 1;
final int chunkEnd = next + 1;
chunk.setEndIndex(chunkEnd);
final int nameEnd = StringUtil.indexOfAny(s, "!:.[}", nameStart, chunkEnd);
if (nameEnd > 0 && nameStart < nameEnd) {
final String name = s.substring(nameStart, nameEnd);
try {
final int number = Integer.parseInt(name);
chunk.setPosition(number);
} catch (NumberFormatException e) {
chunk.setMappingKey(name);
}
}
// TODO: Parse substitution details
results.add(chunk);
}
pos = next + 1;
python/src/com/jetbrains/python/psi/impl/PyArgumentListImpl.java
+4
View File
@@ -1,5 +1,6 @@
package com.jetbrains.python.psi.impl;
import com.intellij.lang.ASTFactory;
import com.intellij.lang.ASTNode;
import com.intellij.psi.PsiElement;
import com.intellij.psi.tree.IElementType;
@@ -105,6 +106,7 @@ public class PyArgumentListImpl extends PyElementImpl implements PyArgumentList
if (before != null && elementPrecedesElementsOfType(before, PythonDialectsTokenSetProvider.INSTANCE.getExpressionTokens())) {
ASTNode comma = PyElementGenerator.getInstance(getProject()).createComma();
node.addChild(comma, before);
node.addChild(ASTFactory.whitespace(" "), before);
anchorBefore = comma;
}
else {
@@ -159,11 +161,13 @@ public class PyArgumentListImpl extends PyElementImpl implements PyArgumentList
ASTNode argNode = arg.getNode();
if (commaFirst) {
node.addChild(comma, beforeThis);
node.addChild(ASTFactory.whitespace(" "), beforeThis);
node.addChild(argNode, beforeThis);
}
else {
node.addChild(argNode, beforeThis);
node.addChild(comma, beforeThis);
node.addChild(ASTFactory.whitespace(" "), beforeThis);
}
}
python/src/com/jetbrains/python/refactoring/PyReplaceExpressionUtil.java
+252 -122
View File
@@ -88,149 +88,278 @@ public class PyReplaceExpressionUtil implements PyElementTypes {
final Pair<String, String> quotes = detectedQuotes != null ? detectedQuotes : Pair.create("'", "'");
final String prefix = fullText.substring(0, textRange.getStartOffset());
final String suffix = fullText.substring(textRange.getEndOffset(), oldExpression.getTextLength());
final PyExpression valueExpression = PyStringFormatParser.getFormatValueExpression(oldExpression);
final PyExpression formatValue = PyStringFormatParser.getFormatValueExpression(oldExpression);
final PyArgumentList newStyleFormatValue = PyStringFormatParser.getNewStyleFormatValueExpression(oldExpression);
final String newText = newExpression.getText();
final PyElementGenerator generator = PyElementGenerator.getInstance(oldExpression.getProject());
final LanguageLevel languageLevel = LanguageLevel.forElement(oldExpression);
final List<PyStringFormatParser.SubstitutionChunk> substitutions = new PyStringFormatParser(fullText).parseSubstitutions();
final List<PyStringFormatParser.SubstitutionChunk> substitutions;
if (newStyleFormatValue != null) {
substitutions = PyStringFormatParser.filterSubstitutions(PyStringFormatParser.parseNewStyleFormat(fullText));
}
else {
substitutions = new PyStringFormatParser(fullText).parseSubstitutions();
}
final boolean hasSubstitutions = substitutions.size() > 0;
if (valueExpression != null && !containsStringFormatting(fullText, textRange)) {
if (valueExpression instanceof PyTupleExpression) {
// 'foo%s' % (x,) -> '%s%s' % (s, x)
final String newLiteralText = prefix + "%s" + suffix;
final PyStringLiteralExpression newLiteralExpression = generator.createStringLiteralAlreadyEscaped(newLiteralText);
oldExpression.replace(newLiteralExpression);
final PyTupleExpression tuple = (PyTupleExpression)valueExpression;
final PyExpression[] members = tuple.getElements();
final List<PyStringFormatParser.SubstitutionChunk> positional = PyStringFormatParser.getPositionalSubstitutions(substitutions);
final int i = getPositionInRanges(PyStringFormatParser.substitutionsToRanges(positional), textRange);
final int n = members.length;
if (n > 0 && i <= n) {
final boolean last = i == n;
final ASTNode trailingComma = PyPsiUtils.getNextComma(members[n - 1].getNode());
if (trailingComma != null) {
tuple.getNode().removeChild(trailingComma);
}
final PyExpression before = last ? null : members[i];
PyUtil.addListNode(tuple, newExpression, before != null ? before.getNode() : null, i == 0 || !last, last, !last);
return newExpression;
}
if (formatValue != null && !containsStringFormatting(substitutions, textRange)) {
if (formatValue instanceof PyTupleExpression) {
return replaceSubstringWithTupleFormatting(oldExpression, newExpression, textRange, prefix, suffix,
(PyTupleExpression)formatValue, substitutions);
}
else if (valueExpression instanceof PyDictLiteralExpression) {
// 'foo%(x)s' % {'x': x} -> '%(s)s%(x)s' % {'x': x, 's': s}
// TODO: Support the dict() function
final String newLiteralText = prefix + "%(" + newText + ")s" + suffix;
final PyStringLiteralExpression newLiteralExpression = generator.createStringLiteralAlreadyEscaped(newLiteralText);
oldExpression.replace(newLiteralExpression);
final PyDictLiteralExpression dict = (PyDictLiteralExpression)valueExpression;
final StringBuilder builder = new StringBuilder();
builder.append("{");
final PyKeyValueExpression[] elements = dict.getElements();
builder.append(StringUtil.join(elements, new Function<PyKeyValueExpression, String>() {
@Override
public String fun(PyKeyValueExpression expression) {
return expression.getText();
}
}, ","));
if (elements.length > 0) {
builder.append(",");
}
builder.append(quotes.getSecond());
builder.append(newText);
builder.append(quotes.getSecond());
builder.append(":");
final int pos = builder.toString().length();
builder.append(newText);
builder.append("}");
final PyExpression newDictLiteral = generator.createExpressionFromText(languageLevel, builder.toString());
final PsiElement newElement = valueExpression.replace(newDictLiteral);
return newElement.findElementAt(pos);
else if (formatValue instanceof PyDictLiteralExpression) {
return replaceSubstringWithDictFormatting(oldExpression, quotes, prefix, suffix, formatValue, newText);
}
else {
final TypeEvalContext context = TypeEvalContext.slow();
final PyType valueType = valueExpression.getType(context);
final PyType valueType = formatValue.getType(context);
final PyBuiltinCache builtinCache = PyBuiltinCache.getInstance(oldExpression);
final PyType tupleType = builtinCache.getTupleType();
final PyType mappingType = PyTypeParser.getTypeByName(null, "collections.Mapping");
if (!PyTypeChecker.match(tupleType, valueType, context) ||
(mappingType != null && !PyTypeChecker.match(mappingType, valueType, context))) {
// 'foo%s' % value if value is not tuple or mapping -> '%s%s' % (s, value)
final String newLiteralText = prefix + "%s" + suffix;
final PyStringLiteralExpression newLiteralExpression = generator.createStringLiteralAlreadyEscaped(newLiteralText);
oldExpression.replace(newLiteralExpression);
final StringBuilder builder = new StringBuilder();
builder.append("(");
final List<PyStringFormatParser.SubstitutionChunk> positional = PyStringFormatParser.getPositionalSubstitutions(substitutions);
final int i = getPositionInRanges(PyStringFormatParser.substitutionsToRanges(positional), textRange);
final int pos;
if (i == 0) {
pos = builder.toString().length();
builder.append(newText);
builder.append(",");
builder.append(valueExpression.getText());
}
else {
builder.append(valueExpression.getText());
builder.append(",");
pos = builder.toString().length();
builder.append(newText);
}
builder.append(")");
final PsiElement newElement = valueExpression.replace(generator.createExpressionFromText(languageLevel, builder.toString()));
return newElement.findElementAt(pos);
return replaceSubstringWithSingleValueFormatting(oldExpression, textRange, prefix, suffix, formatValue, newText, substitutions);
}
}
}
if (isConcatFormatting(oldExpression) || substitutions.size() > 0) {
// 'foobar' + 'baz' -> s + 'bar' + 'baz'
// 'foobar%s' -> s + 'bar%s'
// 'f%soobar' % x -> (s + 'bar') % x
final String leftQuote = quotes.getFirst();
final String rightQuote = quotes.getSecond();
final StringBuilder builder = new StringBuilder();
if (valueExpression != null) {
builder.append("(");
if (newStyleFormatValue != null && hasSubstitutions && !containsStringFormatting(substitutions, textRange)) {
final PyExpression[] arguments = newStyleFormatValue.getArguments();
boolean hasStarArguments = false;
for (PyExpression argument : arguments) {
if (argument instanceof PyStarArgument) {
hasStarArguments = true;
}
}
if (!leftQuote.endsWith(prefix)) {
builder.append(prefix + rightQuote + " + ");
if (!hasStarArguments) {
return replaceSubstringWithNewStyleFormatting(oldExpression, textRange, prefix, suffix, newStyleFormatValue, newText,
substitutions);
}
final int pos = builder.toString().length();
}
if (isConcatFormatting(oldExpression) || hasSubstitutions) {
return replaceSubstringWithConcatFormatting(oldExpression, quotes, prefix, suffix, newText, hasSubstitutions);
}
return replaceSubstringWithoutFormatting(oldExpression, prefix, suffix, newText);
}
private static PsiElement replaceSubstringWithSingleValueFormatting(PyStringLiteralExpression oldExpression,
TextRange textRange,
String prefix,
String suffix,
PyExpression formatValue,
String newText,
List<PyStringFormatParser.SubstitutionChunk> substitutions) {
// 'foo%s' % value if value is not tuple or mapping -> '%s%s' % (s, value)
final PyElementGenerator generator = PyElementGenerator.getInstance(oldExpression.getProject());
final LanguageLevel languageLevel = LanguageLevel.forElement(oldExpression);
final String newLiteralText = prefix + "%s" + suffix;
final PyStringLiteralExpression newLiteralExpression = generator.createStringLiteralAlreadyEscaped(newLiteralText);
oldExpression.replace(newLiteralExpression);
final StringBuilder builder = new StringBuilder();
builder.append("(");
final int i = getPositionInRanges(PyStringFormatParser.substitutionsToRanges(substitutions), textRange);
final int pos;
if (i == 0) {
pos = builder.toString().length();
builder.append(newText);
if (!rightQuote.startsWith(suffix)) {
builder.append(" + " + leftQuote + suffix);
}
if (valueExpression != null) {
builder.append(")");
}
final PsiElement expression = generator.createExpressionFromText(languageLevel, builder.toString());
final PsiElement newElement = oldExpression.replace(expression);
return newElement.findElementAt(pos);
builder.append(",");
builder.append(formatValue.getText());
}
else {
// 'foobar' -> '%sbar' % s
// TODO: Handle extracting substring from a string with new-style formatting
final PsiElement parent = oldExpression.getParent();
final boolean parensNeeded = parent instanceof PyExpression && !(parent instanceof PyParenthesizedExpression);
final StringBuilder builder = new StringBuilder();
if (parensNeeded) {
builder.append("(");
}
builder.append(prefix);
builder.append("%s");
builder.append(suffix);
builder.append(" % ");
final int pos = builder.toString().length();
builder.append(formatValue.getText());
builder.append(",");
pos = builder.toString().length();
builder.append(newText);
if (parensNeeded) {
builder.append(")");
}
builder.append(")");
final PsiElement newElement = formatValue.replace(generator.createExpressionFromText(languageLevel, builder.toString()));
return newElement.findElementAt(pos);
}
private static PsiElement replaceSubstringWithDictFormatting(PyStringLiteralExpression oldExpression,
Pair<String, String> quotes,
String prefix,
String suffix,
PyExpression formatValue,
String newText) {
// 'foo%(x)s' % {'x': x} -> '%(s)s%(x)s' % {'x': x, 's': s}
// TODO: Support the dict() function
final PyElementGenerator generator = PyElementGenerator.getInstance(oldExpression.getProject());
final LanguageLevel languageLevel = LanguageLevel.forElement(oldExpression);
final String newLiteralText = prefix + "%(" + newText + ")s" + suffix;
final PyStringLiteralExpression newLiteralExpression = generator.createStringLiteralAlreadyEscaped(newLiteralText);
oldExpression.replace(newLiteralExpression);
final PyDictLiteralExpression dict = (PyDictLiteralExpression)formatValue;
final StringBuilder builder = new StringBuilder();
builder.append("{");
final PyKeyValueExpression[] elements = dict.getElements();
builder.append(StringUtil.join(elements, new Function<PyKeyValueExpression, String>() {
@Override
public String fun(PyKeyValueExpression expression) {
return expression.getText();
}
final PyExpression expression = generator.createExpressionFromText(languageLevel, builder.toString());
final PsiElement newElement = oldExpression.replace(expression);
return newElement.findElementAt(pos);
}, ","));
if (elements.length > 0) {
builder.append(",");
}
builder.append(quotes.getSecond());
builder.append(newText);
builder.append(quotes.getSecond());
builder.append(":");
final int pos = builder.toString().length();
builder.append(newText);
builder.append("}");
final PyExpression newDictLiteral = generator.createExpressionFromText(languageLevel, builder.toString());
final PsiElement newElement = formatValue.replace(newDictLiteral);
return newElement.findElementAt(pos);
}
private static PsiElement replaceSubstringWithTupleFormatting(PyStringLiteralExpression oldExpression,
PsiElement newExpression,
TextRange textRange,
String prefix,
String suffix,
PyTupleExpression tupleFormatValue,
List<PyStringFormatParser.SubstitutionChunk> substitutions) {
// 'foo%s' % (x,) -> '%s%s' % (s, x)
final String newLiteralText = prefix + "%s" + suffix;
final PyElementGenerator generator = PyElementGenerator.getInstance(oldExpression.getProject());
final PyStringLiteralExpression newLiteralExpression = generator.createStringLiteralAlreadyEscaped(newLiteralText);
oldExpression.replace(newLiteralExpression);
final PyExpression[] members = tupleFormatValue.getElements();
final int n = members.length;
final int i = Math.min(n, Math.max(0, getPositionInRanges(PyStringFormatParser.substitutionsToRanges(substitutions), textRange)));
final boolean last = i == n;
final ASTNode trailingComma = PyPsiUtils.getNextComma(members[n - 1].getNode());
if (trailingComma != null) {
tupleFormatValue.getNode().removeChild(trailingComma);
}
final PyExpression before = last ? null : members[i];
PyUtil.addListNode(tupleFormatValue, newExpression, before != null ? before.getNode() : null, i == 0 || !last, last, !last);
return newExpression;
}
private static PsiElement replaceSubstringWithoutFormatting(@NotNull PyStringLiteralExpression oldExpression,
@NotNull String prefix,
@NotNull String suffix,
@NotNull String newText) {
// 'foobar' -> '%sbar' % s
final PyElementGenerator generator = PyElementGenerator.getInstance(oldExpression.getProject());
final LanguageLevel languageLevel = LanguageLevel.forElement(oldExpression);
final PsiElement parent = oldExpression.getParent();
final boolean parensNeeded = parent instanceof PyExpression && !(parent instanceof PyParenthesizedExpression);
final StringBuilder builder = new StringBuilder();
if (parensNeeded) {
builder.append("(");
}
builder.append(prefix);
builder.append("%s");
builder.append(suffix);
builder.append(" % ");
final int pos = builder.toString().length();
builder.append(newText);
if (parensNeeded) {
builder.append(")");
}
final PyExpression expression = generator.createExpressionFromText(languageLevel, builder.toString());
final PsiElement newElement = oldExpression.replace(expression);
return newElement.findElementAt(pos);
}
private static PsiElement replaceSubstringWithConcatFormatting(@NotNull PyStringLiteralExpression oldExpression,
@NotNull Pair<String, String> quotes,
@NotNull String prefix,
@NotNull String suffix,
@NotNull String newText,
boolean hasSubstitutions) {
// 'foobar' + 'baz' -> s + 'bar' + 'baz'
// 'foobar%s' -> s + 'bar%s'
// 'f%soobar' % x -> (s + 'bar') % x
final PyElementGenerator generator = PyElementGenerator.getInstance(oldExpression.getProject());
final LanguageLevel languageLevel = LanguageLevel.forElement(oldExpression);
final String leftQuote = quotes.getFirst();
final String rightQuote = quotes.getSecond();
final StringBuilder builder = new StringBuilder();
if (hasSubstitutions) {
builder.append("(");
}
if (!leftQuote.endsWith(prefix)) {
builder.append(prefix + rightQuote + " + ");
}
final int pos = builder.toString().length();
builder.append(newText);
if (!rightQuote.startsWith(suffix)) {
builder.append(" + " + leftQuote + suffix);
}
if (hasSubstitutions) {
builder.append(")");
}
final PsiElement expression = generator.createExpressionFromText(languageLevel, builder.toString());
final PsiElement newElement = oldExpression.replace(expression);
return newElement.findElementAt(pos);
}
private static PsiElement replaceSubstringWithNewStyleFormatting(@NotNull PyStringLiteralExpression oldExpression,
@NotNull TextRange textRange,
@NotNull String prefix,
@NotNull String suffix,
@NotNull PyArgumentList newStyleFormatValue,
@NotNull String newText,
@NotNull List<PyStringFormatParser.SubstitutionChunk> substitutions) {
final PyElementGenerator generator = PyElementGenerator.getInstance(oldExpression.getProject());
final LanguageLevel languageLevel = LanguageLevel.forElement(oldExpression);
final PyExpression[] arguments = newStyleFormatValue.getArguments();
boolean hasKeywords = false;
int maxPosition = -1;
for (PyStringFormatParser.SubstitutionChunk substitution : substitutions) {
if (substitution.getMappingKey() != null) {
hasKeywords = true;
}
final Integer position = substitution.getPosition();
if (position != null && position > maxPosition) {
maxPosition = position;
}
}
if (hasKeywords) {
// 'foo{x}'.format(x='bar') -> '{s}oo{x}'.format(x='bar', s=s)
final String newLiteralText = prefix + "{" + newText + "}" + suffix;
final PyStringLiteralExpression newLiteralExpression = generator.createStringLiteralAlreadyEscaped(newLiteralText);
oldExpression.replace(newLiteralExpression);
final PyKeywordArgument kwarg = generator.createKeywordArgument(languageLevel, newText, newText);
newStyleFormatValue.addArgument(kwarg);
return kwarg.getValueExpression();
}
else if (maxPosition >= 0) {
// 'foo{0}'.format('bar') -> '{1}oo{0}'.format('bar', s)
final String newLiteralText = prefix + "{" + (maxPosition + 1) + "}" + suffix;
final PyStringLiteralExpression newLiteralExpression = generator.createStringLiteralAlreadyEscaped(newLiteralText);
oldExpression.replace(newLiteralExpression);
final PyExpression arg = generator.createExpressionFromText(languageLevel, newText);
newStyleFormatValue.addArgument(arg);
return arg;
}
else {
// 'foo{}'.format('bar') -> '{}oo{}'.format(s, 'bar')
final String newLiteralText = prefix + "{}" + suffix;
final PyStringLiteralExpression newLiteralExpression = generator.createStringLiteralAlreadyEscaped(newLiteralText);
oldExpression.replace(newLiteralExpression);
final int i = getPositionInRanges(PyStringFormatParser.substitutionsToRanges(substitutions), textRange);
final PyExpression arg = generator.createExpressionFromText(languageLevel, newText);
if (i == 0) {
newStyleFormatValue.addArgumentFirst(arg);
}
else if (i < arguments.length) {
newStyleFormatValue.addArgumentAfter(arg, arguments[i - 1]);
}
else {
newStyleFormatValue.addArgument(arg);
}
return arg;
}
}
@@ -246,8 +375,9 @@ public class PyReplaceExpressionUtil implements PyElementTypes {
return size;
}
private static boolean containsStringFormatting(@NotNull String s, @NotNull TextRange range) {
final List<TextRange> ranges = PyStringFormatParser.substitutionsToRanges(new PyStringFormatParser(s).parseSubstitutions());
private static boolean containsStringFormatting(@NotNull List<PyStringFormatParser.SubstitutionChunk> substitutions,
@NotNull TextRange range) {
final List<TextRange> ranges = PyStringFormatParser.substitutionsToRanges(substitutions);
for (TextRange r : ranges) {
if (range.contains(r)) {
return true;
python/testData/refactoring/introduceVariable/substringNewStyleAutomaticNumbering.after.py
+2
View File
@@ -0,0 +1,2 @@
a = "Hello"
print("{}, {}!".format(a, "World"))
python/testData/refactoring/introduceVariable/substringNewStyleAutomaticNumbering.py
+1
View File
@@ -0,0 +1 @@
print("<selection>Hello</selection>, {}!".format("World"))
python/testData/refactoring/introduceVariable/substringNewStyleKeywords.after.py
+2
View File
@@ -0,0 +1,2 @@
a = "Hello"
print("{a}, {name}!".format(name="World", a=a))
python/testData/refactoring/introduceVariable/substringNewStyleKeywords.py
+1
View File
@@ -0,0 +1 @@
print("<selection>Hello</selection>, {name}!".format(name="World"))
python/testData/refactoring/introduceVariable/substringNewStylePositional.after.py
+2
View File
@@ -0,0 +1,2 @@
a = "Hello"
print("{1}, {0}!".format("World", a))
python/testData/refactoring/introduceVariable/substringNewStylePositional.py
+1
View File
@@ -0,0 +1 @@
print("<selection>Hello</selection>, {0}!".format("World"))
python/testData/refactoring/pullup/existingsuperclass.after.py
+1 -1
View File
@@ -2,7 +2,7 @@ class Foo:
def foo(self):
print("a")
class Zope(Hand,Foo):
class Zope(Hand, Foo):
def _mine(self):
print "zope"
python/testData/refactoring/pushdown/superclass.after.py
+1 -1
View File
@@ -6,6 +6,6 @@ class Zope():
def _mine(self):
print "zope"
class Boo(Zope,Foo):
class Boo(Zope, Foo):
def boo(self):
print "rrrrr"
python/testSrc/com/jetbrains/python/refactoring/PyIntroduceVariableTest.java
+15
View File
@@ -193,6 +193,21 @@ public class PyIntroduceVariableTest extends PyIntroduceTestCase {
doTestCannotPerform();
}
// PY-8372
public void testSubstringNewStylePositional() {
doTest();
}
// PY-8372
public void testSubstringNewStyleAutomaticNumbering() {
doTest();
}
// PY-8372
public void testSubstringNewStyleKeywords() {
doTest();
}
private void doTestCannotPerform() {
boolean thrownExpectedException = false;
try {