From 0d330d552e1e57ca365108e071a9dec77878d118 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Mon, 14 Oct 2013 15:27:07 +0200 Subject: [PATCH 1/9] Simplified interface for PyStringFormatParser --- .../inspections/PyStringFormatInspection.java | 7 ++++-- .../inspections/PyStringFormatParser.java | 9 ++++++-- .../refactoring/PyReplaceExpressionUtil.java | 7 ++++-- .../introduce/IntroduceHandler.java | 3 +-- .../PythonSpellcheckerStrategy.java | 2 +- .../python/PyStringFormatParserTest.java | 23 +++++++++---------- 6 files changed, 30 insertions(+), 21 deletions(-) diff --git a/python/src/com/jetbrains/python/inspections/PyStringFormatInspection.java b/python/src/com/jetbrains/python/inspections/PyStringFormatInspection.java index 979a5f0d7fa9..b61d2a8ffb6d 100644 --- a/python/src/com/jetbrains/python/inspections/PyStringFormatInspection.java +++ b/python/src/com/jetbrains/python/inspections/PyStringFormatInspection.java @@ -23,6 +23,9 @@ import java.math.BigInteger; import java.util.List; import java.util.Map; +import static com.jetbrains.python.inspections.PyStringFormatParser.filterSubstitutions; +import static com.jetbrains.python.inspections.PyStringFormatParser.parsePercentFormat; + /** * @author Alexey.Ivanov */ @@ -332,8 +335,8 @@ public class PyStringFormatInspection extends PyInspection { } private void inspectFormat(@NotNull final PyStringLiteralExpression formatExpression) { - PyStringFormatParser parser = new PyStringFormatParser(formatExpression.getStringValue()); - final List chunks = parser.parseSubstitutions(); + final String value = formatExpression.getStringValue(); + final List chunks = filterSubstitutions(parsePercentFormat(value)); // 1. The '%' character // Skip the first item in the sections, it's always empty diff --git a/python/src/com/jetbrains/python/inspections/PyStringFormatParser.java b/python/src/com/jetbrains/python/inspections/PyStringFormatParser.java index 97a34fa72a38..650520650364 100644 --- a/python/src/com/jetbrains/python/inspections/PyStringFormatParser.java +++ b/python/src/com/jetbrains/python/inspections/PyStringFormatParser.java @@ -145,10 +145,15 @@ public class PyStringFormatParser { private static final String LENGTH_MODIFIERS = "hlL"; private static final String VALID_CONVERSION_TYPES = "diouxXeEfFgGcrs"; - public PyStringFormatParser(@NotNull String literal) { + private PyStringFormatParser(@NotNull String literal) { myLiteral = literal; } + @NotNull + public static List parsePercentFormat(@NotNull String s) { + return new PyStringFormatParser(s).parse(); + } + @NotNull public static List parseNewStyleFormat(@NotNull String s) { final List results = new ArrayList(); @@ -197,7 +202,7 @@ public class PyStringFormatParser { } @NotNull - public List parse() { + private List parse() { myPos = 0; while(myPos < myLiteral.length()) { int next = myLiteral.indexOf('%', myPos); diff --git a/python/src/com/jetbrains/python/refactoring/PyReplaceExpressionUtil.java b/python/src/com/jetbrains/python/refactoring/PyReplaceExpressionUtil.java index 166b18b82749..379835231e7c 100644 --- a/python/src/com/jetbrains/python/refactoring/PyReplaceExpressionUtil.java +++ b/python/src/com/jetbrains/python/refactoring/PyReplaceExpressionUtil.java @@ -24,6 +24,9 @@ import org.jetbrains.annotations.Nullable; import java.util.List; import static com.jetbrains.python.PyTokenTypes.*; +import static com.jetbrains.python.inspections.PyStringFormatParser.filterSubstitutions; +import static com.jetbrains.python.inspections.PyStringFormatParser.parseNewStyleFormat; +import static com.jetbrains.python.inspections.PyStringFormatParser.parsePercentFormat; /** * @author Dennis.Ushakov @@ -94,10 +97,10 @@ public class PyReplaceExpressionUtil implements PyElementTypes { final List substitutions; if (newStyleFormatValue != null) { - substitutions = PyStringFormatParser.filterSubstitutions(PyStringFormatParser.parseNewStyleFormat(fullText)); + substitutions = filterSubstitutions(parseNewStyleFormat(fullText)); } else { - substitutions = new PyStringFormatParser(fullText).parseSubstitutions(); + substitutions = filterSubstitutions(parsePercentFormat(fullText)); } final boolean hasSubstitutions = substitutions.size() > 0; diff --git a/python/src/com/jetbrains/python/refactoring/introduce/IntroduceHandler.java b/python/src/com/jetbrains/python/refactoring/introduce/IntroduceHandler.java index 411c34396743..0217a936b278 100644 --- a/python/src/com/jetbrains/python/refactoring/introduce/IntroduceHandler.java +++ b/python/src/com/jetbrains/python/refactoring/introduce/IntroduceHandler.java @@ -31,7 +31,6 @@ import com.jetbrains.python.PyBundle; import com.jetbrains.python.PyNames; import com.jetbrains.python.PythonStringUtil; import com.jetbrains.python.codeInsight.dataflow.scope.ScopeUtil; -import com.jetbrains.python.inspections.PyStringFormatParser; import com.jetbrains.python.psi.*; import com.jetbrains.python.psi.resolve.PyResolveContext; import com.jetbrains.python.psi.types.PyNoneType; @@ -306,7 +305,7 @@ abstract public class IntroduceHandler implements RefactoringActionHandler { } private boolean breaksStringFormatting(@NotNull String s, @NotNull TextRange range) { - return breaksRanges(substitutionsToRanges(new PyStringFormatParser(s).parseSubstitutions()), range); + return breaksRanges(substitutionsToRanges(filterSubstitutions(parsePercentFormat(s))), range); } private boolean breaksNewStyleStringFormatting(@NotNull String s, @NotNull TextRange range) { diff --git a/python/src/com/jetbrains/python/spellchecker/PythonSpellcheckerStrategy.java b/python/src/com/jetbrains/python/spellchecker/PythonSpellcheckerStrategy.java index 669ff4b001ba..645a6a26c4a5 100644 --- a/python/src/com/jetbrains/python/spellchecker/PythonSpellcheckerStrategy.java +++ b/python/src/com/jetbrains/python/spellchecker/PythonSpellcheckerStrategy.java @@ -82,7 +82,7 @@ public class PythonSpellcheckerStrategy extends SpellcheckingStrategy { @Override public void tokenize(@NotNull PyStringLiteralExpression element, TokenConsumer consumer) { String stringValue = element.getStringValue(); - List chunks = new PyStringFormatParser(stringValue).parse(); + List chunks = PyStringFormatParser.parsePercentFormat(stringValue); Splitter splitter = PlainTextSplitter.getInstance(); for (PyStringFormatParser.FormatStringChunk chunk : chunks) { if (chunk instanceof PyStringFormatParser.ConstantChunk) { diff --git a/python/testSrc/com/jetbrains/python/PyStringFormatParserTest.java b/python/testSrc/com/jetbrains/python/PyStringFormatParserTest.java index b64593674850..47847e4ed8da 100644 --- a/python/testSrc/com/jetbrains/python/PyStringFormatParserTest.java +++ b/python/testSrc/com/jetbrains/python/PyStringFormatParserTest.java @@ -1,7 +1,6 @@ package com.jetbrains.python; import com.intellij.openapi.util.TextRange; -import com.jetbrains.python.inspections.PyStringFormatParser; import junit.framework.TestCase; import java.util.List; @@ -13,7 +12,7 @@ import static com.jetbrains.python.inspections.PyStringFormatParser.*; */ public class PyStringFormatParserTest extends TestCase { public void testSimple() { - List chunks = new PyStringFormatParser("abc").parse(); + List chunks = parsePercentFormat("abc"); assertEquals(1, chunks.size()); assertConstant(chunks.get(0), 0, 3); } @@ -25,13 +24,13 @@ public class PyStringFormatParserTest extends TestCase { } public void testDoublePercent() { - List chunks = new PyStringFormatParser("abc%%def").parse(); + List chunks = parsePercentFormat("abc%%def"); assertEquals(1, chunks.size()); assertConstant(chunks.get(0), 0, 8); } public void testFormat() { - List chunks = new PyStringFormatParser("%s").parse(); + List chunks = parsePercentFormat("%s"); assertEquals(1, chunks.size()); SubstitutionChunk chunk = (SubstitutionChunk) chunks.get(0); assertEquals(0, chunk.getStartIndex()); @@ -40,13 +39,13 @@ public class PyStringFormatParserTest extends TestCase { } public void testSubstitutionAfterFormat() { - List chunks = new PyStringFormatParser("Hello, %s").parse(); + List chunks = parsePercentFormat("Hello, %s"); assertEquals(2, chunks.size()); assertConstant(chunks.get(0), 0, 7); } public void testMappingKey() { - List chunks = new PyStringFormatParser("%(language)s").parse(); + List chunks = parsePercentFormat("%(language)s"); assertEquals(1, chunks.size()); SubstitutionChunk chunk = (SubstitutionChunk) chunks.get(0); assertEquals("language", chunk.getMappingKey()); @@ -54,35 +53,35 @@ public class PyStringFormatParserTest extends TestCase { } public void testConversionFlags() { - List chunks = new PyStringFormatParser("%#0d").parse(); + List chunks = parsePercentFormat("%#0d"); assertEquals(1, chunks.size()); SubstitutionChunk chunk = (SubstitutionChunk) chunks.get(0); assertEquals("#0", chunk.getConversionFlags()); } public void testWidth() { - List chunks = new PyStringFormatParser("%345d").parse(); + List chunks = parsePercentFormat("%345d"); assertEquals(1, chunks.size()); SubstitutionChunk chunk = (SubstitutionChunk) chunks.get(0); assertEquals("345", chunk.getWidth()); } public void testPrecision() { - List chunks = new PyStringFormatParser("%.2d").parse(); + List chunks = parsePercentFormat("%.2d"); assertEquals(1, chunks.size()); SubstitutionChunk chunk = (SubstitutionChunk) chunks.get(0); assertEquals("2", chunk.getPrecision()); } public void testLengthModifier() { - List chunks = new PyStringFormatParser("%ld").parse(); + List chunks = parsePercentFormat("%ld"); assertEquals(1, chunks.size()); SubstitutionChunk chunk = (SubstitutionChunk) chunks.get(0); assertEquals('l', chunk.getLengthModifier()); } public void testDoubleAsterisk() { - List chunks = new PyStringFormatParser("%**d").parse(); + List chunks = parsePercentFormat("%**d"); assertEquals(2, chunks.size()); SubstitutionChunk chunk = (SubstitutionChunk) chunks.get(0); assertEquals(2, chunk.getEndIndex()); @@ -90,7 +89,7 @@ public class PyStringFormatParserTest extends TestCase { } public void testUnclosedMapping() { - List chunks = new PyStringFormatParser("%(name1s").parse(); + List chunks = parsePercentFormat("%(name1s"); SubstitutionChunk chunk = (SubstitutionChunk) chunks.get(0); assertEquals("name1s", chunk.getMappingKey()); assertTrue(chunk.isUnclosedMapping()); From 07aef2a0938ec8f85f28a5aea684162930832895 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Mon, 14 Oct 2013 15:32:17 +0200 Subject: [PATCH 2/9] Don't inject Regexp language into %-formatted fragments of string literals --- .../regexp/PythonRegexpInjector.java | 44 ++++++++++++++----- .../com/jetbrains/python/PyRegexpTest.java | 8 ++++ 2 files changed, 42 insertions(+), 10 deletions(-) diff --git a/python/src/com/jetbrains/python/codeInsight/regexp/PythonRegexpInjector.java b/python/src/com/jetbrains/python/codeInsight/regexp/PythonRegexpInjector.java index 123ac49c8b1c..e2c869b256da 100644 --- a/python/src/com/jetbrains/python/codeInsight/regexp/PythonRegexpInjector.java +++ b/python/src/com/jetbrains/python/codeInsight/regexp/PythonRegexpInjector.java @@ -8,6 +8,7 @@ import com.intellij.psi.PsiElement; import com.intellij.psi.PsiPolyVariantReference; import com.intellij.psi.util.PsiTreeUtil; import com.intellij.util.ArrayUtil; +import com.jetbrains.python.inspections.PyStringFormatParser; import com.jetbrains.python.psi.*; import com.jetbrains.python.psi.resolve.PyResolveContext; import org.jetbrains.annotations.NotNull; @@ -86,45 +87,68 @@ public class PythonRegexpInjector implements MultiHostInjector { final PyBinaryExpression expr = (PyBinaryExpression)element; final PyExpression left = expr.getLeftExpression(); final PyExpression right = expr.getRightExpression(); - return expr.isOperator("+") && (isStringLiteral(left) || right != null && isStringLiteral(right)); + return (expr.isOperator("+") && (isStringLiteral(left) || right != null && isStringLiteral(right))) || + expr.isOperator("%") && isStringLiteral(left); } return false; } private static void processStringLiteral(@NotNull PsiElement element, @NotNull MultiHostRegistrar registrar) { - processStringLiteral(element, registrar, "", ""); + processStringLiteral(element, registrar, "", "", false); } private static void processStringLiteral(@NotNull PsiElement element, @NotNull MultiHostRegistrar registrar, @NotNull String prefix, - @NotNull String suffix) { + @NotNull String suffix, boolean percentFormatting) { final String missingValue = "missing"; if (element instanceof PyStringLiteralExpression) { final PyStringLiteralExpression expr = (PyStringLiteralExpression)element; final List ranges = expr.getStringValueTextRanges(); + final String text = expr.getText(); for (TextRange range : ranges) { - registrar.addPlace(prefix, suffix, expr, range); + if (percentFormatting) { + final String part = range.substring(text); + final List chunks = PyStringFormatParser.parsePercentFormat(part); + for (int i = 0; i < chunks.size(); i++) { + final PyStringFormatParser.FormatStringChunk chunk = chunks.get(i); + if (chunk instanceof PyStringFormatParser.ConstantChunk) { + final int nextIndex = i + 1; + final String chunkPrefix = i == 1 && chunks.get(0) instanceof PyStringFormatParser.SubstitutionChunk ? missingValue : ""; + final String chunkSuffix = nextIndex < chunks.size() && + chunks.get(nextIndex) instanceof PyStringFormatParser.SubstitutionChunk ? missingValue : ""; + final TextRange chunkRange = chunk.getTextRange().shiftRight(range.getStartOffset()); + registrar.addPlace(chunkPrefix, chunkSuffix, expr, chunkRange); + } + + } + } + else { + registrar.addPlace(prefix, suffix, expr, range); + } } } else if (element instanceof PyParenthesizedExpression) { final PyExpression contained = ((PyParenthesizedExpression)element).getContainedExpression(); if (contained != null) { - processStringLiteral(contained, registrar, prefix, suffix); + processStringLiteral(contained, registrar, prefix, suffix, percentFormatting); } } else if (element instanceof PyBinaryExpression) { final PyBinaryExpression expr = (PyBinaryExpression)element; + final PyExpression left = expr.getLeftExpression(); + final PyExpression right = expr.getRightExpression(); + final boolean isLeftString = isStringLiteral(left); if (expr.isOperator("+")) { - final PyExpression left = expr.getLeftExpression(); - final PyExpression right = expr.getRightExpression(); - final boolean isLeftString = isStringLiteral(left); final boolean isRightString = right != null && isStringLiteral(right); if (isLeftString) { - processStringLiteral(left, registrar, prefix, isRightString ? "" : missingValue); + processStringLiteral(left, registrar, prefix, isRightString ? "" : missingValue, percentFormatting); } if (isRightString) { - processStringLiteral(right, registrar, isLeftString ? "" : missingValue, suffix); + processStringLiteral(right, registrar, isLeftString ? "" : missingValue, suffix, percentFormatting); } } + else if (expr.isOperator("%") && isLeftString) { + processStringLiteral(left, registrar, prefix, suffix, true); + } } } diff --git a/python/testSrc/com/jetbrains/python/PyRegexpTest.java b/python/testSrc/com/jetbrains/python/PyRegexpTest.java index 2b730f9c0ecc..1720800e6a8b 100644 --- a/python/testSrc/com/jetbrains/python/PyRegexpTest.java +++ b/python/testSrc/com/jetbrains/python/PyRegexpTest.java @@ -111,6 +111,14 @@ public class PyRegexpTest extends PyTestCase { ".*(missing)missing"); } + public void testPercentFormattingRegexpAutoInjection() { + doTestInjectedText("import re \n" + + "\n" + + "def f(x, y):\n" + + " re.search('.*%s-%d' % (x, y), 'foo')\n", + ".*missing-missing"); + } + private void doTestInjectedText(@NotNull String text, @NotNull String expected) { myFixture.configureByText(PythonFileType.INSTANCE, text); final InjectedLanguageManager languageManager = InjectedLanguageManager.getInstance(myFixture.getProject()); From 2efaf7375b82c4c561570ecd500a68d56f84e9ed Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Mon, 14 Oct 2013 15:46:38 +0200 Subject: [PATCH 3/9] Extracted name of str.format() function --- python/psi-api/src/com/jetbrains/python/PyNames.java | 2 ++ .../com/jetbrains/python/inspections/PyStringFormatParser.java | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/python/psi-api/src/com/jetbrains/python/PyNames.java b/python/psi-api/src/com/jetbrains/python/PyNames.java index ca7bdadd22ff..c87f04baf95b 100644 --- a/python/psi-api/src/com/jetbrains/python/PyNames.java +++ b/python/psi-api/src/com/jetbrains/python/PyNames.java @@ -119,6 +119,8 @@ public class PyNames { public static final String COLLECTIONS = "collections"; public static final String COLLECTIONS_NAMEDTUPLE = COLLECTIONS + "." + NAMEDTUPLE; + public static final String FORMAT = "format"; + public static final String ABSTRACTMETHOD = "abstractmethod"; public static final String ABSTRACTPROPERTY = "abstractproperty"; diff --git a/python/src/com/jetbrains/python/inspections/PyStringFormatParser.java b/python/src/com/jetbrains/python/inspections/PyStringFormatParser.java index 650520650364..19743668adf6 100644 --- a/python/src/com/jetbrains/python/inspections/PyStringFormatParser.java +++ b/python/src/com/jetbrains/python/inspections/PyStringFormatParser.java @@ -4,6 +4,7 @@ import com.intellij.openapi.util.TextRange; import com.intellij.openapi.util.text.StringUtil; import com.intellij.psi.PsiElement; import com.intellij.util.containers.HashMap; +import com.jetbrains.python.PyNames; import com.jetbrains.python.psi.*; import com.jetbrains.python.psi.impl.PyStringLiteralExpressionImpl; import org.jetbrains.annotations.NotNull; @@ -358,7 +359,7 @@ public class PyStringFormatParser { if (parent instanceof PyQualifiedExpression) { final PyQualifiedExpression qualifiedExpr = (PyQualifiedExpression)parent; final String name = qualifiedExpr.getReferencedName(); - if ("format".equals(name)) { + if (PyNames.FORMAT.equals(name)) { final PsiElement parent2 = qualifiedExpr.getParent(); if (parent2 instanceof PyCallExpression) { final PyCallExpression callExpr = (PyCallExpression)parent2; From 20fc43657884eb285a45b7cdc5647fb0542142c3 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Mon, 14 Oct 2013 15:47:50 +0200 Subject: [PATCH 4/9] Removed unused method --- .../jetbrains/python/inspections/PyStringFormatParser.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/python/src/com/jetbrains/python/inspections/PyStringFormatParser.java b/python/src/com/jetbrains/python/inspections/PyStringFormatParser.java index 19743668adf6..b029f0d9c30c 100644 --- a/python/src/com/jetbrains/python/inspections/PyStringFormatParser.java +++ b/python/src/com/jetbrains/python/inspections/PyStringFormatParser.java @@ -283,11 +283,6 @@ public class PyStringFormatParser { return myLiteral.substring(flagStart, myPos); } - @NotNull - public List parseSubstitutions() { - return filterSubstitutions(parse()); - } - @NotNull public static List filterSubstitutions(@NotNull List chunks) { final List results = new ArrayList(); From cfeb285cf545af30ffdfa45d5ca394bd5fd44b72 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Mon, 14 Oct 2013 18:12:04 +0200 Subject: [PATCH 5/9] OS-specific name for settings menu (IDEA-106855) --- .../src/com/jetbrains/python/sdk/InterpreterPathChooser.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/src/com/jetbrains/python/sdk/InterpreterPathChooser.java b/python/src/com/jetbrains/python/sdk/InterpreterPathChooser.java index 7be161979cd2..aa4e081bef8b 100644 --- a/python/src/com/jetbrains/python/sdk/InterpreterPathChooser.java +++ b/python/src/com/jetbrains/python/sdk/InterpreterPathChooser.java @@ -3,6 +3,7 @@ package com.jetbrains.python.sdk; import com.google.common.collect.Lists; import com.intellij.openapi.application.ApplicationManager; import com.intellij.openapi.application.ModalityState; +import com.intellij.openapi.options.ShowSettingsUtil; import com.intellij.openapi.project.Project; import com.intellij.openapi.projectRoots.Sdk; import com.intellij.openapi.projectRoots.impl.SdkConfigurationUtil; @@ -130,7 +131,9 @@ public class InterpreterPathChooser extends BaseListPopupStep { remoteInterpreterManager.addRemoteSdk(myProject, myOwnerComponent, Lists.newArrayList(myExistingSdks), myCallback); } else { - Messages.showErrorDialog("The Remote Hosts Access plugin is missing. Please enable the plugin in Settings | Plugins.", "Add Remote Interpreter"); + Messages.showErrorDialog("The Remote Hosts Access plugin is missing. Please enable the plugin in " + + ShowSettingsUtil.getSettingsMenuName() + + " | Plugins.", "Add Remote Interpreter"); } } From 1adfd0ff9481dac4ef0c66ad7c0123fd37edd6eb Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Mon, 14 Oct 2013 18:38:38 +0200 Subject: [PATCH 6/9] Fixed off-by-1 bug in parsing constant chunks of new style formatted strings --- .../jetbrains/python/inspections/PyStringFormatParser.java | 2 +- .../com/jetbrains/python/PyStringFormatParserTest.java | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/python/src/com/jetbrains/python/inspections/PyStringFormatParser.java b/python/src/com/jetbrains/python/inspections/PyStringFormatParser.java index b029f0d9c30c..8d4ee00a39ca 100644 --- a/python/src/com/jetbrains/python/inspections/PyStringFormatParser.java +++ b/python/src/com/jetbrains/python/inspections/PyStringFormatParser.java @@ -197,7 +197,7 @@ public class PyStringFormatParser { pos = next + 1; } if (pos < n) { - results.add(new ConstantChunk(pos, n + 1)); + results.add(new ConstantChunk(pos, n)); } return results; } diff --git a/python/testSrc/com/jetbrains/python/PyStringFormatParserTest.java b/python/testSrc/com/jetbrains/python/PyStringFormatParserTest.java index 47847e4ed8da..b2ed8f5f608d 100644 --- a/python/testSrc/com/jetbrains/python/PyStringFormatParserTest.java +++ b/python/testSrc/com/jetbrains/python/PyStringFormatParserTest.java @@ -125,4 +125,10 @@ public class PyStringFormatParserTest extends TestCase { assertEquals(1, chunks.size()); assertEquals(TextRange.create(1, 4), chunks.get(0).getTextRange()); } + + public void testNewStyleConstant() { + List chunks = parseNewStyleFormat("a"); + assertEquals(1, chunks.size()); + assertConstant(chunks.get(0), 0, 1); + } } From d74ab4210e99e261034da1fdfc7d36d0c9284308 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Mon, 14 Oct 2013 18:44:36 +0200 Subject: [PATCH 7/9] Don't inject Regexp language into {}-fragments of str.format() literals --- .../regexp/PythonRegexpInjector.java | 70 ++++++++++++++----- .../com/jetbrains/python/PyRegexpTest.java | 16 +++++ 2 files changed, 69 insertions(+), 17 deletions(-) diff --git a/python/src/com/jetbrains/python/codeInsight/regexp/PythonRegexpInjector.java b/python/src/com/jetbrains/python/codeInsight/regexp/PythonRegexpInjector.java index e2c869b256da..85fe2ad585e3 100644 --- a/python/src/com/jetbrains/python/codeInsight/regexp/PythonRegexpInjector.java +++ b/python/src/com/jetbrains/python/codeInsight/regexp/PythonRegexpInjector.java @@ -8,6 +8,7 @@ import com.intellij.psi.PsiElement; import com.intellij.psi.PsiPolyVariantReference; import com.intellij.psi.util.PsiTreeUtil; import com.intellij.util.ArrayUtil; +import com.jetbrains.python.PyNames; import com.jetbrains.python.inspections.PyStringFormatParser; import com.jetbrains.python.psi.*; import com.jetbrains.python.psi.resolve.PyResolveContext; @@ -72,42 +73,71 @@ public class PythonRegexpInjector implements MultiHostInjector { } } } - } private static boolean isStringLiteral(@NotNull PsiElement element) { + final PsiElement parent = element.getParent(); + return isStringLiteralPart(element) && (parent == null || !isStringLiteralPart(parent)); + } + + private static boolean isStringLiteralPart(@NotNull PsiElement element) { if (element instanceof PyStringLiteralExpression) { return true; } else if (element instanceof PyParenthesizedExpression) { final PyExpression contained = ((PyParenthesizedExpression)element).getContainedExpression(); - return contained != null && isStringLiteral(contained); + return contained != null && isStringLiteralPart(contained); } else if (element instanceof PyBinaryExpression) { final PyBinaryExpression expr = (PyBinaryExpression)element; final PyExpression left = expr.getLeftExpression(); final PyExpression right = expr.getRightExpression(); - return (expr.isOperator("+") && (isStringLiteral(left) || right != null && isStringLiteral(right))) || - expr.isOperator("%") && isStringLiteral(left); + return (expr.isOperator("+") && (isStringLiteralPart(left) || right != null && isStringLiteralPart(right))) || + expr.isOperator("%") && isStringLiteralPart(left); + } + else if (element instanceof PyCallExpression) { + final PyExpression qualifier = getFormatCallQualifier((PyCallExpression)element); + return qualifier != null && isStringLiteralPart(qualifier); } return false; } + @Nullable + private static PyExpression getFormatCallQualifier(@NotNull PyCallExpression element) { + final PyExpression callee = element.getCallee(); + if (callee instanceof PyQualifiedExpression) { + final PyQualifiedExpression qualifiedExpr = (PyQualifiedExpression)callee; + final PyExpression qualifier = qualifiedExpr.getQualifier(); + if (qualifier != null && PyNames.FORMAT.equals(qualifiedExpr.getReferencedName())) { + return qualifier; + } + } + return null; + } + + private enum Formatting { + NONE, + PERCENT, + NEW_STYLE + } + private static void processStringLiteral(@NotNull PsiElement element, @NotNull MultiHostRegistrar registrar) { - processStringLiteral(element, registrar, "", "", false); + processStringLiteral(element, registrar, "", "", Formatting.NONE); } private static void processStringLiteral(@NotNull PsiElement element, @NotNull MultiHostRegistrar registrar, @NotNull String prefix, - @NotNull String suffix, boolean percentFormatting) { + @NotNull String suffix, @NotNull Formatting formatting) { final String missingValue = "missing"; if (element instanceof PyStringLiteralExpression) { final PyStringLiteralExpression expr = (PyStringLiteralExpression)element; final List ranges = expr.getStringValueTextRanges(); final String text = expr.getText(); for (TextRange range : ranges) { - if (percentFormatting) { + if (formatting != Formatting.NONE) { final String part = range.substring(text); - final List chunks = PyStringFormatParser.parsePercentFormat(part); + final List chunks = formatting == Formatting.NEW_STYLE ? + PyStringFormatParser.parseNewStyleFormat(part) : + PyStringFormatParser.parsePercentFormat(part); for (int i = 0; i < chunks.size(); i++) { final PyStringFormatParser.FormatStringChunk chunk = chunks.get(i); if (chunk instanceof PyStringFormatParser.ConstantChunk) { @@ -118,7 +148,6 @@ public class PythonRegexpInjector implements MultiHostInjector { final TextRange chunkRange = chunk.getTextRange().shiftRight(range.getStartOffset()); registrar.addPlace(chunkPrefix, chunkSuffix, expr, chunkRange); } - } } else { @@ -129,25 +158,31 @@ public class PythonRegexpInjector implements MultiHostInjector { else if (element instanceof PyParenthesizedExpression) { final PyExpression contained = ((PyParenthesizedExpression)element).getContainedExpression(); if (contained != null) { - processStringLiteral(contained, registrar, prefix, suffix, percentFormatting); + processStringLiteral(contained, registrar, prefix, suffix, formatting); } } else if (element instanceof PyBinaryExpression) { final PyBinaryExpression expr = (PyBinaryExpression)element; final PyExpression left = expr.getLeftExpression(); final PyExpression right = expr.getRightExpression(); - final boolean isLeftString = isStringLiteral(left); + final boolean isLeftString = isStringLiteralPart(left); if (expr.isOperator("+")) { - final boolean isRightString = right != null && isStringLiteral(right); + final boolean isRightString = right != null && isStringLiteralPart(right); if (isLeftString) { - processStringLiteral(left, registrar, prefix, isRightString ? "" : missingValue, percentFormatting); + processStringLiteral(left, registrar, prefix, isRightString ? "" : missingValue, formatting); } if (isRightString) { - processStringLiteral(right, registrar, isLeftString ? "" : missingValue, suffix, percentFormatting); + processStringLiteral(right, registrar, isLeftString ? "" : missingValue, suffix, formatting); } } - else if (expr.isOperator("%") && isLeftString) { - processStringLiteral(left, registrar, prefix, suffix, true); + else if (expr.isOperator("%")) { + processStringLiteral(left, registrar, prefix, suffix, Formatting.PERCENT); + } + } + else if (element instanceof PyCallExpression) { + final PyExpression qualifier = getFormatCallQualifier((PyCallExpression)element); + if (qualifier != null) { + processStringLiteral(qualifier, registrar, prefix, suffix, Formatting.NEW_STYLE); } } } @@ -155,7 +190,8 @@ public class PythonRegexpInjector implements MultiHostInjector { @NotNull @Override public List> elementsToInjectIn() { - return Arrays.asList(PyStringLiteralExpression.class, PyParenthesizedExpression.class, PyBinaryExpression.class); + return Arrays.asList(PyStringLiteralExpression.class, PyParenthesizedExpression.class, PyBinaryExpression.class, + PyCallExpression.class); } private static boolean isVerbose(@NotNull PyCallExpression call) { diff --git a/python/testSrc/com/jetbrains/python/PyRegexpTest.java b/python/testSrc/com/jetbrains/python/PyRegexpTest.java index 1720800e6a8b..9dddbff96492 100644 --- a/python/testSrc/com/jetbrains/python/PyRegexpTest.java +++ b/python/testSrc/com/jetbrains/python/PyRegexpTest.java @@ -119,6 +119,22 @@ public class PyRegexpTest extends PyTestCase { ".*missing-missing"); } + public void testNewStyleFormattingRegexpAutoInjection() { + doTestInjectedText("import re\n" + + "\n" + + "def f(x, y):\n" + + " re.search('.*{foo}-{}'.format(x, foo=y), 'foo')\n", + ".*missing-missing"); + } + + public void testNewStyleFormattingEndsWithConstant() { + doTestInjectedText("import re\n" + + "\n" + + "def f(**kwargs):" + + " re.search('(foo{bar}baz$)'.format(**kwargs), 'foo')\n", + "(foomissingbaz$)"); + } + private void doTestInjectedText(@NotNull String text, @NotNull String expected) { myFixture.configureByText(PythonFileType.INSTANCE, text); final InjectedLanguageManager languageManager = InjectedLanguageManager.getInstance(myFixture.getProject()); From 56b1a9bf5f531881df828f2b142c0044dbfcc08b Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Mon, 14 Oct 2013 19:13:36 +0200 Subject: [PATCH 8/9] Moved Python string literal injection methods into PyInjectionUtil --- .../python/codeInsight/PyInjectionUtil.java | 138 ++++++++++++++++++ .../regexp/PythonRegexpInjector.java | 120 +-------------- 2 files changed, 141 insertions(+), 117 deletions(-) create mode 100644 python/src/com/jetbrains/python/codeInsight/PyInjectionUtil.java diff --git a/python/src/com/jetbrains/python/codeInsight/PyInjectionUtil.java b/python/src/com/jetbrains/python/codeInsight/PyInjectionUtil.java new file mode 100644 index 000000000000..e43be27b6fc2 --- /dev/null +++ b/python/src/com/jetbrains/python/codeInsight/PyInjectionUtil.java @@ -0,0 +1,138 @@ +package com.jetbrains.python.codeInsight; + +import com.intellij.lang.injection.MultiHostRegistrar; +import com.intellij.openapi.util.TextRange; +import com.intellij.psi.PsiElement; +import com.jetbrains.python.PyNames; +import com.jetbrains.python.psi.*; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.List; + +import static com.jetbrains.python.inspections.PyStringFormatParser.*; + +/** + * @author vlan + */ +public class PyInjectionUtil { + private PyInjectionUtil() {} + + /** + * Returns true if the element is the largest expression that represents a string literal, possibly with concatenation, parentheses, + * or formatting. + */ + public static boolean isLargestStringLiteral(@NotNull PsiElement element) { + final PsiElement parent = element.getParent(); + return isStringLiteralPart(element) && (parent == null || !isStringLiteralPart(parent)); + } + + /** + * Registers language injections in the given registrar for the specified string literal element or its ancestor that contains + * string concatenations or formatting. + */ + public static void registerStringLiteralInjection(@NotNull PsiElement element, @NotNull MultiHostRegistrar registrar) { + processStringLiteral(element, registrar, "", "", Formatting.NONE); + } + + private static boolean isStringLiteralPart(@NotNull PsiElement element) { + if (element instanceof PyStringLiteralExpression) { + return true; + } + else if (element instanceof PyParenthesizedExpression) { + final PyExpression contained = ((PyParenthesizedExpression)element).getContainedExpression(); + return contained != null && isStringLiteralPart(contained); + } + else if (element instanceof PyBinaryExpression) { + final PyBinaryExpression expr = (PyBinaryExpression)element; + final PyExpression left = expr.getLeftExpression(); + final PyExpression right = expr.getRightExpression(); + return (expr.isOperator("+") && (isStringLiteralPart(left) || right != null && isStringLiteralPart(right))) || + expr.isOperator("%") && isStringLiteralPart(left); + } + else if (element instanceof PyCallExpression) { + final PyExpression qualifier = getFormatCallQualifier((PyCallExpression)element); + return qualifier != null && isStringLiteralPart(qualifier); + } + return false; + } + + @Nullable + private static PyExpression getFormatCallQualifier(@NotNull PyCallExpression element) { + final PyExpression callee = element.getCallee(); + if (callee instanceof PyQualifiedExpression) { + final PyQualifiedExpression qualifiedExpr = (PyQualifiedExpression)callee; + final PyExpression qualifier = qualifiedExpr.getQualifier(); + if (qualifier != null && PyNames.FORMAT.equals(qualifiedExpr.getReferencedName())) { + return qualifier; + } + } + return null; + } + + private static void processStringLiteral(@NotNull PsiElement element, @NotNull MultiHostRegistrar registrar, @NotNull String prefix, + @NotNull String suffix, @NotNull Formatting formatting) { + final String missingValue = "missing"; + if (element instanceof PyStringLiteralExpression) { + final PyStringLiteralExpression expr = (PyStringLiteralExpression)element; + final List ranges = expr.getStringValueTextRanges(); + final String text = expr.getText(); + for (TextRange range : ranges) { + if (formatting != Formatting.NONE) { + final String part = range.substring(text); + final List chunks = formatting == Formatting.NEW_STYLE ? parseNewStyleFormat(part) : parsePercentFormat(part); + for (int i = 0; i < chunks.size(); i++) { + final FormatStringChunk chunk = chunks.get(i); + if (chunk instanceof ConstantChunk) { + final int nextIndex = i + 1; + final String chunkPrefix = i == 1 && chunks.get(0) instanceof SubstitutionChunk ? missingValue : ""; + final String chunkSuffix = nextIndex < chunks.size() && + chunks.get(nextIndex) instanceof SubstitutionChunk ? missingValue : ""; + final TextRange chunkRange = chunk.getTextRange().shiftRight(range.getStartOffset()); + registrar.addPlace(chunkPrefix, chunkSuffix, expr, chunkRange); + } + } + } + else { + registrar.addPlace(prefix, suffix, expr, range); + } + } + } + else if (element instanceof PyParenthesizedExpression) { + final PyExpression contained = ((PyParenthesizedExpression)element).getContainedExpression(); + if (contained != null) { + processStringLiteral(contained, registrar, prefix, suffix, formatting); + } + } + else if (element instanceof PyBinaryExpression) { + final PyBinaryExpression expr = (PyBinaryExpression)element; + final PyExpression left = expr.getLeftExpression(); + final PyExpression right = expr.getRightExpression(); + final boolean isLeftString = isStringLiteralPart(left); + if (expr.isOperator("+")) { + final boolean isRightString = right != null && isStringLiteralPart(right); + if (isLeftString) { + processStringLiteral(left, registrar, prefix, isRightString ? "" : missingValue, formatting); + } + if (isRightString) { + processStringLiteral(right, registrar, isLeftString ? "" : missingValue, suffix, formatting); + } + } + else if (expr.isOperator("%")) { + processStringLiteral(left, registrar, prefix, suffix, Formatting.PERCENT); + } + } + else if (element instanceof PyCallExpression) { + final PyExpression qualifier = getFormatCallQualifier((PyCallExpression)element); + if (qualifier != null) { + processStringLiteral(qualifier, registrar, prefix, suffix, Formatting.NEW_STYLE); + } + } + } + + private enum Formatting { + NONE, + PERCENT, + NEW_STYLE + } +} diff --git a/python/src/com/jetbrains/python/codeInsight/regexp/PythonRegexpInjector.java b/python/src/com/jetbrains/python/codeInsight/regexp/PythonRegexpInjector.java index 85fe2ad585e3..b6cdfd738b9a 100644 --- a/python/src/com/jetbrains/python/codeInsight/regexp/PythonRegexpInjector.java +++ b/python/src/com/jetbrains/python/codeInsight/regexp/PythonRegexpInjector.java @@ -3,13 +3,11 @@ package com.jetbrains.python.codeInsight.regexp; import com.intellij.lang.Language; import com.intellij.lang.injection.MultiHostInjector; import com.intellij.lang.injection.MultiHostRegistrar; -import com.intellij.openapi.util.TextRange; import com.intellij.psi.PsiElement; import com.intellij.psi.PsiPolyVariantReference; import com.intellij.psi.util.PsiTreeUtil; import com.intellij.util.ArrayUtil; -import com.jetbrains.python.PyNames; -import com.jetbrains.python.inspections.PyStringFormatParser; +import com.jetbrains.python.codeInsight.PyInjectionUtil; import com.jetbrains.python.psi.*; import com.jetbrains.python.psi.resolve.PyResolveContext; import org.jetbrains.annotations.NotNull; @@ -53,7 +51,7 @@ public class PythonRegexpInjector implements MultiHostInjector { @Override public void getLanguagesToInject(@NotNull MultiHostRegistrar registrar, @NotNull PsiElement context) { final PsiElement contextParent = context.getParent(); - if (isStringLiteral(context) && contextParent instanceof PyArgumentList) { + if (PyInjectionUtil.isLargestStringLiteral(context) && contextParent instanceof PyArgumentList) { final PyExpression[] args = ((PyArgumentList)contextParent).getArguments(); int index = ArrayUtil.indexOf(args, context); PyCallExpression call = PsiTreeUtil.getParentOfType(context, PyCallExpression.class); @@ -66,7 +64,7 @@ public class PythonRegexpInjector implements MultiHostInjector { if (element != null && element.getContainingFile().getName().equals("re.py") && isRegexpMethod(element, index)) { final Language language = isVerbose(call) ? PythonVerboseRegexpLanguage.INSTANCE : PythonRegexpLanguage.INSTANCE; registrar.startInjecting(language); - processStringLiteral(context, registrar); + PyInjectionUtil.registerStringLiteralInjection(context, registrar); registrar.doneInjecting(); } } @@ -75,118 +73,6 @@ public class PythonRegexpInjector implements MultiHostInjector { } } - private static boolean isStringLiteral(@NotNull PsiElement element) { - final PsiElement parent = element.getParent(); - return isStringLiteralPart(element) && (parent == null || !isStringLiteralPart(parent)); - } - - private static boolean isStringLiteralPart(@NotNull PsiElement element) { - if (element instanceof PyStringLiteralExpression) { - return true; - } - else if (element instanceof PyParenthesizedExpression) { - final PyExpression contained = ((PyParenthesizedExpression)element).getContainedExpression(); - return contained != null && isStringLiteralPart(contained); - } - else if (element instanceof PyBinaryExpression) { - final PyBinaryExpression expr = (PyBinaryExpression)element; - final PyExpression left = expr.getLeftExpression(); - final PyExpression right = expr.getRightExpression(); - return (expr.isOperator("+") && (isStringLiteralPart(left) || right != null && isStringLiteralPart(right))) || - expr.isOperator("%") && isStringLiteralPart(left); - } - else if (element instanceof PyCallExpression) { - final PyExpression qualifier = getFormatCallQualifier((PyCallExpression)element); - return qualifier != null && isStringLiteralPart(qualifier); - } - return false; - } - - @Nullable - private static PyExpression getFormatCallQualifier(@NotNull PyCallExpression element) { - final PyExpression callee = element.getCallee(); - if (callee instanceof PyQualifiedExpression) { - final PyQualifiedExpression qualifiedExpr = (PyQualifiedExpression)callee; - final PyExpression qualifier = qualifiedExpr.getQualifier(); - if (qualifier != null && PyNames.FORMAT.equals(qualifiedExpr.getReferencedName())) { - return qualifier; - } - } - return null; - } - - private enum Formatting { - NONE, - PERCENT, - NEW_STYLE - } - - private static void processStringLiteral(@NotNull PsiElement element, @NotNull MultiHostRegistrar registrar) { - processStringLiteral(element, registrar, "", "", Formatting.NONE); - } - - private static void processStringLiteral(@NotNull PsiElement element, @NotNull MultiHostRegistrar registrar, @NotNull String prefix, - @NotNull String suffix, @NotNull Formatting formatting) { - final String missingValue = "missing"; - if (element instanceof PyStringLiteralExpression) { - final PyStringLiteralExpression expr = (PyStringLiteralExpression)element; - final List ranges = expr.getStringValueTextRanges(); - final String text = expr.getText(); - for (TextRange range : ranges) { - if (formatting != Formatting.NONE) { - final String part = range.substring(text); - final List chunks = formatting == Formatting.NEW_STYLE ? - PyStringFormatParser.parseNewStyleFormat(part) : - PyStringFormatParser.parsePercentFormat(part); - for (int i = 0; i < chunks.size(); i++) { - final PyStringFormatParser.FormatStringChunk chunk = chunks.get(i); - if (chunk instanceof PyStringFormatParser.ConstantChunk) { - final int nextIndex = i + 1; - final String chunkPrefix = i == 1 && chunks.get(0) instanceof PyStringFormatParser.SubstitutionChunk ? missingValue : ""; - final String chunkSuffix = nextIndex < chunks.size() && - chunks.get(nextIndex) instanceof PyStringFormatParser.SubstitutionChunk ? missingValue : ""; - final TextRange chunkRange = chunk.getTextRange().shiftRight(range.getStartOffset()); - registrar.addPlace(chunkPrefix, chunkSuffix, expr, chunkRange); - } - } - } - else { - registrar.addPlace(prefix, suffix, expr, range); - } - } - } - else if (element instanceof PyParenthesizedExpression) { - final PyExpression contained = ((PyParenthesizedExpression)element).getContainedExpression(); - if (contained != null) { - processStringLiteral(contained, registrar, prefix, suffix, formatting); - } - } - else if (element instanceof PyBinaryExpression) { - final PyBinaryExpression expr = (PyBinaryExpression)element; - final PyExpression left = expr.getLeftExpression(); - final PyExpression right = expr.getRightExpression(); - final boolean isLeftString = isStringLiteralPart(left); - if (expr.isOperator("+")) { - final boolean isRightString = right != null && isStringLiteralPart(right); - if (isLeftString) { - processStringLiteral(left, registrar, prefix, isRightString ? "" : missingValue, formatting); - } - if (isRightString) { - processStringLiteral(right, registrar, isLeftString ? "" : missingValue, suffix, formatting); - } - } - else if (expr.isOperator("%")) { - processStringLiteral(left, registrar, prefix, suffix, Formatting.PERCENT); - } - } - else if (element instanceof PyCallExpression) { - final PyExpression qualifier = getFormatCallQualifier((PyCallExpression)element); - if (qualifier != null) { - processStringLiteral(qualifier, registrar, prefix, suffix, Formatting.NEW_STYLE); - } - } - } - @NotNull @Override public List> elementsToInjectIn() { From 6f0f73fe46d216a3ce760396951e815775efe446 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Mon, 14 Oct 2013 20:21:36 +0200 Subject: [PATCH 9/9] Updated versions of ReST and GetText files plug-ins --- python/rest/resources/META-INF/plugin.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/rest/resources/META-INF/plugin.xml b/python/rest/resources/META-INF/plugin.xml index 5a75d1bd0033..06cd1f494960 100644 --- a/python/rest/resources/META-INF/plugin.xml +++ b/python/rest/resources/META-INF/plugin.xml @@ -3,7 +3,7 @@ org.jetbrains.plugins.rest This plugin enables support for reStructuredText files (*.rst) JetBrains - 130.SNAPSHOT + 132.SNAPSHOT com.intellij.modules.lang