From 6caccc8e4d9ef0fe9cb1d1ec7ca7e781bd025ca6 Mon Sep 17 00:00:00 2001 From: Mikhail Golubev Date: Tue, 3 Jul 2018 18:21:09 +0300 Subject: [PATCH] PY-21493 Python regexp injector is aware of f-strings --- .../python/codeInsight/PyInjectionUtil.java | 72 +++++++++++++------ .../codeInsight/fstrings/FStringParser.java | 3 +- .../fstrings/PyFStringsInjector.java | 6 +- .../regexp/PythonRegexpInjector.java | 4 ++ .../documentation/doctest/PyDocReference.java | 16 +++-- .../resolve/RegexpAndFStringCombined.py | 5 ++ .../python/PyInjectionResolveTest.java | 5 ++ .../com/jetbrains/python/PyRegexpTest.java | 37 ++++++++++ 8 files changed, 119 insertions(+), 29 deletions(-) create mode 100644 python/testData/resolve/RegexpAndFStringCombined.py diff --git a/python/src/com/jetbrains/python/codeInsight/PyInjectionUtil.java b/python/src/com/jetbrains/python/codeInsight/PyInjectionUtil.java index 97578c6e754b..2456e147a9cb 100644 --- a/python/src/com/jetbrains/python/codeInsight/PyInjectionUtil.java +++ b/python/src/com/jetbrains/python/codeInsight/PyInjectionUtil.java @@ -15,13 +15,17 @@ */ package com.jetbrains.python.codeInsight; +import com.intellij.lang.ASTNode; import com.intellij.lang.injection.MultiHostRegistrar; import com.intellij.openapi.util.TextRange; import com.intellij.psi.PsiComment; import com.intellij.psi.PsiElement; +import com.intellij.util.containers.ContainerUtil; import com.jetbrains.python.PyNames; +import com.jetbrains.python.codeInsight.fstrings.FStringParser; import com.jetbrains.python.psi.*; import com.jetbrains.python.psi.impl.PyCallExpressionNavigator; +import one.util.streamex.StreamEx; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -139,46 +143,74 @@ public class PyInjectionUtil { boolean injected = false; boolean strict = true; final PyStringLiteralExpression expr = (PyStringLiteralExpression)element; - final List ranges = expr.getStringValueTextRanges(); - final String text = expr.getText(); - for (TextRange range : ranges) { - if (formatting != Formatting.NONE) { - final String part = range.substring(text); - final List chunks = formatting == Formatting.NEW_STYLE ? parseNewStyleFormat(part) : parsePercentFormat(part); - if (!filterSubstitutions(chunks).isEmpty()) { + for (ASTNode node : expr.getStringNodes()) { + final int nodeOffsetInParent = node.getStartOffset() - expr.getTextRange().getStartOffset(); + final PyUtil.StringNodeInfo nodeInfo = new PyUtil.StringNodeInfo(node); + final TextRange contentRange = nodeInfo.getContentRange(); + final int contentStartOffset = contentRange.getStartOffset(); + if (formatting != Formatting.NONE || nodeInfo.isFormatted()) { + // Each range is relative to the start of the string node + final List subsRanges; + if (formatting != Formatting.NONE) { + final String content = nodeInfo.getContent(); + subsRanges = StreamEx.of(formatting == Formatting.NEW_STYLE ? parseNewStyleFormat(content) : parsePercentFormat(content)) + .select(SubstitutionChunk.class) + .map(chunk -> chunk.getTextRange().shiftRight(contentStartOffset)) + .toList(); + } + else { + // f-string fragment parser handles string literal prefix and opening quotes itself + subsRanges = StreamEx.of(FStringParser.parse(node.getText()).getFragments()) + .filter(f -> f.getDepth() == 1) // don't consider nested fragments like {foo:{bar}} + .map(f -> TextRange.create(f.getLeftBraceOffset(), + Math.max(f.getRightBraceOffset() + 1, f.getContentEndOffset()))) + .toList(); + } + if (!subsRanges.isEmpty()) { strict = false; } - for (int i = 0; i < chunks.size(); i++) { - final FormatStringChunk chunk = chunks.get(i); - if (chunk instanceof ConstantChunk) { - final int nextIndex = i + 1; + + + final TextRange sentinel = TextRange.from(contentRange.getEndOffset(), 0); + final List withSentinel = ContainerUtil.append(subsRanges, sentinel); + + int literalChunkStart = contentStartOffset; + int literalChunkEnd; + for (int i = 0; i < withSentinel.size(); i++) { + final TextRange subRange = withSentinel.get(i); + literalChunkEnd = subRange.getStartOffset(); + if (literalChunkEnd > literalChunkStart) { final String chunkPrefix; - if (i == 1 && chunks.get(0) instanceof SubstitutionChunk) { + if (i == 0) { + chunkPrefix = prefix; + } + else if (i == 1 && withSentinel.get(0).getStartOffset() == contentStartOffset) { chunkPrefix = missingValue; } - else if (i == 0) { - chunkPrefix = prefix; - } else { + else { chunkPrefix = ""; } + final String chunkSuffix; - if (nextIndex < chunks.size() && chunks.get(nextIndex) instanceof SubstitutionChunk) { + if (i < withSentinel.size() - 1) { chunkSuffix = missingValue; } - else if (nextIndex == chunks.size()) { + else if (i == withSentinel.size() - 1) { chunkSuffix = suffix; } else { chunkSuffix = ""; } - final TextRange chunkRange = chunk.getTextRange().shiftRight(range.getStartOffset()); - registrar.addPlace(chunkPrefix, chunkSuffix, expr, chunkRange); + + final TextRange chunkRange = TextRange.create(literalChunkStart, literalChunkEnd); + registrar.addPlace(chunkPrefix, chunkSuffix, expr, chunkRange.shiftRight(nodeOffsetInParent)); injected = true; } + literalChunkStart = subRange.getEndOffset(); } } else { - registrar.addPlace(prefix, suffix, expr, range); + registrar.addPlace(prefix, suffix, expr, contentRange.shiftRight(nodeOffsetInParent)); injected = true; } } diff --git a/python/src/com/jetbrains/python/codeInsight/fstrings/FStringParser.java b/python/src/com/jetbrains/python/codeInsight/fstrings/FStringParser.java index 95107e12dc37..eb093aeb7d27 100644 --- a/python/src/com/jetbrains/python/codeInsight/fstrings/FStringParser.java +++ b/python/src/com/jetbrains/python/codeInsight/fstrings/FStringParser.java @@ -23,6 +23,7 @@ import org.jetbrains.annotations.NotNull; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.List; /** @@ -234,7 +235,7 @@ public class FStringParser { private ParseResult(@NotNull List singleRightBraces, @NotNull List fragments) { mySingleRightBraces = singleRightBraces; - myFragments = ContainerUtil.sorted(fragments, (f1, f2) -> f1.getLeftBraceOffset() - f2.getLeftBraceOffset()); + myFragments = ContainerUtil.sorted(fragments, Comparator.comparingInt(Fragment::getLeftBraceOffset)); } @NotNull diff --git a/python/src/com/jetbrains/python/codeInsight/fstrings/PyFStringsInjector.java b/python/src/com/jetbrains/python/codeInsight/fstrings/PyFStringsInjector.java index 365d40cd20c5..13ad270b3517 100644 --- a/python/src/com/jetbrains/python/codeInsight/fstrings/PyFStringsInjector.java +++ b/python/src/com/jetbrains/python/codeInsight/fstrings/PyFStringsInjector.java @@ -42,7 +42,11 @@ public class PyFStringsInjector extends PyInjectorBase { if (pyString == null) { return; } - + + injectFStringFragments(registrar, pyString); + } + + public static void injectFStringFragments(@NotNull MultiHostRegistrar registrar, PyStringLiteralExpression pyString) { for (ASTNode node : pyString.getStringNodes()) { final int relNodeOffset = node.getTextRange().getStartOffset() - pyString.getTextRange().getStartOffset(); for (Fragment offsets : getInjectionRanges(node)) { diff --git a/python/src/com/jetbrains/python/codeInsight/regexp/PythonRegexpInjector.java b/python/src/com/jetbrains/python/codeInsight/regexp/PythonRegexpInjector.java index 988f9ae72f1b..dd3ecf996aff 100644 --- a/python/src/com/jetbrains/python/codeInsight/regexp/PythonRegexpInjector.java +++ b/python/src/com/jetbrains/python/codeInsight/regexp/PythonRegexpInjector.java @@ -25,6 +25,7 @@ import com.intellij.psi.util.PsiTreeUtil; import com.intellij.util.ArrayUtil; import com.jetbrains.python.codeInsight.PyInjectionUtil; import com.jetbrains.python.codeInsight.dataflow.scope.ScopeUtil; +import com.jetbrains.python.codeInsight.fstrings.PyFStringsInjector; import com.jetbrains.python.psi.*; import com.jetbrains.python.psi.resolve.PyResolveContext; import com.jetbrains.python.psi.types.TypeEvalContext; @@ -107,6 +108,9 @@ public class PythonRegexpInjector implements MultiHostInjector { } } } + if (context instanceof PyStringLiteralExpression) { + PyFStringsInjector.injectFStringFragments(registrar, (PyStringLiteralExpression)context); + } } @NotNull diff --git a/python/src/com/jetbrains/python/documentation/doctest/PyDocReference.java b/python/src/com/jetbrains/python/documentation/doctest/PyDocReference.java index 6190edbd868e..f8b2ea389465 100644 --- a/python/src/com/jetbrains/python/documentation/doctest/PyDocReference.java +++ b/python/src/com/jetbrains/python/documentation/doctest/PyDocReference.java @@ -69,14 +69,16 @@ public class PyDocReference extends PyReferenceImpl { final List> files = languageManager.getInjectedPsiFiles(host); if (files != null) { for (Pair pair : files) { - final PyResolveProcessor processor = new PyResolveProcessor(referencedName); + if (pair.getFirst() instanceof PyFile) { + final PyResolveProcessor processor = new PyResolveProcessor(referencedName); - PyResolveUtil.scopeCrawlUp(processor, (ScopeOwner)pair.getFirst(), referencedName, pair.getFirst()); - final List resultList = getResultsFromProcessor(referencedName, processor, pair.getFirst(), - pair.getFirst()); - if (resultList.size() > 0) { - List ret = RatedResolveResult.sorted(resultList); - return ret.toArray(RatedResolveResult.EMPTY_ARRAY); + PyResolveUtil.scopeCrawlUp(processor, (ScopeOwner)pair.getFirst(), referencedName, pair.getFirst()); + final List resultList = getResultsFromProcessor(referencedName, processor, pair.getFirst(), + pair.getFirst()); + if (resultList.size() > 0) { + List ret = RatedResolveResult.sorted(resultList); + return ret.toArray(RatedResolveResult.EMPTY_ARRAY); + } } } } diff --git a/python/testData/resolve/RegexpAndFStringCombined.py b/python/testData/resolve/RegexpAndFStringCombined.py new file mode 100644 index 000000000000..10e1d09076c9 --- /dev/null +++ b/python/testData/resolve/RegexpAndFStringCombined.py @@ -0,0 +1,5 @@ +import re + +foo = 42 +re.compile(rf'.*{foo}.*') +# \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/python/PyInjectionResolveTest.java b/python/testSrc/com/jetbrains/python/PyInjectionResolveTest.java index 0257c09c3b16..d4a9a260602e 100644 --- a/python/testSrc/com/jetbrains/python/PyInjectionResolveTest.java +++ b/python/testSrc/com/jetbrains/python/PyInjectionResolveTest.java @@ -123,4 +123,9 @@ public class PyInjectionResolveTest extends PyResolveTestCase { public void testFStringInsideAssertStatement() { runWithLanguageLevel(LanguageLevel.PYTHON36, () -> assertResolvesTo(PyParameter.class, "name")); } + + // PY-21493 + public void testRegexpAndFStringCombined() { + runWithLanguageLevel(LanguageLevel.PYTHON36, () -> assertResolvesTo(PyTargetExpression.class, "foo")); + } } diff --git a/python/testSrc/com/jetbrains/python/PyRegexpTest.java b/python/testSrc/com/jetbrains/python/PyRegexpTest.java index 4408573647a9..363601b7042b 100644 --- a/python/testSrc/com/jetbrains/python/PyRegexpTest.java +++ b/python/testSrc/com/jetbrains/python/PyRegexpTest.java @@ -170,6 +170,43 @@ public class PyRegexpTest extends PyTestCase { "(foomissing_valuebaz$)"); } + // PY-21493 + public void testFStringSingleStringRegexpFragmentFirst() { + doTestInjectedText("import re\n" + + "\n" + + "re.search(rf'{42}.*{42}', 'foo')", "missing_value.*missing_value"); + } + + // PY-21493 + public void testFStringSingleStringRegexpFirstFragmentInMiddle() { + doTestInjectedText("import re\n" + + "\n" + + "re.search(rf'.*{42}.*{42}', 'foo')", ".*missing_value.*missing_value"); + } + + // PY-21493 + public void testFStringMultiStringRegexp() { + doTestInjectedText("import re\n" + + "\n" + + "re.search(rf'.*{42}'\n" + + " r'.*{42}.*'\n" + + " rf'{42}.*', 'foo')", ".*missing_value.*{42}.*missing_value.*"); + } + + // PY-21493 + public void testFStringSingleStringIncompleteFragment() { + doTestInjectedText("import re\n" + + "\n" + + "re.search(rf'.*{42.*', 'foo')", ".*missing_value"); + } + + // PY-21493 + public void testFStringSingleStringNestedFragments() { + doTestInjectedText("import re\n" + + "\n" + + "re.search(rf'.*{42:{42}}.*{42}', 'foo')", ".*missing_value.*missing_value"); + } + // PY-18881 public void testVerboseSyntaxWithShortFlag() { final PsiElement element =