From fbc4b7255f212c4fcc6589342833b9273b47acde Mon Sep 17 00:00:00 2001 From: Mikhail Golubev Date: Tue, 11 Sep 2018 19:02:59 +0300 Subject: [PATCH] PY-31442 Remove the rest of old f-strings implementation --- .../codeInsight/fstrings/FStringParser.java | 251 ------------------ .../PyFStringAwareRecursiveVisitor.java | 62 ----- .../PyMethodMayBeStaticInspection.java | 3 +- .../com/jetbrains/python/PyFStringTest.java | 137 ---------- 4 files changed, 1 insertion(+), 452 deletions(-) delete mode 100644 python/src/com/jetbrains/python/codeInsight/fstrings/FStringParser.java delete mode 100644 python/src/com/jetbrains/python/codeInsight/fstrings/PyFStringAwareRecursiveVisitor.java delete mode 100644 python/testSrc/com/jetbrains/python/PyFStringTest.java diff --git a/python/src/com/jetbrains/python/codeInsight/fstrings/FStringParser.java b/python/src/com/jetbrains/python/codeInsight/fstrings/FStringParser.java deleted file mode 100644 index ccba619b5832..000000000000 --- a/python/src/com/jetbrains/python/codeInsight/fstrings/FStringParser.java +++ /dev/null @@ -1,251 +0,0 @@ -/* - * Copyright 2000-2017 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.jetbrains.python.codeInsight.fstrings; - -import com.intellij.openapi.util.TextRange; -import com.intellij.openapi.util.text.StringUtil; -import com.intellij.util.containers.ContainerUtil; -import com.jetbrains.python.psi.PyStringLiteralUtil; -import org.jetbrains.annotations.NotNull; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; - -/** - * @author Mikhail Golubev - */ -public class FStringParser { - private final String myNodeText; - private final TextRange myNodeContentRange; - private final List myFragments = new ArrayList<>(); - private final List mySingleRightBraces = new ArrayList<>(); - - @NotNull - public static ParseResult parse(@NotNull String nodeText) { - final FStringParser parser = new FStringParser(nodeText); - parser.parseTopLevel(); - return new ParseResult(parser.mySingleRightBraces, parser.myFragments); - } - - private FStringParser(@NotNull String nodeText) { - myNodeText = nodeText; - myNodeContentRange = PyStringLiteralUtil.getContentRange(myNodeText); - } - - private void parseTopLevel() { - int offset = myNodeContentRange.getStartOffset(); - while (offset < myNodeContentRange.getEndOffset()) { - // First, skip named unicode escapes like "\N{LATIN SMALL LETTER A}" wherever they are - final int nextOffset = skipNamedUnicodeEscape(offset); - if (offset != nextOffset) { - offset = nextOffset; - continue; - } - - final char c1 = myNodeText.charAt(offset); - final char c2 = offset + 1 < myNodeContentRange.getEndOffset() ? myNodeText.charAt(offset + 1) : '\0'; - - if ((c1 == '{' && c2 == '{') || (c1 == '}' && c2 == '}')) { - offset += 2; - continue; - } - else if (c1 == '{') { - offset = parseFragment(offset, 1); - continue; - } - // Will be marked as errors - else if (c1 == '}') { - mySingleRightBraces.add(offset); - } - offset++; - } - } - - private int parseFragment(int leftBraceOffset, int depth) { - assert myNodeText.charAt(leftBraceOffset) == '{'; - - int contentEndOffset = -1; - int rightBraceOffset = -1; - - int bracesBalance = 0; - char stringLiteralQuote = '\0'; - int quotesNum = 0; - - // Used for f-strings validation - boolean containsNamedUnicodeEscape = false; - int firstHashOffset = -1; - - int offset = leftBraceOffset + 1; - while (offset < myNodeContentRange.getEndOffset()) { - // Actually they aren't allowed inside expression fragments, but we skip them anyway to prevent injection errors - final int nextOffset = skipNamedUnicodeEscape(offset); - if (offset != nextOffset) { - containsNamedUnicodeEscape = true; - offset = nextOffset; - continue; - } - - final char c1 = myNodeText.charAt(offset); - final char c2 = offset + 1 < myNodeContentRange.getEndOffset() ? myNodeText.charAt(offset + 1) : '\0'; - final char c3 = offset + 2 < myNodeContentRange.getEndOffset() ? myNodeText.charAt(offset + 2) : '\0'; - if (contentEndOffset == -1) { - if (stringLiteralQuote != '\0') { - if (c1 == '\'' || c1 == '"') { - final int size = c2 == c1 && c3 == c1 ? 3 : 1; - if (stringLiteralQuote == c1 && size == quotesNum) { - stringLiteralQuote = '\0'; - offset += size; - continue; - } - } - else if (c1 == '\\') { - offset += 2; - continue; - } - } - else if (c1 == '\'' || c1 == '"') { - quotesNum = c2 == c1 && c3 == c1 ? 3 : 1; - stringLiteralQuote = c1; - offset += quotesNum; - continue; - } - else if (c1 == '#' && firstHashOffset == -1) { - firstHashOffset = offset; - } - else if (c1 == '{' || c1 == '[' || c1 == '(') { - bracesBalance++; - } - else if (bracesBalance > 0 && (c1 == '}' || c1 == ']' || c1 == ')')) { - bracesBalance--; - } - else if (bracesBalance == 0 && (c1 == '}' || (c1 == '!' && c2 != '=') || c1 == ':')) { - contentEndOffset = offset; - if (c1 == '}') { - rightBraceOffset = offset; - offset++; - break; - } - } - } - else if (c1 == '{') { - offset = parseFragment(offset, depth + 1); - continue; - } - else if (c1 == '}') { - rightBraceOffset = offset; - offset++; - break; - } - offset++; - } - if (contentEndOffset == -1) { - contentEndOffset = offset; - } - myFragments.add(new Fragment(leftBraceOffset, - contentEndOffset, - rightBraceOffset, - containsNamedUnicodeEscape, - firstHashOffset, - depth)); - return offset; - } - - private int skipNamedUnicodeEscape(int offset) { - if (StringUtil.startsWith(myNodeText, offset, "\\N{")) { - final int rightBraceOffset = myNodeText.indexOf('}', offset + 3); - return rightBraceOffset < 0 ? myNodeContentRange.getEndOffset() : rightBraceOffset + 1; - } - return offset; - } - - public static class Fragment { - private final int myLeftBraceOffset; - private final int myRightBraceOffset; - private final int myContentEndOffset; - private final boolean myContainsNamedUnicodeEscape; - private final int myFirstHashOffset; - private final int myDepth; - - private Fragment(int leftBraceOffset, - int contentEndOffset, - int rightBraceOffset, - boolean containsUnicodeEscape, - int firstHashOffset, - int depth) { - assert leftBraceOffset < contentEndOffset; - assert rightBraceOffset < 0 || contentEndOffset <= rightBraceOffset; - assert firstHashOffset < 0 || leftBraceOffset < firstHashOffset && firstHashOffset < contentEndOffset; - - myLeftBraceOffset = leftBraceOffset; - myRightBraceOffset = rightBraceOffset; - myContentEndOffset = contentEndOffset; - myContainsNamedUnicodeEscape = containsUnicodeEscape; - myFirstHashOffset = firstHashOffset; - myDepth = depth; - } - - public int getLeftBraceOffset() { - return myLeftBraceOffset; - } - - public int getRightBraceOffset() { - return myRightBraceOffset; - } - - public int getContentEndOffset() { - return myContentEndOffset; - } - - public boolean containsNamedUnicodeEscape() { - return myContainsNamedUnicodeEscape; - } - - public int getFirstHashOffset() { - return myFirstHashOffset; - } - - public int getDepth() { - return myDepth; - } - - @NotNull - public TextRange getContentRange() { - return TextRange.create(myLeftBraceOffset + 1, myContentEndOffset); - } - } - - public static class ParseResult { - private final List mySingleRightBraces; - private final List myFragments; - - private ParseResult(@NotNull List singleRightBraces, @NotNull List fragments) { - mySingleRightBraces = singleRightBraces; - myFragments = ContainerUtil.sorted(fragments, Comparator.comparingInt(Fragment::getLeftBraceOffset)); - } - - @NotNull - public List getSingleRightBraces() { - return Collections.unmodifiableList(mySingleRightBraces); - } - - @NotNull - public List getFragments() { - return Collections.unmodifiableList(myFragments); - } - } -} diff --git a/python/src/com/jetbrains/python/codeInsight/fstrings/PyFStringAwareRecursiveVisitor.java b/python/src/com/jetbrains/python/codeInsight/fstrings/PyFStringAwareRecursiveVisitor.java deleted file mode 100644 index 2e615369b830..000000000000 --- a/python/src/com/jetbrains/python/codeInsight/fstrings/PyFStringAwareRecursiveVisitor.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2000-2017 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.jetbrains.python.codeInsight.fstrings; - -import com.intellij.lang.injection.InjectedLanguageManager; -import com.intellij.openapi.util.Pair; -import com.intellij.openapi.util.TextRange; -import com.intellij.psi.PsiElement; -import com.intellij.util.containers.ContainerUtil; -import com.jetbrains.python.psi.PyFile; -import com.jetbrains.python.psi.PyRecursiveElementVisitor; -import com.jetbrains.python.psi.PyStringLiteralExpression; -import com.jetbrains.python.psi.PyUtil; - -import java.util.List; - -import static com.jetbrains.python.psi.PyUtil.as; - -/** - * Implementation of {@link PyRecursiveElementVisitor} that recursively visits Python files injected into f-strings. - * You can check whether you are inside injected fragment using {@link #myContainingFString} field. - */ -public class PyFStringAwareRecursiveVisitor extends PyRecursiveElementVisitor { - protected PyStringLiteralExpression myContainingFString; - - @Override - public void visitPyStringLiteralExpression(PyStringLiteralExpression pyString) { - final List formatNodeRanges = ContainerUtil.mapNotNull(pyString.getStringNodes(), node -> { - final PyUtil.StringNodeInfo nodeInfo = new PyUtil.StringNodeInfo(node); - return nodeInfo.isFormatted() ? nodeInfo.getAbsoluteContentRange().shiftRight(-pyString.getTextOffset()) : null; - }); - if (!formatNodeRanges.isEmpty()) { - final InjectedLanguageManager injectionManager = InjectedLanguageManager.getInstance(pyString.getProject()); - for (Pair pair : ContainerUtil.notNullize(injectionManager.getInjectedPsiFiles(pyString))) { - final PyFile pyFile = as(pair.getFirst(), PyFile.class); - if (pyFile != null && ContainerUtil.exists(formatNodeRanges, range -> range.contains(pair.getSecond()))) { - myContainingFString = pyString; - try { - pyFile.accept(this); - } - finally { - myContainingFString = null; - } - } - } - } - super.visitPyStringLiteralExpression(pyString); - } -} diff --git a/python/src/com/jetbrains/python/inspections/PyMethodMayBeStaticInspection.java b/python/src/com/jetbrains/python/inspections/PyMethodMayBeStaticInspection.java index d9db08fe5de3..cbceb31c84f1 100644 --- a/python/src/com/jetbrains/python/inspections/PyMethodMayBeStaticInspection.java +++ b/python/src/com/jetbrains/python/inspections/PyMethodMayBeStaticInspection.java @@ -22,7 +22,6 @@ import com.intellij.psi.PsiElement; import com.intellij.psi.PsiElementVisitor; import com.jetbrains.python.PyBundle; import com.jetbrains.python.PyNames; -import com.jetbrains.python.codeInsight.fstrings.PyFStringAwareRecursiveVisitor; import com.jetbrains.python.inspections.quickfix.PyMakeFunctionFromMethodQuickFix; import com.jetbrains.python.inspections.quickfix.PyMakeMethodStaticQuickFix; import com.jetbrains.python.psi.*; @@ -93,7 +92,7 @@ public class PyMethodMayBeStaticInspection extends PyInspection { } final boolean[] mayBeStatic = {true}; - PyRecursiveElementVisitor visitor = new PyFStringAwareRecursiveVisitor() { + PyRecursiveElementVisitor visitor = new PyRecursiveElementVisitor() { @Override public void visitPyRaiseStatement(PyRaiseStatement node) { super.visitPyRaiseStatement(node); diff --git a/python/testSrc/com/jetbrains/python/PyFStringTest.java b/python/testSrc/com/jetbrains/python/PyFStringTest.java deleted file mode 100644 index 18b627af52b8..000000000000 --- a/python/testSrc/com/jetbrains/python/PyFStringTest.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright 2000-2016 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.jetbrains.python; - -import com.intellij.openapi.util.Pair; -import com.intellij.openapi.util.TextRange; -import com.intellij.util.containers.ContainerUtil; -import com.jetbrains.python.codeInsight.fstrings.FStringParser; -import com.jetbrains.python.codeInsight.fstrings.FStringParser.Fragment; -import com.jetbrains.python.fixtures.PyTestCase; -import org.intellij.lang.annotations.Language; -import org.jetbrains.annotations.NotNull; - -import java.util.ArrayList; -import java.util.List; - -/** - * @author Mikhail Golubev - */ -public class PyFStringTest extends PyTestCase { - - private static void doTestRanges(@NotNull @Language("Python") String fstringLiteral) { - final Pair> pair = extractOffsets(fstringLiteral); - final List barOffsets = pair.getSecond(); - assertTrue("Odd number of markers", barOffsets.size() % 2 == 0); - final List fragments = FStringParser.parse(pair.getFirst()).getFragments(); - final List actualRanges = ContainerUtil.map(fragments, Fragment::getContentRange); - final List expectedRanges = new ArrayList<>(); - for (int i = 0; i < barOffsets.size(); i += 2) { - expectedRanges.add(TextRange.create(barOffsets.get(i), barOffsets.get(i + 1))); - } - assertSameElements(actualRanges, expectedRanges); - } - - private static void doTestSingleRightBraces(@NotNull @Language("Python") String fstringLiteral) { - final Pair> pair = extractOffsets(fstringLiteral); - assertEquals(pair.getSecond(), FStringParser.parse(pair.getFirst()).getSingleRightBraces()); - } - - @NotNull - private static Pair> extractOffsets(@NotNull String fstringLiteral) { - final StringBuilder builder = new StringBuilder(); - int lastBarOffset = -1; - final List offsets = new ArrayList<>(); - while (true) { - final int nextOffset = fstringLiteral.indexOf("|", lastBarOffset + 1); - if (nextOffset < 0) { - builder.append(fstringLiteral.substring(lastBarOffset + 1)); - break; - } - builder.append(fstringLiteral, lastBarOffset + 1, nextOffset); - lastBarOffset = nextOffset; - offsets.add(lastBarOffset - offsets.size()); - } - return Pair.create(builder.toString(), offsets); - } - - public void testSimple() { - doTestRanges("f'{|x|} {|y|} {|42|'"); - } - - public void testEscapedBraces() { - doTestRanges("f'{{{|x|}}}{{}}{{{|x|}'"); - } - - public void testBracesInside() { - doTestRanges("f'{| {1, 2, 3} |} {| {x for x in range(10)} |}'"); - } - - public void testBraceInsideNestedLiteral() { - doTestRanges("f'{|\"{x}}}\"|}'"); - doTestRanges("f'{|d[\"}\"]|}'"); - doTestRanges("f'''{|\"'}\"|}'''"); - doTestRanges("f'''{|\"\\\"}\"|}'''"); - doTestRanges("f'''{|\"\"\"}'\"}'\"\"\"|}'''"); - doTestRanges("f\"{|'''}\\\"'\\\"'''|}\""); - } - - public void testChunkTypeConversionsAndFormatSpecifiers() { - doTestRanges("f'{|x|!s} {|y|!r}}'"); - doTestRanges("f'{|x|:^42}}'"); - doTestRanges("f'{|x|!s:{|y|}.{|z|}}'"); - doTestRanges("f'{|x|:{|sum({x for x in range(10)})|}}'"); - } - - public void testNestedLambda() { - doTestRanges("f'{|(lambda: x)|}'"); - } - - public void testNotEquals() { - doTestRanges("f'{|x != 42|}'"); - } - - // PY-20785 - public void testNamedUnicodeEscapes() { - doTestRanges("f'\\N{foo}\\N{}\\N{{{{{}{|42 + \\N{DIGIT ONE}|}'"); - doTestRanges("f'{|x|:\\N{DIGIT_ONE}}'"); - } - - // PY-20785 - public void testUnicodeEscapeInsideExpressionFragment() { - doTestUnicodeEscapeDetection("f'{\\N{FOO}}'", true); - doTestUnicodeEscapeDetection("f'{\"\\N{FOO\"}'", true); - doTestUnicodeEscapeDetection("f'{\"\\N{\"}'", true); - doTestUnicodeEscapeDetection("f'{\"\\\\N{FOO}\"}'", false); - } - - private static void doTestUnicodeEscapeDetection(String fStringText, boolean expected) { - final List fragments = FStringParser.parse(fStringText).getFragments(); - assertSize(1, fragments); - final Fragment offsets = fragments.get(0); - assertEquals(expected, offsets.containsNamedUnicodeEscape()); - } - - // PY-20897 - public void testSingleRightBraces() { - doTestSingleRightBraces("f'|}'"); - doTestSingleRightBraces("f'{x}{y}|}'"); - doTestSingleRightBraces("f'{x}}}|}'"); - doTestSingleRightBraces("f'{x:{y}}|}'"); - doTestSingleRightBraces("f'{x:{y}}}}'"); - doTestSingleRightBraces("f'{\"}\":{\"}\"}}'"); - } -}