From a5f412d2ed8567a9976636a7e1f00896c062d8ce Mon Sep 17 00:00:00 2001 From: Mikhail Golubev Date: Tue, 11 Sep 2018 21:55:57 +0300 Subject: [PATCH] PY-31442 Spellchecker properly handles f-strings It also now correctly processes escape sequences in mixed up raw and normal string elements (previously it stopped extracting escapes altogether once there was at least one raw string element among those belonging to a string literal expression). --- .../PythonSpellcheckerStrategy.java | 48 ++++++++++--------- ...scapesInRawAndNormalGluedStringElements.py | 1 + .../inspections/spelling/fStringExpression.py | 2 + .../inspections/spelling/rawFString.py | 1 + .../jetbrains/python/PySpellCheckerTest.java | 12 ++++- .../spellchecker/tokenizer/TokenConsumer.java | 8 ++++ 6 files changed, 47 insertions(+), 25 deletions(-) create mode 100644 python/testData/inspections/spelling/escapesInRawAndNormalGluedStringElements.py create mode 100644 python/testData/inspections/spelling/fStringExpression.py create mode 100644 python/testData/inspections/spelling/rawFString.py diff --git a/python/src/com/jetbrains/python/spellchecker/PythonSpellcheckerStrategy.java b/python/src/com/jetbrains/python/spellchecker/PythonSpellcheckerStrategy.java index 93cf180e468e..68f10809cc79 100644 --- a/python/src/com/jetbrains/python/spellchecker/PythonSpellcheckerStrategy.java +++ b/python/src/com/jetbrains/python/spellchecker/PythonSpellcheckerStrategy.java @@ -15,10 +15,7 @@ */ package com.jetbrains.python.spellchecker; -import com.intellij.lang.ASTNode; -import com.intellij.openapi.util.Pair; import com.intellij.openapi.util.TextRange; -import com.intellij.openapi.util.text.StringUtil; import com.intellij.psi.PsiElement; import com.intellij.spellchecker.inspections.PlainTextSplitter; import com.intellij.spellchecker.inspections.Splitter; @@ -29,14 +26,15 @@ import com.intellij.util.containers.ContainerUtil; import com.jetbrains.python.PyTokenTypes; import com.jetbrains.python.inspections.PyStringFormatParser; import com.jetbrains.python.psi.PyBinaryExpression; +import com.jetbrains.python.psi.PyFormattedStringElement; +import com.jetbrains.python.psi.PyStringElement; import com.jetbrains.python.psi.PyStringLiteralExpression; -import com.jetbrains.python.psi.PyStringLiteralUtil; +import com.jetbrains.python.psi.impl.PyStringLiteralDecoder; import org.jetbrains.annotations.NotNull; +import java.util.Collections; import java.util.List; -import static com.jetbrains.python.psi.PyUtil.StringNodeInfo; - /** * @author yole */ @@ -45,25 +43,29 @@ public class PythonSpellcheckerStrategy extends SpellcheckingStrategy { @Override public void tokenize(@NotNull PyStringLiteralExpression element, TokenConsumer consumer) { final Splitter splitter = PlainTextSplitter.getInstance(); - final List strNodes = element.getStringNodes(); - final List prefixes = ContainerUtil.mapNotNull(strNodes, n -> StringUtil.nullize(new StringNodeInfo(n).getPrefix())); - - if (element.textContains('\\') && prefixes.stream().noneMatch(PyStringLiteralUtil::isRawPrefix)) { - for (Pair fragment : element.getDecodedFragments()) { - final String value = fragment.getSecond(); - final int startOffset = fragment.getFirst().getStartOffset(); - consumer.consumeToken(element, value, false, startOffset, TextRange.allOf(value), splitter); + for (PyStringElement stringElement : element.getStringElements()) { + final List literalPartRanges; + if (stringElement.isFormatted()) { + literalPartRanges = ((PyFormattedStringElement)stringElement).getLiteralPartRanges(); } - } - else if (!prefixes.isEmpty()) { - for (TextRange valueTextRange : element.getStringValueTextRanges()) { - final String value = valueTextRange.substring(element.getText()); - final int startOffset = valueTextRange.getStartOffset(); - consumer.consumeToken(element, value, false, startOffset, TextRange.allOf(value), splitter); + else { + literalPartRanges = Collections.singletonList(stringElement.getContentRange()); + } + final PyStringLiteralDecoder decoder = new PyStringLiteralDecoder(stringElement); + final boolean containsEscapes = stringElement.textContains('\\'); + for (TextRange literalPartRange : literalPartRanges) { + final List escapeAwareRanges; + if (stringElement.isRaw() || !containsEscapes) { + escapeAwareRanges = Collections.singletonList(literalPartRange); + } + else { + escapeAwareRanges = ContainerUtil.map(decoder.decodeRange(literalPartRange), x -> x.getFirst()); + } + for (TextRange escapeAwareRange : escapeAwareRanges) { + final String valueText = escapeAwareRange.substring(stringElement.getText()); + consumer.consumeToken(stringElement, valueText, false, escapeAwareRange.getStartOffset(), TextRange.allOf(valueText), splitter); + } } - } - else { - consumer.consumeToken(element, splitter); } } } diff --git a/python/testData/inspections/spelling/escapesInRawAndNormalGluedStringElements.py b/python/testData/inspections/spelling/escapesInRawAndNormalGluedStringElements.py new file mode 100644 index 000000000000..80e059183288 --- /dev/null +++ b/python/testData/inspections/spelling/escapesInRawAndNormalGluedStringElements.py @@ -0,0 +1 @@ +('\ncorrect' r'\ncorrect') \ No newline at end of file diff --git a/python/testData/inspections/spelling/fStringExpression.py b/python/testData/inspections/spelling/fStringExpression.py new file mode 100644 index 000000000000..a689a0914b18 --- /dev/null +++ b/python/testData/inspections/spelling/fStringExpression.py @@ -0,0 +1,2 @@ +mstyped = 42 +f'{mstyped}' diff --git a/python/testData/inspections/spelling/rawFString.py b/python/testData/inspections/spelling/rawFString.py new file mode 100644 index 000000000000..a151ca64d3be --- /dev/null +++ b/python/testData/inspections/spelling/rawFString.py @@ -0,0 +1 @@ +(f'\teapot{42}teapot' rf'\teapot{42}teapot') \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/python/PySpellCheckerTest.java b/python/testSrc/com/jetbrains/python/PySpellCheckerTest.java index 84f4e4c6ec51..356aec5e156f 100644 --- a/python/testSrc/com/jetbrains/python/PySpellCheckerTest.java +++ b/python/testSrc/com/jetbrains/python/PySpellCheckerTest.java @@ -48,13 +48,21 @@ public class PySpellCheckerTest extends PyTestCase { runWithLanguageLevel(LanguageLevel.PYTHON36, this::doTest); } + public void testFStringExpression() { + runWithLanguageLevel(LanguageLevel.PYTHON36, this::doTest); + } + + public void testRawFString() { + runWithLanguageLevel(LanguageLevel.PYTHON36, this::doTest); + } + // PY-20987 - public void testGluedStringNodesAfterFirstWithPrefix() { + public void testEscapesInRawAndNormalGluedStringElements() { doTest(); } // PY-20987 - public void testGluedStringNodesAfterFirstRawWithBackslashes() { + public void testGluedStringNodesAfterFirstWithPrefix() { doTest(); } diff --git a/spellchecker/src/com/intellij/spellchecker/tokenizer/TokenConsumer.java b/spellchecker/src/com/intellij/spellchecker/tokenizer/TokenConsumer.java index 8a371a7ff00b..69e9b88b7d41 100644 --- a/spellchecker/src/com/intellij/spellchecker/tokenizer/TokenConsumer.java +++ b/spellchecker/src/com/intellij/spellchecker/tokenizer/TokenConsumer.java @@ -32,6 +32,14 @@ public abstract class TokenConsumer { consumeToken(element, text, useRename, 0, TextRange.allOf(text), splitter); } + /** + * @param element PSI element on which problem descriptor will be set + * @param text literal text that will be analyzed by spellchecker + * @param useRename whether rename quick fix should be suggested instead of "change to" + * @param offset offset inside element that serves as an anchor point for {@code rangeToCheck} + * @param rangeToCheck range text value corresponds to + * @param splitter + */ public abstract void consumeToken(PsiElement element, String text, boolean useRename,