diff --git a/python/IntelliLang-python/src/pyInjections.xml b/python/IntelliLang-python/src/pyInjections.xml index a75015e02441..f561c1d33016 100644 --- a/python/IntelliLang-python/src/pyInjections.xml +++ b/python/IntelliLang-python/src/pyInjections.xml @@ -9,6 +9,6 @@ "SQL select/delete/insert/update/create" - + diff --git a/python/openapi/src/com/jetbrains/python/psi/PyStringLiteralFileReferenceSet.java b/python/openapi/src/com/jetbrains/python/psi/PyStringLiteralFileReferenceSet.java index e74cd80a3260..6c470c739c36 100644 --- a/python/openapi/src/com/jetbrains/python/psi/PyStringLiteralFileReferenceSet.java +++ b/python/openapi/src/com/jetbrains/python/psi/PyStringLiteralFileReferenceSet.java @@ -20,17 +20,19 @@ import com.intellij.openapi.util.SystemInfo; import com.intellij.openapi.util.TextRange; import com.intellij.psi.PsiReferenceProvider; import com.intellij.psi.impl.source.resolve.reference.impl.providers.FileReference; -import com.intellij.psi.impl.source.resolve.reference.impl.providers.FileReferenceSet; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import java.util.ArrayList; import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * @author traff */ public class PyStringLiteralFileReferenceSet extends RootFileReferenceSet { + public static final Pattern DELIMITERS = Pattern.compile("\\\\|/"); private final PyStringLiteralExpression myStringLiteralExpression; @@ -57,61 +59,33 @@ public class PyStringLiteralFileReferenceSet extends RootFileReferenceSet { protected void reparse() { //noinspection ConstantConditions if (myStringLiteralExpression != null) { - MyTextRangeConsumer textRangeConsumer = new MyTextRangeConsumer(this); - - myStringLiteralExpression.iterateCharacterRanges(textRangeConsumer); - textRangeConsumer.finish(); - - List referencesList = textRangeConsumer.myReferenceList; - - myReferences = referencesList.toArray(new FileReference[referencesList.size()]); + final List references = getFileReferences(myStringLiteralExpression); + myReferences = references.toArray(new FileReference[references.size()]); } } - private static class MyTextRangeConsumer implements PyStringLiteralExpression.TextRangeConsumer { - private final StringBuilder myItem = new StringBuilder(); - private int myStartOffset = -1; - private int myIndex = 0; - private int myEndOffset = -1; - private final FileReferenceSet myFileReferenceSet; - - - private final List myReferenceList = new ArrayList(); - - private MyTextRangeConsumer(FileReferenceSet set) { - myFileReferenceSet = set; - } - - @Override - public boolean process(int startOffset, int endOffset, String value) { - if ("\\".equals(value) || "/".equals(value)) { - addReference(startOffset); + @NotNull + private List getFileReferences(@NotNull PyStringLiteralExpression expression) { + final String value = expression.getStringValue(); + final Matcher matcher = DELIMITERS.matcher(value); + int start = 0; + int index = 0; + final List results = new ArrayList(); + while (matcher.find()) { + final String s = value.substring(start, matcher.start()); + if (!s.isEmpty()) { + final TextRange range = TextRange.create(expression.valueOffsetToTextOffset(start), + expression.valueOffsetToTextOffset(matcher.start())); + results.add(createFileReference(range, index++, s)); } - else { - if (myStartOffset == -1) { - myStartOffset = startOffset; - } - myEndOffset = endOffset; - myItem.append(value); - } - return true; + start = matcher.end(); } - - private void addReference(int startOffset) { - if (myStartOffset != -1) { - final FileReference ref = myFileReferenceSet.createFileReference( - new TextRange(myStartOffset, startOffset), - myIndex++, - myItem.toString()); - myReferenceList.add(ref); - myStartOffset = -1; - myItem.setLength(0); - } - } - - - public void finish() { - addReference(myEndOffset); + final String s = value.substring(start); + if (!s.isEmpty()) { + final TextRange range = TextRange.create(expression.valueOffsetToTextOffset(start), + expression.valueOffsetToTextOffset(value.length())); + results.add(createFileReference(range, index, s)); } + return results; } } diff --git a/python/psi-api/src/com/jetbrains/python/psi/PyStringLiteralExpression.java b/python/psi-api/src/com/jetbrains/python/psi/PyStringLiteralExpression.java index 8c3182db377f..f7dd122680df 100644 --- a/python/psi-api/src/com/jetbrains/python/psi/PyStringLiteralExpression.java +++ b/python/psi-api/src/com/jetbrains/python/psi/PyStringLiteralExpression.java @@ -16,8 +16,10 @@ package com.jetbrains.python.psi; import com.intellij.lang.ASTNode; +import com.intellij.openapi.util.Pair; import com.intellij.openapi.util.TextRange; import com.intellij.psi.PsiLanguageInjectionHost; +import org.jetbrains.annotations.NotNull; import java.util.List; @@ -28,20 +30,6 @@ public interface PyStringLiteralExpression extends PyLiteralExpression, StringLi int valueOffsetToTextOffset(int valueOffset); - void iterateCharacterRanges(TextRangeConsumer consumer); - - /** - * Iterator over decoded string characters. - */ - interface TextRangeConsumer { - /** - * Process a decoded character. - * - * @param startOffset start offset in the un-decoded string - * @param endOffset end offset in the un-decoded string - * @param value decoded character value - * @return false in order to stop iteration - */ - boolean process(int startOffset, int endOffset, String value); - } + @NotNull + List> getDecodedFragments(); } diff --git a/python/src/com/jetbrains/python/codeInsight/PyInjectionUtil.java b/python/src/com/jetbrains/python/codeInsight/PyInjectionUtil.java index 7ce44b5531c9..9a7edad88cc5 100644 --- a/python/src/com/jetbrains/python/codeInsight/PyInjectionUtil.java +++ b/python/src/com/jetbrains/python/codeInsight/PyInjectionUtil.java @@ -44,7 +44,7 @@ public class PyInjectionUtil { @Nullable public static PsiElement getLargestStringLiteral(@NotNull PsiElement context) { PsiElement element = null; - for (PsiElement current = context; current != null && isStringLiteralPart(current); current = current.getParent()) { + for (PsiElement current = context; current != null && isStringLiteralPart(current, element); current = current.getParent()) { element = current; } return element; @@ -58,24 +58,32 @@ public class PyInjectionUtil { processStringLiteral(element, registrar, "", "", Formatting.PERCENT); } - private static boolean isStringLiteralPart(@NotNull PsiElement element) { - if (element instanceof PyStringLiteralExpression) { + private static boolean isStringLiteralPart(@NotNull PsiElement element, @Nullable PsiElement context) { + if (element == context) { + return true; + } + else if (element instanceof PyStringLiteralExpression) { return true; } else if (element instanceof PyParenthesizedExpression) { final PyExpression contained = ((PyParenthesizedExpression)element).getContainedExpression(); - return contained != null && isStringLiteralPart(contained); + return contained != null && isStringLiteralPart(contained, context); } else if (element instanceof PyBinaryExpression) { final PyBinaryExpression expr = (PyBinaryExpression)element; final PyExpression left = expr.getLeftExpression(); final PyExpression right = expr.getRightExpression(); - return (expr.isOperator("+") && (isStringLiteralPart(left) || right != null && isStringLiteralPart(right))) || - expr.isOperator("%") && isStringLiteralPart(left); + if (expr.isOperator("+")) { + return isStringLiteralPart(left, context) || right != null && isStringLiteralPart(right, context); + } + else if (expr.isOperator("%")) { + return right != context && isStringLiteralPart(left, context); + } + return false; } else if (element instanceof PyCallExpression) { final PyExpression qualifier = getFormatCallQualifier((PyCallExpression)element); - return qualifier != null && isStringLiteralPart(qualifier); + return qualifier != null && isStringLiteralPart(qualifier, context); } return false; } @@ -147,9 +155,9 @@ public class PyInjectionUtil { final PyBinaryExpression expr = (PyBinaryExpression)element; final PyExpression left = expr.getLeftExpression(); final PyExpression right = expr.getRightExpression(); - final boolean isLeftString = isStringLiteralPart(left); + final boolean isLeftString = isStringLiteralPart(left, null); if (expr.isOperator("+")) { - final boolean isRightString = right != null && isStringLiteralPart(right); + final boolean isRightString = right != null && isStringLiteralPart(right, null); if (isLeftString) { processStringLiteral(left, registrar, prefix, isRightString ? "" : missingValue, formatting); } diff --git a/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java b/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java index 7f03dcdb05a8..7595fde33fe8 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java @@ -21,7 +21,6 @@ import com.intellij.lang.Language; import com.intellij.lang.injection.InjectedLanguageManager; import com.intellij.navigation.ItemPresentation; import com.intellij.openapi.util.Pair; -import com.intellij.openapi.util.Ref; import com.intellij.openapi.util.TextRange; import com.intellij.psi.*; import com.intellij.psi.impl.source.resolve.reference.ReferenceProvidersRegistry; @@ -50,9 +49,21 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt public static final Pattern PATTERN_ESCAPE = Pattern .compile("\\\\(\n|\\\\|'|\"|a|b|f|n|r|t|v|([0-7]{1,3})|x([0-9a-fA-F]{1,2})" + "|N(\\{.*?\\})|u([0-9a-fA-F]{4})|U([0-9a-fA-F]{8}))"); // -> 1 -> 2 <--> 3 <- -> 4 <--> 5 <- -> 6 <-<- + + private enum EscapeRegexGroup { + WHOLE_MATCH, + ESCAPED_SUBSTRING, + OCTAL, + HEXADECIMAL, + UNICODE_NAMED, + UNICODE_16BIT, + UNICODE_32BIT + } + private static final Map escapeMap = initializeEscapeMap(); private String stringValue; private List valueTextRanges; + @Nullable private List> myDecodedFragments; private final DefaultRegExpPropertiesProvider myPropertiesProvider; private static Map initializeEscapeMap() { @@ -85,6 +96,7 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt super.subtreeChanged(); stringValue = null; valueTextRanges = null; + myDecodedFragments = null; } public List getStringValueTextRanges() { @@ -143,20 +155,82 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt return text.length() > 0 && Character.toUpperCase(text.charAt(0)) == 'C'; } - public void iterateCharacterRanges(TextRangeConsumer consumer) { - int elStart = getTextRange().getStartOffset(); - for (ASTNode child : getStringNodes()) { - final String text = child.getText(); - TextRange textRange = getNodeTextRange(text); - int offset = child.getTextRange().getStartOffset() - elStart + textRange.getStartOffset(); - String undecoded = textRange.substring(text); - if (!iterateCharacterRanges(consumer, undecoded, offset, isRaw(text), isUnicode(text))) { - break; + @Override + @NotNull + public List> getDecodedFragments() { + if (myDecodedFragments == null) { + final List> result = new ArrayList>(); + final int elementStart = getTextRange().getStartOffset(); + for (ASTNode node : getStringNodes()) { + final String text = node.getText(); + final TextRange textRange = getNodeTextRange(text); + final int offset = node.getTextRange().getStartOffset() - elementStart + textRange.getStartOffset(); + final String encoded = textRange.substring(text); + result.addAll(getDecodedFragments(encoded, offset, isRaw(text), isUnicode(text))); } + myDecodedFragments = result; } + return myDecodedFragments; } + @NotNull + private static List> getDecodedFragments(@NotNull String encoded, int offset, boolean raw, boolean unicode) { + final List> result = new ArrayList>(); + final Matcher escMatcher = PATTERN_ESCAPE.matcher(encoded); + int index = 0; + while (escMatcher.find(index)) { + if (index < escMatcher.start()) { + final TextRange range = TextRange.create(index, escMatcher.start()); + final TextRange offsetRange = range.shiftRight(offset); + result.add(Pair.create(offsetRange, range.substring(encoded))); + } + final String octal = escapeRegexGroup(escMatcher, EscapeRegexGroup.OCTAL); + final String hex = escapeRegexGroup(escMatcher, EscapeRegexGroup.HEXADECIMAL); + // TODO: Implement unicode character name escapes: EscapeRegexGroup.UNICODE_NAMED + final String unicode16 = escapeRegexGroup(escMatcher, EscapeRegexGroup.UNICODE_16BIT); + final String unicode32 = escapeRegexGroup(escMatcher, EscapeRegexGroup.UNICODE_32BIT); + + final boolean escapedUnicode = raw && unicode || !raw; + + final String str; + if (!raw && octal != null) { + str = new String(new char[]{(char)Integer.parseInt(octal, 8)}); + } + else if (!raw && hex != null) { + str = new String(new char[]{(char)Integer.parseInt(hex, 16)}); + } + else if (escapedUnicode && unicode16 != null) { + str = unicode ? new String(new char[]{(char)Integer.parseInt(unicode16, 16)}) : unicode16; + } + else if (escapedUnicode && unicode32 != null) { + str = unicode ? new String(Character.toChars((int)Long.parseLong(unicode32, 16))) : unicode32; + } + else if (raw) { + str = escapeRegexGroup(escMatcher, EscapeRegexGroup.WHOLE_MATCH); + } + else { + final String toReplace = escapeRegexGroup(escMatcher, EscapeRegexGroup.ESCAPED_SUBSTRING); + str = escapeMap.get(toReplace); + } + + if (str != null) { + final TextRange wholeMatch = TextRange.create(escMatcher.start(), escMatcher.end()); + result.add(Pair.create(wholeMatch.shiftRight(offset), str)); + } + + index = escMatcher.end(); + } + final TextRange range = TextRange.create(index, encoded.length()); + final TextRange offRange = range.shiftRight(offset); + result.add(Pair.create(offRange, range.substring(encoded))); + return result; + } + + @Nullable + private static String escapeRegexGroup(@NotNull Matcher matcher, EscapeRegexGroup group) { + return matcher.group(group.ordinal()); + } public List getStringNodes() { return Arrays.asList(getNode().getChildren(PyTokenTypes.STRING_NODES)); @@ -165,14 +239,11 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt public String getStringValue() { //ASTNode child = getNode().getFirstChildNode(); //assert child != null; - if (stringValue == null) { + if (stringValue == null) { final StringBuilder out = new StringBuilder(); - iterateCharacterRanges(new TextRangeConsumer() { - public boolean process(int startOffset, int endOffset, String value) { - out.append(value); - return true; - } - }); + for (Pair fragment : getDecodedFragments()) { + out.append(fragment.getSecond()); + } stringValue = out.toString(); } return stringValue; @@ -190,106 +261,6 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt return new TextRange(0, getTextLength()); } - private static boolean iterateCharacterRanges(TextRangeConsumer consumer, String undecoded, int off, boolean raw, boolean unicode) { - if (raw) { - return iterateRawCharacterRanges(consumer, undecoded, off, unicode); - } - Matcher escMatcher = PATTERN_ESCAPE.matcher(undecoded); - int index = 0; - while (escMatcher.find(index)) { - for (int i = index; i < escMatcher.start(); i++) { - if (!consumer.process(off + i, off + i + 1, Character.toString(undecoded.charAt(i)))) { - return false; - } - } - String octal = escMatcher.group(2); - String hex = escMatcher.group(3); - String str = null; - if (octal != null) { - str = new String(new char[]{(char)Integer.parseInt(octal, 8)}); - - } - else if (hex != null) { - str = new String(new char[]{(char)Integer.parseInt(hex, 16)}); - - } - else { - String toReplace = escMatcher.group(1); - String replacement = escapeMap.get(toReplace); - if (replacement != null) { - str = replacement; - } - } - String unicodeName = escMatcher.group(4); - String unicode32 = escMatcher.group(6); - - if (unicode32 != null) { - str = unicode ? new String(Character.toChars((int)Long.parseLong(unicode32, 16))) : unicode32; - } - if (unicodeName != null) { - //TOLATER: implement unicode character name escapes - } - String unicode16 = escMatcher.group(5); - if (unicode16 != null) { - str = unicode ? new String(new char[]{(char)Integer.parseInt(unicode16, 16)}) : unicode16; - } - - if (str != null) { - int start = escMatcher.start(); - int end = escMatcher.end(); - if (!consumer.process(off + start, off + end, str)) { - return false; - } - } - index = escMatcher.end(); - } - for (int i = index; i < undecoded.length(); i++) { - if (!consumer.process(off + i, off + i + 1, Character.toString(undecoded.charAt(i)))) { - return false; - } - } - return true; - } - - private static boolean iterateRawCharacterRanges(TextRangeConsumer consumer, String undecoded, int off, boolean unicode) { - for (int i = 0; i < undecoded.length(); i++) { - char c = undecoded.charAt(i); - if (unicode && c == '\\' && i < undecoded.length()-1) { - char c2 = undecoded.charAt(i+1); - if (c2 == 'u' && i < undecoded.length()-5) { - try { - char u = (char) Integer.parseInt(undecoded.substring(i+2, i+6), 16); - if (!consumer.process(off+i, off+i+ 6, Character.toString(u))) { - return false; - } - } - catch (NumberFormatException ignore) { } - //noinspection AssignmentToForLoopParameter - i += 5; - continue; - } - if (c2 == 'U' && i < undecoded.length()-9) { - // note: Java has 16-bit chars, so this code will truncate characters which don't fit in 16 bits - try { - char u = (char) Long.parseLong(undecoded.substring(i+2, i+10), 16); - if (!consumer.process(off+i, off+i+10, Character.toString(u))) { - return false; - } - } - catch (NumberFormatException ignore) { } - //noinspection AssignmentToForLoopParameter - i += 9; - continue; - } - } - if (!consumer.process(off + i, off + i + 1, Character.toString(c))) { - return false; - } - } - - return true; - } - @Override public String toString() { return super.toString() + ": " + getStringValue(); @@ -368,57 +339,60 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt @Override public boolean decode(@NotNull final TextRange rangeInsideHost, @NotNull final StringBuilder outChars) { - final PyDocStringOwner - docStringOwner = PsiTreeUtil.getParentOfType(myHost, PyDocStringOwner.class); - if (docStringOwner != null && myHost.equals(docStringOwner.getDocStringExpression())) { - outChars.append(myHost.getText(), rangeInsideHost.getStartOffset(), rangeInsideHost.getEndOffset()); - } - else { - myHost.iterateCharacterRanges(new TextRangeConsumer() { - public boolean process(int startOffset, int endOffset, String value) { - int xsectStart = Math.max(startOffset, rangeInsideHost.getStartOffset()); - int xsectEnd = Math.min(endOffset, rangeInsideHost.getEndOffset()); - if (xsectEnd > xsectStart) { - outChars.append(value); - } - return endOffset < rangeInsideHost.getEndOffset(); + for (Pair fragment : myHost.getDecodedFragments()) { + final TextRange encodedTextRange = fragment.getFirst(); + final TextRange intersection = encodedTextRange.intersection(rangeInsideHost); + if (intersection != null && !intersection.isEmpty()) { + final String value = fragment.getSecond(); + final String intersectedValue; + if (value.length() == 1 || value.length() == intersection.getLength()) { + intersectedValue = value; } - }); + else { + final int start = Math.max(0, rangeInsideHost.getStartOffset() - encodedTextRange.getStartOffset()); + final int end = Math.min(value.length(), start + intersection.getLength()); + intersectedValue = value.substring(start, end); + } + outChars.append(intersectedValue); + } } return true; } @Override public int getOffsetInHost(final int offsetInDecoded, @NotNull final TextRange rangeInsideHost) { - final Ref resultRef = Ref.create(-1); - final Ref indexRef = Ref.create(0); - final Ref lastEndOffsetRef = Ref.create(-1); - myHost.iterateCharacterRanges(new TextRangeConsumer() { - @Override - public boolean process(int startOffset, int endOffset, String value) { - if (startOffset > rangeInsideHost.getEndOffset()) { - return false; + int offset = 0; + int endOffset = -1; + for (Pair fragment : myHost.getDecodedFragments()) { + final TextRange encodedTextRange = fragment.getFirst(); + final TextRange intersection = encodedTextRange.intersection(rangeInsideHost); + if (intersection != null && !intersection.isEmpty()) { + final String value = fragment.getSecond(); + final int valueLength = value.length(); + final int intersectionLength = intersection.getLength(); + if (valueLength == 0) { + return -1; } - lastEndOffsetRef.set(endOffset); - if (startOffset >= rangeInsideHost.getStartOffset()) { - final int i = indexRef.get(); - if (i == offsetInDecoded) { - resultRef.set(startOffset); - return false; + else if (valueLength == 1) { + if (offset == offsetInDecoded) { + return intersection.getStartOffset(); } - indexRef.set(i + 1); + offset++; } - return true; + else { + if (offset + intersectionLength >= offsetInDecoded) { + final int delta = offsetInDecoded - offset; + return intersection.getStartOffset() + delta; + } + offset += intersectionLength; + } + endOffset = intersection.getEndOffset(); } - }); - final int result = resultRef.get(); - if (result != -1) { - return result; } - // We should handle the position of a character at the end of rangeInsideHost, because LeafPatcher expects it to be valid - final int lastEndOffset = lastEndOffsetRef.get(); - if (indexRef.get() == offsetInDecoded && lastEndOffset == rangeInsideHost.getEndOffset()) { - return lastEndOffset; + // XXX: According to the real use of getOffsetInHost() it should return the correct host offset for the offset in decoded at the + // end of the range inside host, not -1 + if (offset == offsetInDecoded) { + return endOffset; } return -1; } @@ -431,23 +405,7 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt @Override public int valueOffsetToTextOffset(int valueOffset) { - final Ref offsetInDecodedRef = new Ref(valueOffset); - final Ref result = new Ref(-1); - iterateCharacterRanges(new TextRangeConsumer() { - public boolean process(int startOffset, int endOffset, String value) { - if (value.length() > offsetInDecodedRef.get()) { - result.set(startOffset + offsetInDecodedRef.get()); - return false; - } - offsetInDecodedRef.set(offsetInDecodedRef.get() - value.length()); - if (offsetInDecodedRef.get() == 0) { - result.set(endOffset); - return false; - } - return true; - } - }); - return result.get(); + return createLiteralTextEscaper().getOffsetInHost(valueOffset, getStringValueTextRange()); } public boolean characterNeedsEscaping(char c) { diff --git a/python/src/com/jetbrains/python/spellchecker/PythonSpellcheckerStrategy.java b/python/src/com/jetbrains/python/spellchecker/PythonSpellcheckerStrategy.java index 4a0a59d15c1f..b24e53c557ad 100644 --- a/python/src/com/jetbrains/python/spellchecker/PythonSpellcheckerStrategy.java +++ b/python/src/com/jetbrains/python/spellchecker/PythonSpellcheckerStrategy.java @@ -15,6 +15,7 @@ */ package com.jetbrains.python.spellchecker; +import com.intellij.openapi.util.Pair; import com.intellij.openapi.util.TextRange; import com.intellij.psi.PsiElement; import com.intellij.spellchecker.inspections.PlainTextSplitter; @@ -26,7 +27,6 @@ import com.jetbrains.python.PyTokenTypes; import com.jetbrains.python.inspections.PyStringFormatParser; import com.jetbrains.python.psi.PyBinaryExpression; import com.jetbrains.python.psi.PyStringLiteralExpression; -import com.jetbrains.python.psi.impl.PyStringLiteralExpressionImpl; import org.jetbrains.annotations.NotNull; import java.util.List; @@ -41,9 +41,11 @@ public class PythonSpellcheckerStrategy extends SpellcheckingStrategy { Splitter splitter = PlainTextSplitter.getInstance(); String text = element.getText(); if (text.indexOf('\\') >= 0) { - MyTextRangeConsumer textRangeConsumer = new MyTextRangeConsumer(element, consumer); - ((PyStringLiteralExpressionImpl) element).iterateCharacterRanges(textRangeConsumer); - textRangeConsumer.processCurrentToken(); // process last token + for (Pair fragment : element.getDecodedFragments()) { + final String value = fragment.getSecond(); + consumer.consumeToken(element, value, false, fragment.getFirst().getStartOffset(), TextRange.allOf(value), + PlainTextSplitter.getInstance()); + } } else if (text.startsWith("u") || text.startsWith("U") || text.startsWith("r") || text.startsWith("R") || text.startsWith("b") || text.startsWith("B")) { @@ -57,40 +59,6 @@ public class PythonSpellcheckerStrategy extends SpellcheckingStrategy { consumer.consumeToken(element, splitter); } } - - private static class MyTextRangeConsumer implements PyStringLiteralExpressionImpl.TextRangeConsumer { - private final StringBuilder myCurrentToken = new StringBuilder(); - private final PyStringLiteralExpression myElement; - private final TokenConsumer myTokenConsumer; - private int myTokenStart; - - public MyTextRangeConsumer(PyStringLiteralExpression element, TokenConsumer tokenConsumer) { - myElement = element; - myTokenConsumer = tokenConsumer; - } - - @Override - public boolean process(int startOffset, int endOffset, String value) { - if (endOffset == startOffset + 1) { - if (myCurrentToken.length() == 0) { - myTokenStart = startOffset; - } - myCurrentToken.append(value); - } - else { - if (myCurrentToken.length() > 0) { - processCurrentToken(); - myCurrentToken.setLength(0); - } - } - return true; - } - - private void processCurrentToken() { - String token = myCurrentToken.toString(); - myTokenConsumer.consumeToken(myElement, token, false, myTokenStart, TextRange.allOf(token), PlainTextSplitter.getInstance()); - } - } } private static class FormatStringTokenizer extends Tokenizer { diff --git a/python/testData/inspections/spelling/ignoreEscapeSequence.py b/python/testData/inspections/spelling/ignoreEscapeSequence.py index 3bd99651707d..95e8df407454 100644 --- a/python/testData/inspections/spelling/ignoreEscapeSequence.py +++ b/python/testData/inspections/spelling/ignoreEscapeSequence.py @@ -1 +1,2 @@ -print "foo\nsomething" \ No newline at end of file +print "foo\nsomething" +print """foo\nbrbrbr""" diff --git a/python/testSrc/com/jetbrains/python/PyStringLiteralTest.java b/python/testSrc/com/jetbrains/python/PyStringLiteralTest.java index 7f5b08ae2533..393358443576 100644 --- a/python/testSrc/com/jetbrains/python/PyStringLiteralTest.java +++ b/python/testSrc/com/jetbrains/python/PyStringLiteralTest.java @@ -15,6 +15,7 @@ */ package com.jetbrains.python; +import com.intellij.openapi.util.Pair; import com.intellij.openapi.util.TextRange; import com.intellij.psi.LiteralTextEscaper; import com.intellij.psi.PsiFile; @@ -64,19 +65,54 @@ public class PyStringLiteralTest extends PyTestCase { assertEquals(-1, escaper.getOffsetInHost(4, fooOnly)); } - public void testIterateCharacterRanges() { - final PyStringLiteralExpression expr = createLiteralFromText("'\\nfoo' 'bar'"); + public void testEscaperOffsetInSingleCharString() { + final PyStringLiteralExpression expr = createLiteralFromText("'c'"); assertNotNull(expr); - final List characters = new ArrayList(); - expr.iterateCharacterRanges(new PyStringLiteralExpression.TextRangeConsumer() { - @Override - public boolean process(int startOffset, int endOffset, String value) { - characters.add(value); - return true; - } - }); - final List expected = Arrays.asList("\n", "f", "o", "o", "b", "a", "r"); - assertSameElements(characters, expected); + final LiteralTextEscaper escaper = expr.createLiteralTextEscaper(); + final TextRange range = TextRange.create(1, 2); + assertEquals(1, escaper.getOffsetInHost(0, range)); + assertEquals(2, escaper.getOffsetInHost(1, range)); + assertEquals(-1, escaper.getOffsetInHost(2, range)); + } + + public void testEscaperOffsetInSingleEscapedCharString() { + final PyStringLiteralExpression expr = createLiteralFromText("'\\n'"); + assertNotNull(expr); + final LiteralTextEscaper escaper = expr.createLiteralTextEscaper(); + final TextRange range = TextRange.create(1, 3); + assertEquals(1, escaper.getOffsetInHost(0, range)); + assertEquals(3, escaper.getOffsetInHost(1, range)); + assertEquals(-1, escaper.getOffsetInHost(2, range)); + } + + public void testIterateCharacterRanges() { + assertSameElements(getCharacterRanges("'\\nfoo' 'bar'"), + Arrays.asList("\n", "foo", "bar")); + } + + public void testIterateEscapedBackslash() { + assertSameElements(getCharacterRanges("'''\n" + + "foo.\\\\\n" + + "bar\n" + + "'''\n"), + Arrays.asList("\nfoo.", "\\", "\nbar\n")); + } + + public void testEscaperOffsetInEscapedBackslash() { + final PyStringLiteralExpression expr = createLiteralFromText("'XXX foo.\\\\bar YYY'"); + assertNotNull(expr); + final LiteralTextEscaper escaper = expr.createLiteralTextEscaper(); + final TextRange range = TextRange.create(5, 14); + assertEquals(5, escaper.getOffsetInHost(0, range)); + assertEquals(6, escaper.getOffsetInHost(1, range)); + assertEquals(7, escaper.getOffsetInHost(2, range)); + assertEquals(8, escaper.getOffsetInHost(3, range)); + assertEquals(9, escaper.getOffsetInHost(4, range)); + assertEquals(11, escaper.getOffsetInHost(5, range)); + assertEquals(12, escaper.getOffsetInHost(6, range)); + assertEquals(13, escaper.getOffsetInHost(7, range)); + assertEquals(14, escaper.getOffsetInHost(8, range)); + assertEquals(-1, escaper.getOffsetInHost(9, range)); } private static String decodeRange(PyStringLiteralExpression expr, TextRange range) { @@ -100,4 +136,14 @@ public class PyStringLiteralTest extends PyTestCase { assertEquals("b\\n", createLiteralFromText("ur'\\u0062\\n'").getStringValue()); assertEquals("\\8", createLiteralFromText("'\\8'").getStringValue()); } + + private List getCharacterRanges(String text) { + final PyStringLiteralExpression expr = createLiteralFromText(text); + assertNotNull(expr); + final List characters = new ArrayList(); + for (Pair fragment : expr.getDecodedFragments()) { + characters.add(fragment.getSecond()); + } + return characters; + } }