diff --git a/python/IntelliLang-python/src/pyInjections.xml b/python/IntelliLang-python/src/pyInjections.xml
index a75015e02441..f561c1d33016 100644
--- a/python/IntelliLang-python/src/pyInjections.xml
+++ b/python/IntelliLang-python/src/pyInjections.xml
@@ -9,6 +9,6 @@
"SQL select/delete/insert/update/create"
-
+
diff --git a/python/openapi/src/com/jetbrains/python/psi/PyStringLiteralFileReferenceSet.java b/python/openapi/src/com/jetbrains/python/psi/PyStringLiteralFileReferenceSet.java
index e74cd80a3260..6c470c739c36 100644
--- a/python/openapi/src/com/jetbrains/python/psi/PyStringLiteralFileReferenceSet.java
+++ b/python/openapi/src/com/jetbrains/python/psi/PyStringLiteralFileReferenceSet.java
@@ -20,17 +20,19 @@ import com.intellij.openapi.util.SystemInfo;
import com.intellij.openapi.util.TextRange;
import com.intellij.psi.PsiReferenceProvider;
import com.intellij.psi.impl.source.resolve.reference.impl.providers.FileReference;
-import com.intellij.psi.impl.source.resolve.reference.impl.providers.FileReferenceSet;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.util.ArrayList;
import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
/**
* @author traff
*/
public class PyStringLiteralFileReferenceSet extends RootFileReferenceSet {
+ public static final Pattern DELIMITERS = Pattern.compile("\\\\|/");
private final PyStringLiteralExpression myStringLiteralExpression;
@@ -57,61 +59,33 @@ public class PyStringLiteralFileReferenceSet extends RootFileReferenceSet {
protected void reparse() {
//noinspection ConstantConditions
if (myStringLiteralExpression != null) {
- MyTextRangeConsumer textRangeConsumer = new MyTextRangeConsumer(this);
-
- myStringLiteralExpression.iterateCharacterRanges(textRangeConsumer);
- textRangeConsumer.finish();
-
- List referencesList = textRangeConsumer.myReferenceList;
-
- myReferences = referencesList.toArray(new FileReference[referencesList.size()]);
+ final List references = getFileReferences(myStringLiteralExpression);
+ myReferences = references.toArray(new FileReference[references.size()]);
}
}
- private static class MyTextRangeConsumer implements PyStringLiteralExpression.TextRangeConsumer {
- private final StringBuilder myItem = new StringBuilder();
- private int myStartOffset = -1;
- private int myIndex = 0;
- private int myEndOffset = -1;
- private final FileReferenceSet myFileReferenceSet;
-
-
- private final List myReferenceList = new ArrayList();
-
- private MyTextRangeConsumer(FileReferenceSet set) {
- myFileReferenceSet = set;
- }
-
- @Override
- public boolean process(int startOffset, int endOffset, String value) {
- if ("\\".equals(value) || "/".equals(value)) {
- addReference(startOffset);
+ @NotNull
+ private List getFileReferences(@NotNull PyStringLiteralExpression expression) {
+ final String value = expression.getStringValue();
+ final Matcher matcher = DELIMITERS.matcher(value);
+ int start = 0;
+ int index = 0;
+ final List results = new ArrayList();
+ while (matcher.find()) {
+ final String s = value.substring(start, matcher.start());
+ if (!s.isEmpty()) {
+ final TextRange range = TextRange.create(expression.valueOffsetToTextOffset(start),
+ expression.valueOffsetToTextOffset(matcher.start()));
+ results.add(createFileReference(range, index++, s));
}
- else {
- if (myStartOffset == -1) {
- myStartOffset = startOffset;
- }
- myEndOffset = endOffset;
- myItem.append(value);
- }
- return true;
+ start = matcher.end();
}
-
- private void addReference(int startOffset) {
- if (myStartOffset != -1) {
- final FileReference ref = myFileReferenceSet.createFileReference(
- new TextRange(myStartOffset, startOffset),
- myIndex++,
- myItem.toString());
- myReferenceList.add(ref);
- myStartOffset = -1;
- myItem.setLength(0);
- }
- }
-
-
- public void finish() {
- addReference(myEndOffset);
+ final String s = value.substring(start);
+ if (!s.isEmpty()) {
+ final TextRange range = TextRange.create(expression.valueOffsetToTextOffset(start),
+ expression.valueOffsetToTextOffset(value.length()));
+ results.add(createFileReference(range, index, s));
}
+ return results;
}
}
diff --git a/python/psi-api/src/com/jetbrains/python/psi/PyStringLiteralExpression.java b/python/psi-api/src/com/jetbrains/python/psi/PyStringLiteralExpression.java
index 8c3182db377f..f7dd122680df 100644
--- a/python/psi-api/src/com/jetbrains/python/psi/PyStringLiteralExpression.java
+++ b/python/psi-api/src/com/jetbrains/python/psi/PyStringLiteralExpression.java
@@ -16,8 +16,10 @@
package com.jetbrains.python.psi;
import com.intellij.lang.ASTNode;
+import com.intellij.openapi.util.Pair;
import com.intellij.openapi.util.TextRange;
import com.intellij.psi.PsiLanguageInjectionHost;
+import org.jetbrains.annotations.NotNull;
import java.util.List;
@@ -28,20 +30,6 @@ public interface PyStringLiteralExpression extends PyLiteralExpression, StringLi
int valueOffsetToTextOffset(int valueOffset);
- void iterateCharacterRanges(TextRangeConsumer consumer);
-
- /**
- * Iterator over decoded string characters.
- */
- interface TextRangeConsumer {
- /**
- * Process a decoded character.
- *
- * @param startOffset start offset in the un-decoded string
- * @param endOffset end offset in the un-decoded string
- * @param value decoded character value
- * @return false in order to stop iteration
- */
- boolean process(int startOffset, int endOffset, String value);
- }
+ @NotNull
+ List> getDecodedFragments();
}
diff --git a/python/src/com/jetbrains/python/codeInsight/PyInjectionUtil.java b/python/src/com/jetbrains/python/codeInsight/PyInjectionUtil.java
index 7ce44b5531c9..9a7edad88cc5 100644
--- a/python/src/com/jetbrains/python/codeInsight/PyInjectionUtil.java
+++ b/python/src/com/jetbrains/python/codeInsight/PyInjectionUtil.java
@@ -44,7 +44,7 @@ public class PyInjectionUtil {
@Nullable
public static PsiElement getLargestStringLiteral(@NotNull PsiElement context) {
PsiElement element = null;
- for (PsiElement current = context; current != null && isStringLiteralPart(current); current = current.getParent()) {
+ for (PsiElement current = context; current != null && isStringLiteralPart(current, element); current = current.getParent()) {
element = current;
}
return element;
@@ -58,24 +58,32 @@ public class PyInjectionUtil {
processStringLiteral(element, registrar, "", "", Formatting.PERCENT);
}
- private static boolean isStringLiteralPart(@NotNull PsiElement element) {
- if (element instanceof PyStringLiteralExpression) {
+ private static boolean isStringLiteralPart(@NotNull PsiElement element, @Nullable PsiElement context) {
+ if (element == context) {
+ return true;
+ }
+ else if (element instanceof PyStringLiteralExpression) {
return true;
}
else if (element instanceof PyParenthesizedExpression) {
final PyExpression contained = ((PyParenthesizedExpression)element).getContainedExpression();
- return contained != null && isStringLiteralPart(contained);
+ return contained != null && isStringLiteralPart(contained, context);
}
else if (element instanceof PyBinaryExpression) {
final PyBinaryExpression expr = (PyBinaryExpression)element;
final PyExpression left = expr.getLeftExpression();
final PyExpression right = expr.getRightExpression();
- return (expr.isOperator("+") && (isStringLiteralPart(left) || right != null && isStringLiteralPart(right))) ||
- expr.isOperator("%") && isStringLiteralPart(left);
+ if (expr.isOperator("+")) {
+ return isStringLiteralPart(left, context) || right != null && isStringLiteralPart(right, context);
+ }
+ else if (expr.isOperator("%")) {
+ return right != context && isStringLiteralPart(left, context);
+ }
+ return false;
}
else if (element instanceof PyCallExpression) {
final PyExpression qualifier = getFormatCallQualifier((PyCallExpression)element);
- return qualifier != null && isStringLiteralPart(qualifier);
+ return qualifier != null && isStringLiteralPart(qualifier, context);
}
return false;
}
@@ -147,9 +155,9 @@ public class PyInjectionUtil {
final PyBinaryExpression expr = (PyBinaryExpression)element;
final PyExpression left = expr.getLeftExpression();
final PyExpression right = expr.getRightExpression();
- final boolean isLeftString = isStringLiteralPart(left);
+ final boolean isLeftString = isStringLiteralPart(left, null);
if (expr.isOperator("+")) {
- final boolean isRightString = right != null && isStringLiteralPart(right);
+ final boolean isRightString = right != null && isStringLiteralPart(right, null);
if (isLeftString) {
processStringLiteral(left, registrar, prefix, isRightString ? "" : missingValue, formatting);
}
diff --git a/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java b/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java
index 7f03dcdb05a8..7595fde33fe8 100644
--- a/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java
+++ b/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java
@@ -21,7 +21,6 @@ import com.intellij.lang.Language;
import com.intellij.lang.injection.InjectedLanguageManager;
import com.intellij.navigation.ItemPresentation;
import com.intellij.openapi.util.Pair;
-import com.intellij.openapi.util.Ref;
import com.intellij.openapi.util.TextRange;
import com.intellij.psi.*;
import com.intellij.psi.impl.source.resolve.reference.ReferenceProvidersRegistry;
@@ -50,9 +49,21 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt
public static final Pattern PATTERN_ESCAPE = Pattern
.compile("\\\\(\n|\\\\|'|\"|a|b|f|n|r|t|v|([0-7]{1,3})|x([0-9a-fA-F]{1,2})" + "|N(\\{.*?\\})|u([0-9a-fA-F]{4})|U([0-9a-fA-F]{8}))");
// -> 1 -> 2 <--> 3 <- -> 4 <--> 5 <- -> 6 <-<-
+
+ private enum EscapeRegexGroup {
+ WHOLE_MATCH,
+ ESCAPED_SUBSTRING,
+ OCTAL,
+ HEXADECIMAL,
+ UNICODE_NAMED,
+ UNICODE_16BIT,
+ UNICODE_32BIT
+ }
+
private static final Map escapeMap = initializeEscapeMap();
private String stringValue;
private List valueTextRanges;
+ @Nullable private List> myDecodedFragments;
private final DefaultRegExpPropertiesProvider myPropertiesProvider;
private static Map initializeEscapeMap() {
@@ -85,6 +96,7 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt
super.subtreeChanged();
stringValue = null;
valueTextRanges = null;
+ myDecodedFragments = null;
}
public List getStringValueTextRanges() {
@@ -143,20 +155,82 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt
return text.length() > 0 && Character.toUpperCase(text.charAt(0)) == 'C';
}
- public void iterateCharacterRanges(TextRangeConsumer consumer) {
- int elStart = getTextRange().getStartOffset();
- for (ASTNode child : getStringNodes()) {
- final String text = child.getText();
- TextRange textRange = getNodeTextRange(text);
- int offset = child.getTextRange().getStartOffset() - elStart + textRange.getStartOffset();
- String undecoded = textRange.substring(text);
- if (!iterateCharacterRanges(consumer, undecoded, offset, isRaw(text), isUnicode(text))) {
- break;
+ @Override
+ @NotNull
+ public List> getDecodedFragments() {
+ if (myDecodedFragments == null) {
+ final List> result = new ArrayList>();
+ final int elementStart = getTextRange().getStartOffset();
+ for (ASTNode node : getStringNodes()) {
+ final String text = node.getText();
+ final TextRange textRange = getNodeTextRange(text);
+ final int offset = node.getTextRange().getStartOffset() - elementStart + textRange.getStartOffset();
+ final String encoded = textRange.substring(text);
+ result.addAll(getDecodedFragments(encoded, offset, isRaw(text), isUnicode(text)));
}
+ myDecodedFragments = result;
}
+ return myDecodedFragments;
}
+ @NotNull
+ private static List> getDecodedFragments(@NotNull String encoded, int offset, boolean raw, boolean unicode) {
+ final List> result = new ArrayList>();
+ final Matcher escMatcher = PATTERN_ESCAPE.matcher(encoded);
+ int index = 0;
+ while (escMatcher.find(index)) {
+ if (index < escMatcher.start()) {
+ final TextRange range = TextRange.create(index, escMatcher.start());
+ final TextRange offsetRange = range.shiftRight(offset);
+ result.add(Pair.create(offsetRange, range.substring(encoded)));
+ }
+ final String octal = escapeRegexGroup(escMatcher, EscapeRegexGroup.OCTAL);
+ final String hex = escapeRegexGroup(escMatcher, EscapeRegexGroup.HEXADECIMAL);
+ // TODO: Implement unicode character name escapes: EscapeRegexGroup.UNICODE_NAMED
+ final String unicode16 = escapeRegexGroup(escMatcher, EscapeRegexGroup.UNICODE_16BIT);
+ final String unicode32 = escapeRegexGroup(escMatcher, EscapeRegexGroup.UNICODE_32BIT);
+
+ final boolean escapedUnicode = raw && unicode || !raw;
+
+ final String str;
+ if (!raw && octal != null) {
+ str = new String(new char[]{(char)Integer.parseInt(octal, 8)});
+ }
+ else if (!raw && hex != null) {
+ str = new String(new char[]{(char)Integer.parseInt(hex, 16)});
+ }
+ else if (escapedUnicode && unicode16 != null) {
+ str = unicode ? new String(new char[]{(char)Integer.parseInt(unicode16, 16)}) : unicode16;
+ }
+ else if (escapedUnicode && unicode32 != null) {
+ str = unicode ? new String(Character.toChars((int)Long.parseLong(unicode32, 16))) : unicode32;
+ }
+ else if (raw) {
+ str = escapeRegexGroup(escMatcher, EscapeRegexGroup.WHOLE_MATCH);
+ }
+ else {
+ final String toReplace = escapeRegexGroup(escMatcher, EscapeRegexGroup.ESCAPED_SUBSTRING);
+ str = escapeMap.get(toReplace);
+ }
+
+ if (str != null) {
+ final TextRange wholeMatch = TextRange.create(escMatcher.start(), escMatcher.end());
+ result.add(Pair.create(wholeMatch.shiftRight(offset), str));
+ }
+
+ index = escMatcher.end();
+ }
+ final TextRange range = TextRange.create(index, encoded.length());
+ final TextRange offRange = range.shiftRight(offset);
+ result.add(Pair.create(offRange, range.substring(encoded)));
+ return result;
+ }
+
+ @Nullable
+ private static String escapeRegexGroup(@NotNull Matcher matcher, EscapeRegexGroup group) {
+ return matcher.group(group.ordinal());
+ }
public List getStringNodes() {
return Arrays.asList(getNode().getChildren(PyTokenTypes.STRING_NODES));
@@ -165,14 +239,11 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt
public String getStringValue() {
//ASTNode child = getNode().getFirstChildNode();
//assert child != null;
- if (stringValue == null) {
+ if (stringValue == null) {
final StringBuilder out = new StringBuilder();
- iterateCharacterRanges(new TextRangeConsumer() {
- public boolean process(int startOffset, int endOffset, String value) {
- out.append(value);
- return true;
- }
- });
+ for (Pair fragment : getDecodedFragments()) {
+ out.append(fragment.getSecond());
+ }
stringValue = out.toString();
}
return stringValue;
@@ -190,106 +261,6 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt
return new TextRange(0, getTextLength());
}
- private static boolean iterateCharacterRanges(TextRangeConsumer consumer, String undecoded, int off, boolean raw, boolean unicode) {
- if (raw) {
- return iterateRawCharacterRanges(consumer, undecoded, off, unicode);
- }
- Matcher escMatcher = PATTERN_ESCAPE.matcher(undecoded);
- int index = 0;
- while (escMatcher.find(index)) {
- for (int i = index; i < escMatcher.start(); i++) {
- if (!consumer.process(off + i, off + i + 1, Character.toString(undecoded.charAt(i)))) {
- return false;
- }
- }
- String octal = escMatcher.group(2);
- String hex = escMatcher.group(3);
- String str = null;
- if (octal != null) {
- str = new String(new char[]{(char)Integer.parseInt(octal, 8)});
-
- }
- else if (hex != null) {
- str = new String(new char[]{(char)Integer.parseInt(hex, 16)});
-
- }
- else {
- String toReplace = escMatcher.group(1);
- String replacement = escapeMap.get(toReplace);
- if (replacement != null) {
- str = replacement;
- }
- }
- String unicodeName = escMatcher.group(4);
- String unicode32 = escMatcher.group(6);
-
- if (unicode32 != null) {
- str = unicode ? new String(Character.toChars((int)Long.parseLong(unicode32, 16))) : unicode32;
- }
- if (unicodeName != null) {
- //TOLATER: implement unicode character name escapes
- }
- String unicode16 = escMatcher.group(5);
- if (unicode16 != null) {
- str = unicode ? new String(new char[]{(char)Integer.parseInt(unicode16, 16)}) : unicode16;
- }
-
- if (str != null) {
- int start = escMatcher.start();
- int end = escMatcher.end();
- if (!consumer.process(off + start, off + end, str)) {
- return false;
- }
- }
- index = escMatcher.end();
- }
- for (int i = index; i < undecoded.length(); i++) {
- if (!consumer.process(off + i, off + i + 1, Character.toString(undecoded.charAt(i)))) {
- return false;
- }
- }
- return true;
- }
-
- private static boolean iterateRawCharacterRanges(TextRangeConsumer consumer, String undecoded, int off, boolean unicode) {
- for (int i = 0; i < undecoded.length(); i++) {
- char c = undecoded.charAt(i);
- if (unicode && c == '\\' && i < undecoded.length()-1) {
- char c2 = undecoded.charAt(i+1);
- if (c2 == 'u' && i < undecoded.length()-5) {
- try {
- char u = (char) Integer.parseInt(undecoded.substring(i+2, i+6), 16);
- if (!consumer.process(off+i, off+i+ 6, Character.toString(u))) {
- return false;
- }
- }
- catch (NumberFormatException ignore) { }
- //noinspection AssignmentToForLoopParameter
- i += 5;
- continue;
- }
- if (c2 == 'U' && i < undecoded.length()-9) {
- // note: Java has 16-bit chars, so this code will truncate characters which don't fit in 16 bits
- try {
- char u = (char) Long.parseLong(undecoded.substring(i+2, i+10), 16);
- if (!consumer.process(off+i, off+i+10, Character.toString(u))) {
- return false;
- }
- }
- catch (NumberFormatException ignore) { }
- //noinspection AssignmentToForLoopParameter
- i += 9;
- continue;
- }
- }
- if (!consumer.process(off + i, off + i + 1, Character.toString(c))) {
- return false;
- }
- }
-
- return true;
- }
-
@Override
public String toString() {
return super.toString() + ": " + getStringValue();
@@ -368,57 +339,60 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt
@Override
public boolean decode(@NotNull final TextRange rangeInsideHost, @NotNull final StringBuilder outChars) {
- final PyDocStringOwner
- docStringOwner = PsiTreeUtil.getParentOfType(myHost, PyDocStringOwner.class);
- if (docStringOwner != null && myHost.equals(docStringOwner.getDocStringExpression())) {
- outChars.append(myHost.getText(), rangeInsideHost.getStartOffset(), rangeInsideHost.getEndOffset());
- }
- else {
- myHost.iterateCharacterRanges(new TextRangeConsumer() {
- public boolean process(int startOffset, int endOffset, String value) {
- int xsectStart = Math.max(startOffset, rangeInsideHost.getStartOffset());
- int xsectEnd = Math.min(endOffset, rangeInsideHost.getEndOffset());
- if (xsectEnd > xsectStart) {
- outChars.append(value);
- }
- return endOffset < rangeInsideHost.getEndOffset();
+ for (Pair fragment : myHost.getDecodedFragments()) {
+ final TextRange encodedTextRange = fragment.getFirst();
+ final TextRange intersection = encodedTextRange.intersection(rangeInsideHost);
+ if (intersection != null && !intersection.isEmpty()) {
+ final String value = fragment.getSecond();
+ final String intersectedValue;
+ if (value.length() == 1 || value.length() == intersection.getLength()) {
+ intersectedValue = value;
}
- });
+ else {
+ final int start = Math.max(0, rangeInsideHost.getStartOffset() - encodedTextRange.getStartOffset());
+ final int end = Math.min(value.length(), start + intersection.getLength());
+ intersectedValue = value.substring(start, end);
+ }
+ outChars.append(intersectedValue);
+ }
}
return true;
}
@Override
public int getOffsetInHost(final int offsetInDecoded, @NotNull final TextRange rangeInsideHost) {
- final Ref resultRef = Ref.create(-1);
- final Ref indexRef = Ref.create(0);
- final Ref lastEndOffsetRef = Ref.create(-1);
- myHost.iterateCharacterRanges(new TextRangeConsumer() {
- @Override
- public boolean process(int startOffset, int endOffset, String value) {
- if (startOffset > rangeInsideHost.getEndOffset()) {
- return false;
+ int offset = 0;
+ int endOffset = -1;
+ for (Pair fragment : myHost.getDecodedFragments()) {
+ final TextRange encodedTextRange = fragment.getFirst();
+ final TextRange intersection = encodedTextRange.intersection(rangeInsideHost);
+ if (intersection != null && !intersection.isEmpty()) {
+ final String value = fragment.getSecond();
+ final int valueLength = value.length();
+ final int intersectionLength = intersection.getLength();
+ if (valueLength == 0) {
+ return -1;
}
- lastEndOffsetRef.set(endOffset);
- if (startOffset >= rangeInsideHost.getStartOffset()) {
- final int i = indexRef.get();
- if (i == offsetInDecoded) {
- resultRef.set(startOffset);
- return false;
+ else if (valueLength == 1) {
+ if (offset == offsetInDecoded) {
+ return intersection.getStartOffset();
}
- indexRef.set(i + 1);
+ offset++;
}
- return true;
+ else {
+ if (offset + intersectionLength >= offsetInDecoded) {
+ final int delta = offsetInDecoded - offset;
+ return intersection.getStartOffset() + delta;
+ }
+ offset += intersectionLength;
+ }
+ endOffset = intersection.getEndOffset();
}
- });
- final int result = resultRef.get();
- if (result != -1) {
- return result;
}
- // We should handle the position of a character at the end of rangeInsideHost, because LeafPatcher expects it to be valid
- final int lastEndOffset = lastEndOffsetRef.get();
- if (indexRef.get() == offsetInDecoded && lastEndOffset == rangeInsideHost.getEndOffset()) {
- return lastEndOffset;
+ // XXX: According to the real use of getOffsetInHost() it should return the correct host offset for the offset in decoded at the
+ // end of the range inside host, not -1
+ if (offset == offsetInDecoded) {
+ return endOffset;
}
return -1;
}
@@ -431,23 +405,7 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt
@Override
public int valueOffsetToTextOffset(int valueOffset) {
- final Ref offsetInDecodedRef = new Ref(valueOffset);
- final Ref result = new Ref(-1);
- iterateCharacterRanges(new TextRangeConsumer() {
- public boolean process(int startOffset, int endOffset, String value) {
- if (value.length() > offsetInDecodedRef.get()) {
- result.set(startOffset + offsetInDecodedRef.get());
- return false;
- }
- offsetInDecodedRef.set(offsetInDecodedRef.get() - value.length());
- if (offsetInDecodedRef.get() == 0) {
- result.set(endOffset);
- return false;
- }
- return true;
- }
- });
- return result.get();
+ return createLiteralTextEscaper().getOffsetInHost(valueOffset, getStringValueTextRange());
}
public boolean characterNeedsEscaping(char c) {
diff --git a/python/src/com/jetbrains/python/spellchecker/PythonSpellcheckerStrategy.java b/python/src/com/jetbrains/python/spellchecker/PythonSpellcheckerStrategy.java
index 4a0a59d15c1f..b24e53c557ad 100644
--- a/python/src/com/jetbrains/python/spellchecker/PythonSpellcheckerStrategy.java
+++ b/python/src/com/jetbrains/python/spellchecker/PythonSpellcheckerStrategy.java
@@ -15,6 +15,7 @@
*/
package com.jetbrains.python.spellchecker;
+import com.intellij.openapi.util.Pair;
import com.intellij.openapi.util.TextRange;
import com.intellij.psi.PsiElement;
import com.intellij.spellchecker.inspections.PlainTextSplitter;
@@ -26,7 +27,6 @@ import com.jetbrains.python.PyTokenTypes;
import com.jetbrains.python.inspections.PyStringFormatParser;
import com.jetbrains.python.psi.PyBinaryExpression;
import com.jetbrains.python.psi.PyStringLiteralExpression;
-import com.jetbrains.python.psi.impl.PyStringLiteralExpressionImpl;
import org.jetbrains.annotations.NotNull;
import java.util.List;
@@ -41,9 +41,11 @@ public class PythonSpellcheckerStrategy extends SpellcheckingStrategy {
Splitter splitter = PlainTextSplitter.getInstance();
String text = element.getText();
if (text.indexOf('\\') >= 0) {
- MyTextRangeConsumer textRangeConsumer = new MyTextRangeConsumer(element, consumer);
- ((PyStringLiteralExpressionImpl) element).iterateCharacterRanges(textRangeConsumer);
- textRangeConsumer.processCurrentToken(); // process last token
+ for (Pair fragment : element.getDecodedFragments()) {
+ final String value = fragment.getSecond();
+ consumer.consumeToken(element, value, false, fragment.getFirst().getStartOffset(), TextRange.allOf(value),
+ PlainTextSplitter.getInstance());
+ }
}
else if (text.startsWith("u") || text.startsWith("U") || text.startsWith("r") || text.startsWith("R") ||
text.startsWith("b") || text.startsWith("B")) {
@@ -57,40 +59,6 @@ public class PythonSpellcheckerStrategy extends SpellcheckingStrategy {
consumer.consumeToken(element, splitter);
}
}
-
- private static class MyTextRangeConsumer implements PyStringLiteralExpressionImpl.TextRangeConsumer {
- private final StringBuilder myCurrentToken = new StringBuilder();
- private final PyStringLiteralExpression myElement;
- private final TokenConsumer myTokenConsumer;
- private int myTokenStart;
-
- public MyTextRangeConsumer(PyStringLiteralExpression element, TokenConsumer tokenConsumer) {
- myElement = element;
- myTokenConsumer = tokenConsumer;
- }
-
- @Override
- public boolean process(int startOffset, int endOffset, String value) {
- if (endOffset == startOffset + 1) {
- if (myCurrentToken.length() == 0) {
- myTokenStart = startOffset;
- }
- myCurrentToken.append(value);
- }
- else {
- if (myCurrentToken.length() > 0) {
- processCurrentToken();
- myCurrentToken.setLength(0);
- }
- }
- return true;
- }
-
- private void processCurrentToken() {
- String token = myCurrentToken.toString();
- myTokenConsumer.consumeToken(myElement, token, false, myTokenStart, TextRange.allOf(token), PlainTextSplitter.getInstance());
- }
- }
}
private static class FormatStringTokenizer extends Tokenizer {
diff --git a/python/testData/inspections/spelling/ignoreEscapeSequence.py b/python/testData/inspections/spelling/ignoreEscapeSequence.py
index 3bd99651707d..95e8df407454 100644
--- a/python/testData/inspections/spelling/ignoreEscapeSequence.py
+++ b/python/testData/inspections/spelling/ignoreEscapeSequence.py
@@ -1 +1,2 @@
-print "foo\nsomething"
\ No newline at end of file
+print "foo\nsomething"
+print """foo\nbrbrbr"""
diff --git a/python/testSrc/com/jetbrains/python/PyStringLiteralTest.java b/python/testSrc/com/jetbrains/python/PyStringLiteralTest.java
index 7f5b08ae2533..393358443576 100644
--- a/python/testSrc/com/jetbrains/python/PyStringLiteralTest.java
+++ b/python/testSrc/com/jetbrains/python/PyStringLiteralTest.java
@@ -15,6 +15,7 @@
*/
package com.jetbrains.python;
+import com.intellij.openapi.util.Pair;
import com.intellij.openapi.util.TextRange;
import com.intellij.psi.LiteralTextEscaper;
import com.intellij.psi.PsiFile;
@@ -64,19 +65,54 @@ public class PyStringLiteralTest extends PyTestCase {
assertEquals(-1, escaper.getOffsetInHost(4, fooOnly));
}
- public void testIterateCharacterRanges() {
- final PyStringLiteralExpression expr = createLiteralFromText("'\\nfoo' 'bar'");
+ public void testEscaperOffsetInSingleCharString() {
+ final PyStringLiteralExpression expr = createLiteralFromText("'c'");
assertNotNull(expr);
- final List characters = new ArrayList();
- expr.iterateCharacterRanges(new PyStringLiteralExpression.TextRangeConsumer() {
- @Override
- public boolean process(int startOffset, int endOffset, String value) {
- characters.add(value);
- return true;
- }
- });
- final List expected = Arrays.asList("\n", "f", "o", "o", "b", "a", "r");
- assertSameElements(characters, expected);
+ final LiteralTextEscaper extends PsiLanguageInjectionHost> escaper = expr.createLiteralTextEscaper();
+ final TextRange range = TextRange.create(1, 2);
+ assertEquals(1, escaper.getOffsetInHost(0, range));
+ assertEquals(2, escaper.getOffsetInHost(1, range));
+ assertEquals(-1, escaper.getOffsetInHost(2, range));
+ }
+
+ public void testEscaperOffsetInSingleEscapedCharString() {
+ final PyStringLiteralExpression expr = createLiteralFromText("'\\n'");
+ assertNotNull(expr);
+ final LiteralTextEscaper extends PsiLanguageInjectionHost> escaper = expr.createLiteralTextEscaper();
+ final TextRange range = TextRange.create(1, 3);
+ assertEquals(1, escaper.getOffsetInHost(0, range));
+ assertEquals(3, escaper.getOffsetInHost(1, range));
+ assertEquals(-1, escaper.getOffsetInHost(2, range));
+ }
+
+ public void testIterateCharacterRanges() {
+ assertSameElements(getCharacterRanges("'\\nfoo' 'bar'"),
+ Arrays.asList("\n", "foo", "bar"));
+ }
+
+ public void testIterateEscapedBackslash() {
+ assertSameElements(getCharacterRanges("'''\n" +
+ "foo.\\\\\n" +
+ "bar\n" +
+ "'''\n"),
+ Arrays.asList("\nfoo.", "\\", "\nbar\n"));
+ }
+
+ public void testEscaperOffsetInEscapedBackslash() {
+ final PyStringLiteralExpression expr = createLiteralFromText("'XXX foo.\\\\bar YYY'");
+ assertNotNull(expr);
+ final LiteralTextEscaper extends PsiLanguageInjectionHost> escaper = expr.createLiteralTextEscaper();
+ final TextRange range = TextRange.create(5, 14);
+ assertEquals(5, escaper.getOffsetInHost(0, range));
+ assertEquals(6, escaper.getOffsetInHost(1, range));
+ assertEquals(7, escaper.getOffsetInHost(2, range));
+ assertEquals(8, escaper.getOffsetInHost(3, range));
+ assertEquals(9, escaper.getOffsetInHost(4, range));
+ assertEquals(11, escaper.getOffsetInHost(5, range));
+ assertEquals(12, escaper.getOffsetInHost(6, range));
+ assertEquals(13, escaper.getOffsetInHost(7, range));
+ assertEquals(14, escaper.getOffsetInHost(8, range));
+ assertEquals(-1, escaper.getOffsetInHost(9, range));
}
private static String decodeRange(PyStringLiteralExpression expr, TextRange range) {
@@ -100,4 +136,14 @@ public class PyStringLiteralTest extends PyTestCase {
assertEquals("b\\n", createLiteralFromText("ur'\\u0062\\n'").getStringValue());
assertEquals("\\8", createLiteralFromText("'\\8'").getStringValue());
}
+
+ private List getCharacterRanges(String text) {
+ final PyStringLiteralExpression expr = createLiteralFromText(text);
+ assertNotNull(expr);
+ final List characters = new ArrayList();
+ for (Pair fragment : expr.getDecodedFragments()) {
+ characters.add(fragment.getSecond());
+ }
+ return characters;
+ }
}