From 1fe89e98969439d3030e0e13fc945247771dc823 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Tue, 26 Nov 2013 14:34:03 +0400 Subject: [PATCH 01/10] Fixed detection of the largest string literal for right parts of '%' operator --- .../python/codeInsight/PyInjectionUtil.java | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/python/src/com/jetbrains/python/codeInsight/PyInjectionUtil.java b/python/src/com/jetbrains/python/codeInsight/PyInjectionUtil.java index 7ce44b5531c9..9a7edad88cc5 100644 --- a/python/src/com/jetbrains/python/codeInsight/PyInjectionUtil.java +++ b/python/src/com/jetbrains/python/codeInsight/PyInjectionUtil.java @@ -44,7 +44,7 @@ public class PyInjectionUtil { @Nullable public static PsiElement getLargestStringLiteral(@NotNull PsiElement context) { PsiElement element = null; - for (PsiElement current = context; current != null && isStringLiteralPart(current); current = current.getParent()) { + for (PsiElement current = context; current != null && isStringLiteralPart(current, element); current = current.getParent()) { element = current; } return element; @@ -58,24 +58,32 @@ public class PyInjectionUtil { processStringLiteral(element, registrar, "", "", Formatting.PERCENT); } - private static boolean isStringLiteralPart(@NotNull PsiElement element) { - if (element instanceof PyStringLiteralExpression) { + private static boolean isStringLiteralPart(@NotNull PsiElement element, @Nullable PsiElement context) { + if (element == context) { + return true; + } + else if (element instanceof PyStringLiteralExpression) { return true; } else if (element instanceof PyParenthesizedExpression) { final PyExpression contained = ((PyParenthesizedExpression)element).getContainedExpression(); - return contained != null && isStringLiteralPart(contained); + return contained != null && isStringLiteralPart(contained, context); } else if (element instanceof PyBinaryExpression) { final PyBinaryExpression expr = (PyBinaryExpression)element; final PyExpression left = expr.getLeftExpression(); final PyExpression right = expr.getRightExpression(); - return (expr.isOperator("+") && (isStringLiteralPart(left) || right != null && isStringLiteralPart(right))) || - expr.isOperator("%") && isStringLiteralPart(left); + if (expr.isOperator("+")) { + return isStringLiteralPart(left, context) || right != null && isStringLiteralPart(right, context); + } + else if (expr.isOperator("%")) { + return right != context && isStringLiteralPart(left, context); + } + return false; } else if (element instanceof PyCallExpression) { final PyExpression qualifier = getFormatCallQualifier((PyCallExpression)element); - return qualifier != null && isStringLiteralPart(qualifier); + return qualifier != null && isStringLiteralPart(qualifier, context); } return false; } @@ -147,9 +155,9 @@ public class PyInjectionUtil { final PyBinaryExpression expr = (PyBinaryExpression)element; final PyExpression left = expr.getLeftExpression(); final PyExpression right = expr.getRightExpression(); - final boolean isLeftString = isStringLiteralPart(left); + final boolean isLeftString = isStringLiteralPart(left, null); if (expr.isOperator("+")) { - final boolean isRightString = right != null && isStringLiteralPart(right); + final boolean isRightString = right != null && isStringLiteralPart(right, null); if (isLeftString) { processStringLiteral(left, registrar, prefix, isRightString ? "" : missingValue, formatting); } From 7e72dbfc8bbe010d862faa559878ec9fa24f8bf3 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Wed, 4 Dec 2013 15:31:14 +0400 Subject: [PATCH 02/10] Added PyStringLiteralExpression.getDecodedFragments() --- .../python/psi/PyStringLiteralExpression.java | 12 + .../impl/PyStringLiteralExpressionImpl.java | 329 +++++++++--------- .../jetbrains/python/PyStringLiteralTest.java | 56 ++- 3 files changed, 211 insertions(+), 186 deletions(-) diff --git a/python/psi-api/src/com/jetbrains/python/psi/PyStringLiteralExpression.java b/python/psi-api/src/com/jetbrains/python/psi/PyStringLiteralExpression.java index 8c3182db377f..d968d8c04ee1 100644 --- a/python/psi-api/src/com/jetbrains/python/psi/PyStringLiteralExpression.java +++ b/python/psi-api/src/com/jetbrains/python/psi/PyStringLiteralExpression.java @@ -18,6 +18,7 @@ package com.jetbrains.python.psi; import com.intellij.lang.ASTNode; import com.intellij.openapi.util.TextRange; import com.intellij.psi.PsiLanguageInjectionHost; +import org.jetbrains.annotations.NotNull; import java.util.List; @@ -28,6 +29,17 @@ public interface PyStringLiteralExpression extends PyLiteralExpression, StringLi int valueOffsetToTextOffset(int valueOffset); + interface DecodedFragment { + @NotNull + TextRange getEncodedTextRange(); + + @NotNull + String getValue(); + } + + @NotNull + List getDecodedFragments(); + void iterateCharacterRanges(TextRangeConsumer consumer); /** diff --git a/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java b/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java index 7f03dcdb05a8..9402b873e5a7 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java @@ -21,7 +21,6 @@ import com.intellij.lang.Language; import com.intellij.lang.injection.InjectedLanguageManager; import com.intellij.navigation.ItemPresentation; import com.intellij.openapi.util.Pair; -import com.intellij.openapi.util.Ref; import com.intellij.openapi.util.TextRange; import com.intellij.psi.*; import com.intellij.psi.impl.source.resolve.reference.ReferenceProvidersRegistry; @@ -50,6 +49,17 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt public static final Pattern PATTERN_ESCAPE = Pattern .compile("\\\\(\n|\\\\|'|\"|a|b|f|n|r|t|v|([0-7]{1,3})|x([0-9a-fA-F]{1,2})" + "|N(\\{.*?\\})|u([0-9a-fA-F]{4})|U([0-9a-fA-F]{8}))"); // -> 1 -> 2 <--> 3 <- -> 4 <--> 5 <- -> 6 <-<- + + private enum EscapeRegexGroup { + WHOLE_MATCH, + ESCAPED_SUBSTRING, + OCTAL, + HEXADECIMAL, + UNICODE_NAMED, + UNICODE_16BIT, + UNICODE_32BIT + } + private static final Map escapeMap = initializeEscapeMap(); private String stringValue; private List valueTextRanges; @@ -143,20 +153,109 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt return text.length() > 0 && Character.toUpperCase(text.charAt(0)) == 'C'; } - public void iterateCharacterRanges(TextRangeConsumer consumer) { - int elStart = getTextRange().getStartOffset(); - for (ASTNode child : getStringNodes()) { - final String text = child.getText(); - TextRange textRange = getNodeTextRange(text); - int offset = child.getTextRange().getStartOffset() - elStart + textRange.getStartOffset(); - String undecoded = textRange.substring(text); - if (!iterateCharacterRanges(consumer, undecoded, offset, isRaw(text), isUnicode(text))) { - break; - } + public static class DecodedFragmentImpl implements DecodedFragment { + @NotNull private final TextRange myRange; + @NotNull private final String myValue; + + public DecodedFragmentImpl(@NotNull TextRange range, @NotNull String value) { + myRange = range; + myValue = value; + } + + @Override + @NotNull + public TextRange getEncodedTextRange() { + return myRange; + } + + @Override + @NotNull + public String getValue() { + return myValue; } } + @Override + public void iterateCharacterRanges(PyStringLiteralExpression.TextRangeConsumer consumer) { + for (PyStringLiteralExpression.DecodedFragment fragment : getDecodedFragments()) { + final TextRange range = fragment.getEncodedTextRange(); + consumer.process(range.getStartOffset(), range.getEndOffset(), fragment.getValue()); + } + } + @Override + @NotNull + public List getDecodedFragments() { + final List result = new ArrayList(); + final int elementStart = getTextRange().getStartOffset(); + for (ASTNode node : getStringNodes()) { + final String text = node.getText(); + final TextRange textRange = getNodeTextRange(text); + final int offset = node.getTextRange().getStartOffset() - elementStart + textRange.getStartOffset(); + final String encoded = textRange.substring(text); + result.addAll(getDecodedFragments(encoded, offset, isRaw(text), isUnicode(text))); + } + return result; + } + + @NotNull + private static List getDecodedFragments(@NotNull String encoded, int offset, boolean raw, boolean unicode) { + final List result = new ArrayList(); + final Matcher escMatcher = PATTERN_ESCAPE.matcher(encoded); + int index = 0; + while (escMatcher.find(index)) { + if (index < escMatcher.start()) { + final TextRange range = TextRange.create(index, escMatcher.start()); + final TextRange offsetRange = range.shiftRight(offset); + result.add(new DecodedFragmentImpl(offsetRange, range.substring(encoded))); + } + + final String octal = escapeRegexGroup(escMatcher, EscapeRegexGroup.OCTAL); + final String hex = escapeRegexGroup(escMatcher, EscapeRegexGroup.HEXADECIMAL); + // TODO: Implement unicode character name escapes: EscapeRegexGroup.UNICODE_NAMED + final String unicode16 = escapeRegexGroup(escMatcher, EscapeRegexGroup.UNICODE_16BIT); + final String unicode32 = escapeRegexGroup(escMatcher, EscapeRegexGroup.UNICODE_32BIT); + + final boolean escapedUnicode = raw && unicode || !raw; + + final String str; + if (!raw && octal != null) { + str = new String(new char[]{(char)Integer.parseInt(octal, 8)}); + } + else if (!raw && hex != null) { + str = new String(new char[]{(char)Integer.parseInt(hex, 16)}); + } + else if (escapedUnicode && unicode16 != null) { + str = unicode ? new String(new char[]{(char)Integer.parseInt(unicode16, 16)}) : unicode16; + } + else if (escapedUnicode && unicode32 != null) { + str = unicode ? new String(Character.toChars((int)Long.parseLong(unicode32, 16))) : unicode32; + } + else if (raw) { + str = escapeRegexGroup(escMatcher, EscapeRegexGroup.WHOLE_MATCH); + } + else { + final String toReplace = escapeRegexGroup(escMatcher, EscapeRegexGroup.ESCAPED_SUBSTRING); + str = escapeMap.get(toReplace); + } + + if (str != null) { + final TextRange wholeMatch = TextRange.create(escMatcher.start(), escMatcher.end()); + result.add(new DecodedFragmentImpl(wholeMatch.shiftRight(offset), str)); + } + + index = escMatcher.end(); + } + final TextRange range = TextRange.create(index, encoded.length()); + final TextRange offRange = range.shiftRight(offset); + result.add(new DecodedFragmentImpl(offRange, range.substring(encoded))); + return result; + } + + @Nullable + private static String escapeRegexGroup(@NotNull Matcher matcher, EscapeRegexGroup group) { + return matcher.group(group.ordinal()); + } public List getStringNodes() { return Arrays.asList(getNode().getChildren(PyTokenTypes.STRING_NODES)); @@ -165,14 +264,11 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt public String getStringValue() { //ASTNode child = getNode().getFirstChildNode(); //assert child != null; - if (stringValue == null) { + if (stringValue == null) { final StringBuilder out = new StringBuilder(); - iterateCharacterRanges(new TextRangeConsumer() { - public boolean process(int startOffset, int endOffset, String value) { - out.append(value); - return true; - } - }); + for (DecodedFragment fragment : getDecodedFragments()) { + out.append(fragment.getValue()); + } stringValue = out.toString(); } return stringValue; @@ -190,106 +286,6 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt return new TextRange(0, getTextLength()); } - private static boolean iterateCharacterRanges(TextRangeConsumer consumer, String undecoded, int off, boolean raw, boolean unicode) { - if (raw) { - return iterateRawCharacterRanges(consumer, undecoded, off, unicode); - } - Matcher escMatcher = PATTERN_ESCAPE.matcher(undecoded); - int index = 0; - while (escMatcher.find(index)) { - for (int i = index; i < escMatcher.start(); i++) { - if (!consumer.process(off + i, off + i + 1, Character.toString(undecoded.charAt(i)))) { - return false; - } - } - String octal = escMatcher.group(2); - String hex = escMatcher.group(3); - String str = null; - if (octal != null) { - str = new String(new char[]{(char)Integer.parseInt(octal, 8)}); - - } - else if (hex != null) { - str = new String(new char[]{(char)Integer.parseInt(hex, 16)}); - - } - else { - String toReplace = escMatcher.group(1); - String replacement = escapeMap.get(toReplace); - if (replacement != null) { - str = replacement; - } - } - String unicodeName = escMatcher.group(4); - String unicode32 = escMatcher.group(6); - - if (unicode32 != null) { - str = unicode ? new String(Character.toChars((int)Long.parseLong(unicode32, 16))) : unicode32; - } - if (unicodeName != null) { - //TOLATER: implement unicode character name escapes - } - String unicode16 = escMatcher.group(5); - if (unicode16 != null) { - str = unicode ? new String(new char[]{(char)Integer.parseInt(unicode16, 16)}) : unicode16; - } - - if (str != null) { - int start = escMatcher.start(); - int end = escMatcher.end(); - if (!consumer.process(off + start, off + end, str)) { - return false; - } - } - index = escMatcher.end(); - } - for (int i = index; i < undecoded.length(); i++) { - if (!consumer.process(off + i, off + i + 1, Character.toString(undecoded.charAt(i)))) { - return false; - } - } - return true; - } - - private static boolean iterateRawCharacterRanges(TextRangeConsumer consumer, String undecoded, int off, boolean unicode) { - for (int i = 0; i < undecoded.length(); i++) { - char c = undecoded.charAt(i); - if (unicode && c == '\\' && i < undecoded.length()-1) { - char c2 = undecoded.charAt(i+1); - if (c2 == 'u' && i < undecoded.length()-5) { - try { - char u = (char) Integer.parseInt(undecoded.substring(i+2, i+6), 16); - if (!consumer.process(off+i, off+i+ 6, Character.toString(u))) { - return false; - } - } - catch (NumberFormatException ignore) { } - //noinspection AssignmentToForLoopParameter - i += 5; - continue; - } - if (c2 == 'U' && i < undecoded.length()-9) { - // note: Java has 16-bit chars, so this code will truncate characters which don't fit in 16 bits - try { - char u = (char) Long.parseLong(undecoded.substring(i+2, i+10), 16); - if (!consumer.process(off+i, off+i+10, Character.toString(u))) { - return false; - } - } - catch (NumberFormatException ignore) { } - //noinspection AssignmentToForLoopParameter - i += 9; - continue; - } - } - if (!consumer.process(off + i, off + i + 1, Character.toString(c))) { - return false; - } - } - - return true; - } - @Override public String toString() { return super.toString() + ": " + getStringValue(); @@ -368,57 +364,58 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt @Override public boolean decode(@NotNull final TextRange rangeInsideHost, @NotNull final StringBuilder outChars) { - final PyDocStringOwner - docStringOwner = PsiTreeUtil.getParentOfType(myHost, PyDocStringOwner.class); - if (docStringOwner != null && myHost.equals(docStringOwner.getDocStringExpression())) { - outChars.append(myHost.getText(), rangeInsideHost.getStartOffset(), rangeInsideHost.getEndOffset()); - } - else { - myHost.iterateCharacterRanges(new TextRangeConsumer() { - public boolean process(int startOffset, int endOffset, String value) { - int xsectStart = Math.max(startOffset, rangeInsideHost.getStartOffset()); - int xsectEnd = Math.min(endOffset, rangeInsideHost.getEndOffset()); - if (xsectEnd > xsectStart) { - outChars.append(value); - } - return endOffset < rangeInsideHost.getEndOffset(); + for (DecodedFragment fragment : myHost.getDecodedFragments()) { + final TextRange encodedTextRange = fragment.getEncodedTextRange(); + final TextRange intersection = encodedTextRange.intersection(rangeInsideHost); + if (intersection != null && !intersection.isEmpty()) { + final String value = fragment.getValue(); + final String intersectedValue; + if (value.length() == 1 || value.length() == intersection.getLength()) { + intersectedValue = value; } - }); + else { + final int start = Math.max(0, rangeInsideHost.getStartOffset() - encodedTextRange.getStartOffset()); + final int end = Math.min(value.length(), start + intersection.getLength()); + intersectedValue = value.substring(start, end); + } + outChars.append(intersectedValue); + } } return true; } @Override public int getOffsetInHost(final int offsetInDecoded, @NotNull final TextRange rangeInsideHost) { - final Ref resultRef = Ref.create(-1); - final Ref indexRef = Ref.create(0); - final Ref lastEndOffsetRef = Ref.create(-1); - myHost.iterateCharacterRanges(new TextRangeConsumer() { - @Override - public boolean process(int startOffset, int endOffset, String value) { - if (startOffset > rangeInsideHost.getEndOffset()) { - return false; + int offset = 0; + int endOffset = -1; + for (DecodedFragment fragment : myHost.getDecodedFragments()) { + final TextRange encodedTextRange = fragment.getEncodedTextRange(); + final TextRange intersection = encodedTextRange.intersection(rangeInsideHost); + if (intersection != null && !intersection.isEmpty()) { + final String value = fragment.getValue(); + final int valueLength = value.length(); + if (valueLength == 0) { + return -1; } - lastEndOffsetRef.set(endOffset); - if (startOffset >= rangeInsideHost.getStartOffset()) { - final int i = indexRef.get(); - if (i == offsetInDecoded) { - resultRef.set(startOffset); - return false; + else if (valueLength == 1) { + if (offset == offsetInDecoded) { + return intersection.getStartOffset(); } - indexRef.set(i + 1); } - return true; + else { + if (offset + valueLength >= offsetInDecoded) { + final int delta = offsetInDecoded - offset; + return intersection.getStartOffset() + delta; + } + } + offset += valueLength; + endOffset = intersection.getEndOffset(); } - }); - final int result = resultRef.get(); - if (result != -1) { - return result; } - // We should handle the position of a character at the end of rangeInsideHost, because LeafPatcher expects it to be valid - final int lastEndOffset = lastEndOffsetRef.get(); - if (indexRef.get() == offsetInDecoded && lastEndOffset == rangeInsideHost.getEndOffset()) { - return lastEndOffset; + // XXX: According to the real use of getOffsetInHost() it should return the correct host offset for the offset in decoded at the + // end of the range inside host, not -1 + if (offset == offsetInDecoded) { + return endOffset; } return -1; } @@ -431,23 +428,7 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt @Override public int valueOffsetToTextOffset(int valueOffset) { - final Ref offsetInDecodedRef = new Ref(valueOffset); - final Ref result = new Ref(-1); - iterateCharacterRanges(new TextRangeConsumer() { - public boolean process(int startOffset, int endOffset, String value) { - if (value.length() > offsetInDecodedRef.get()) { - result.set(startOffset + offsetInDecodedRef.get()); - return false; - } - offsetInDecodedRef.set(offsetInDecodedRef.get() - value.length()); - if (offsetInDecodedRef.get() == 0) { - result.set(endOffset); - return false; - } - return true; - } - }); - return result.get(); + return createLiteralTextEscaper().getOffsetInHost(valueOffset, getStringValueTextRange()); } public boolean characterNeedsEscaping(char c) { diff --git a/python/testSrc/com/jetbrains/python/PyStringLiteralTest.java b/python/testSrc/com/jetbrains/python/PyStringLiteralTest.java index 7f5b08ae2533..016bbe5f8e0b 100644 --- a/python/testSrc/com/jetbrains/python/PyStringLiteralTest.java +++ b/python/testSrc/com/jetbrains/python/PyStringLiteralTest.java @@ -64,19 +64,37 @@ public class PyStringLiteralTest extends PyTestCase { assertEquals(-1, escaper.getOffsetInHost(4, fooOnly)); } - public void testIterateCharacterRanges() { - final PyStringLiteralExpression expr = createLiteralFromText("'\\nfoo' 'bar'"); + public void testEscaperOffsetInSingleCharString() { + final PyStringLiteralExpression expr = createLiteralFromText("'c'"); assertNotNull(expr); - final List characters = new ArrayList(); - expr.iterateCharacterRanges(new PyStringLiteralExpression.TextRangeConsumer() { - @Override - public boolean process(int startOffset, int endOffset, String value) { - characters.add(value); - return true; - } - }); - final List expected = Arrays.asList("\n", "f", "o", "o", "b", "a", "r"); - assertSameElements(characters, expected); + final LiteralTextEscaper escaper = expr.createLiteralTextEscaper(); + final TextRange range = TextRange.create(1, 2); + assertEquals(1, escaper.getOffsetInHost(0, range)); + assertEquals(2, escaper.getOffsetInHost(1, range)); + assertEquals(-1, escaper.getOffsetInHost(2, range)); + } + + public void testEscaperOffsetInSingleEscapedCharString() { + final PyStringLiteralExpression expr = createLiteralFromText("'\\n'"); + assertNotNull(expr); + final LiteralTextEscaper escaper = expr.createLiteralTextEscaper(); + final TextRange range = TextRange.create(1, 3); + assertEquals(1, escaper.getOffsetInHost(0, range)); + assertEquals(3, escaper.getOffsetInHost(1, range)); + assertEquals(-1, escaper.getOffsetInHost(2, range)); + } + + public void testIterateCharacterRanges() { + assertSameElements(getCharacterRanges("'\\nfoo' 'bar'"), + Arrays.asList("\n", "foo", "bar")); + } + + public void testIterateEscapedBackslash() { + assertSameElements(getCharacterRanges("'''\n" + + "foo.\\\\\n" + + "bar\n" + + "'''\n"), + Arrays.asList("\nfoo.", "\\", "\nbar\n")); } private static String decodeRange(PyStringLiteralExpression expr, TextRange range) { @@ -100,4 +118,18 @@ public class PyStringLiteralTest extends PyTestCase { assertEquals("b\\n", createLiteralFromText("ur'\\u0062\\n'").getStringValue()); assertEquals("\\8", createLiteralFromText("'\\8'").getStringValue()); } + + private List getCharacterRanges(String text) { + final PyStringLiteralExpression expr = createLiteralFromText(text); + assertNotNull(expr); + final List characters = new ArrayList(); + expr.iterateCharacterRanges(new PyStringLiteralExpression.TextRangeConsumer() { + @Override + public boolean process(int startOffset, int endOffset, String value) { + characters.add(value); + return true; + } + }); + return characters; + } } From f6f19e8651b48d07d9ef50551ffbc43a0ec4ab58 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Thu, 5 Dec 2013 17:33:02 +0400 Subject: [PATCH 03/10] Switched to new string value iteration in Python spellchecker --- .../impl/PyStringLiteralExpressionImpl.java | 5 +++ .../PythonSpellcheckerStrategy.java | 43 +++---------------- .../spelling/ignoreEscapeSequence.py | 3 +- 3 files changed, 12 insertions(+), 39 deletions(-) diff --git a/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java b/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java index 9402b873e5a7..5bc50dd269b8 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java @@ -173,6 +173,11 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt public String getValue() { return myValue; } + + @Override + public String toString() { + return String.format("DecodedFragmentImpl(%s, \"%s\")", myRange, myValue); + } } @Override diff --git a/python/src/com/jetbrains/python/spellchecker/PythonSpellcheckerStrategy.java b/python/src/com/jetbrains/python/spellchecker/PythonSpellcheckerStrategy.java index 4a0a59d15c1f..5d6033541294 100644 --- a/python/src/com/jetbrains/python/spellchecker/PythonSpellcheckerStrategy.java +++ b/python/src/com/jetbrains/python/spellchecker/PythonSpellcheckerStrategy.java @@ -26,7 +26,6 @@ import com.jetbrains.python.PyTokenTypes; import com.jetbrains.python.inspections.PyStringFormatParser; import com.jetbrains.python.psi.PyBinaryExpression; import com.jetbrains.python.psi.PyStringLiteralExpression; -import com.jetbrains.python.psi.impl.PyStringLiteralExpressionImpl; import org.jetbrains.annotations.NotNull; import java.util.List; @@ -41,9 +40,11 @@ public class PythonSpellcheckerStrategy extends SpellcheckingStrategy { Splitter splitter = PlainTextSplitter.getInstance(); String text = element.getText(); if (text.indexOf('\\') >= 0) { - MyTextRangeConsumer textRangeConsumer = new MyTextRangeConsumer(element, consumer); - ((PyStringLiteralExpressionImpl) element).iterateCharacterRanges(textRangeConsumer); - textRangeConsumer.processCurrentToken(); // process last token + for (PyStringLiteralExpression.DecodedFragment fragment : element.getDecodedFragments()) { + final String value = fragment.getValue(); + consumer.consumeToken(element, value, false, fragment.getEncodedTextRange().getStartOffset(), TextRange.allOf(value), + PlainTextSplitter.getInstance()); + } } else if (text.startsWith("u") || text.startsWith("U") || text.startsWith("r") || text.startsWith("R") || text.startsWith("b") || text.startsWith("B")) { @@ -57,40 +58,6 @@ public class PythonSpellcheckerStrategy extends SpellcheckingStrategy { consumer.consumeToken(element, splitter); } } - - private static class MyTextRangeConsumer implements PyStringLiteralExpressionImpl.TextRangeConsumer { - private final StringBuilder myCurrentToken = new StringBuilder(); - private final PyStringLiteralExpression myElement; - private final TokenConsumer myTokenConsumer; - private int myTokenStart; - - public MyTextRangeConsumer(PyStringLiteralExpression element, TokenConsumer tokenConsumer) { - myElement = element; - myTokenConsumer = tokenConsumer; - } - - @Override - public boolean process(int startOffset, int endOffset, String value) { - if (endOffset == startOffset + 1) { - if (myCurrentToken.length() == 0) { - myTokenStart = startOffset; - } - myCurrentToken.append(value); - } - else { - if (myCurrentToken.length() > 0) { - processCurrentToken(); - myCurrentToken.setLength(0); - } - } - return true; - } - - private void processCurrentToken() { - String token = myCurrentToken.toString(); - myTokenConsumer.consumeToken(myElement, token, false, myTokenStart, TextRange.allOf(token), PlainTextSplitter.getInstance()); - } - } } private static class FormatStringTokenizer extends Tokenizer { diff --git a/python/testData/inspections/spelling/ignoreEscapeSequence.py b/python/testData/inspections/spelling/ignoreEscapeSequence.py index 3bd99651707d..95e8df407454 100644 --- a/python/testData/inspections/spelling/ignoreEscapeSequence.py +++ b/python/testData/inspections/spelling/ignoreEscapeSequence.py @@ -1 +1,2 @@ -print "foo\nsomething" \ No newline at end of file +print "foo\nsomething" +print """foo\nbrbrbr""" From 2d0dc7d61fda46dd30817063f28be1a17793b8d8 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Thu, 5 Dec 2013 18:12:42 +0400 Subject: [PATCH 04/10] Use Pair instead of a custom class --- .../python/psi/PyStringLiteralExpression.java | 11 +--- .../impl/PyStringLiteralExpressionImpl.java | 63 ++++++------------- .../PythonSpellcheckerStrategy.java | 7 ++- 3 files changed, 24 insertions(+), 57 deletions(-) diff --git a/python/psi-api/src/com/jetbrains/python/psi/PyStringLiteralExpression.java b/python/psi-api/src/com/jetbrains/python/psi/PyStringLiteralExpression.java index d968d8c04ee1..836df6de724b 100644 --- a/python/psi-api/src/com/jetbrains/python/psi/PyStringLiteralExpression.java +++ b/python/psi-api/src/com/jetbrains/python/psi/PyStringLiteralExpression.java @@ -16,6 +16,7 @@ package com.jetbrains.python.psi; import com.intellij.lang.ASTNode; +import com.intellij.openapi.util.Pair; import com.intellij.openapi.util.TextRange; import com.intellij.psi.PsiLanguageInjectionHost; import org.jetbrains.annotations.NotNull; @@ -29,16 +30,8 @@ public interface PyStringLiteralExpression extends PyLiteralExpression, StringLi int valueOffsetToTextOffset(int valueOffset); - interface DecodedFragment { - @NotNull - TextRange getEncodedTextRange(); - - @NotNull - String getValue(); - } - @NotNull - List getDecodedFragments(); + List> getDecodedFragments(); void iterateCharacterRanges(TextRangeConsumer consumer); diff --git a/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java b/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java index 5bc50dd269b8..1c876ac23a15 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java @@ -153,45 +153,18 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt return text.length() > 0 && Character.toUpperCase(text.charAt(0)) == 'C'; } - public static class DecodedFragmentImpl implements DecodedFragment { - @NotNull private final TextRange myRange; - @NotNull private final String myValue; - - public DecodedFragmentImpl(@NotNull TextRange range, @NotNull String value) { - myRange = range; - myValue = value; - } - - @Override - @NotNull - public TextRange getEncodedTextRange() { - return myRange; - } - - @Override - @NotNull - public String getValue() { - return myValue; - } - - @Override - public String toString() { - return String.format("DecodedFragmentImpl(%s, \"%s\")", myRange, myValue); - } - } - @Override public void iterateCharacterRanges(PyStringLiteralExpression.TextRangeConsumer consumer) { - for (PyStringLiteralExpression.DecodedFragment fragment : getDecodedFragments()) { - final TextRange range = fragment.getEncodedTextRange(); - consumer.process(range.getStartOffset(), range.getEndOffset(), fragment.getValue()); + for (Pair fragment : getDecodedFragments()) { + final TextRange range = fragment.getFirst(); + consumer.process(range.getStartOffset(), range.getEndOffset(), fragment.getSecond()); } } @Override @NotNull - public List getDecodedFragments() { - final List result = new ArrayList(); + public List> getDecodedFragments() { + final List> result = new ArrayList>(); final int elementStart = getTextRange().getStartOffset(); for (ASTNode node : getStringNodes()) { final String text = node.getText(); @@ -204,15 +177,15 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt } @NotNull - private static List getDecodedFragments(@NotNull String encoded, int offset, boolean raw, boolean unicode) { - final List result = new ArrayList(); + private static List> getDecodedFragments(@NotNull String encoded, int offset, boolean raw, boolean unicode) { + final List> result = new ArrayList>(); final Matcher escMatcher = PATTERN_ESCAPE.matcher(encoded); int index = 0; while (escMatcher.find(index)) { if (index < escMatcher.start()) { final TextRange range = TextRange.create(index, escMatcher.start()); final TextRange offsetRange = range.shiftRight(offset); - result.add(new DecodedFragmentImpl(offsetRange, range.substring(encoded))); + result.add(Pair.create(offsetRange, range.substring(encoded))); } final String octal = escapeRegexGroup(escMatcher, EscapeRegexGroup.OCTAL); @@ -246,14 +219,14 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt if (str != null) { final TextRange wholeMatch = TextRange.create(escMatcher.start(), escMatcher.end()); - result.add(new DecodedFragmentImpl(wholeMatch.shiftRight(offset), str)); + result.add(Pair.create(wholeMatch.shiftRight(offset), str)); } index = escMatcher.end(); } final TextRange range = TextRange.create(index, encoded.length()); final TextRange offRange = range.shiftRight(offset); - result.add(new DecodedFragmentImpl(offRange, range.substring(encoded))); + result.add(Pair.create(offRange, range.substring(encoded))); return result; } @@ -271,8 +244,8 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt //assert child != null; if (stringValue == null) { final StringBuilder out = new StringBuilder(); - for (DecodedFragment fragment : getDecodedFragments()) { - out.append(fragment.getValue()); + for (Pair fragment : getDecodedFragments()) { + out.append(fragment.getSecond()); } stringValue = out.toString(); } @@ -369,11 +342,11 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt @Override public boolean decode(@NotNull final TextRange rangeInsideHost, @NotNull final StringBuilder outChars) { - for (DecodedFragment fragment : myHost.getDecodedFragments()) { - final TextRange encodedTextRange = fragment.getEncodedTextRange(); + for (Pair fragment : myHost.getDecodedFragments()) { + final TextRange encodedTextRange = fragment.getFirst(); final TextRange intersection = encodedTextRange.intersection(rangeInsideHost); if (intersection != null && !intersection.isEmpty()) { - final String value = fragment.getValue(); + final String value = fragment.getSecond(); final String intersectedValue; if (value.length() == 1 || value.length() == intersection.getLength()) { intersectedValue = value; @@ -393,11 +366,11 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt public int getOffsetInHost(final int offsetInDecoded, @NotNull final TextRange rangeInsideHost) { int offset = 0; int endOffset = -1; - for (DecodedFragment fragment : myHost.getDecodedFragments()) { - final TextRange encodedTextRange = fragment.getEncodedTextRange(); + for (Pair fragment : myHost.getDecodedFragments()) { + final TextRange encodedTextRange = fragment.getFirst(); final TextRange intersection = encodedTextRange.intersection(rangeInsideHost); if (intersection != null && !intersection.isEmpty()) { - final String value = fragment.getValue(); + final String value = fragment.getSecond(); final int valueLength = value.length(); if (valueLength == 0) { return -1; diff --git a/python/src/com/jetbrains/python/spellchecker/PythonSpellcheckerStrategy.java b/python/src/com/jetbrains/python/spellchecker/PythonSpellcheckerStrategy.java index 5d6033541294..b24e53c557ad 100644 --- a/python/src/com/jetbrains/python/spellchecker/PythonSpellcheckerStrategy.java +++ b/python/src/com/jetbrains/python/spellchecker/PythonSpellcheckerStrategy.java @@ -15,6 +15,7 @@ */ package com.jetbrains.python.spellchecker; +import com.intellij.openapi.util.Pair; import com.intellij.openapi.util.TextRange; import com.intellij.psi.PsiElement; import com.intellij.spellchecker.inspections.PlainTextSplitter; @@ -40,9 +41,9 @@ public class PythonSpellcheckerStrategy extends SpellcheckingStrategy { Splitter splitter = PlainTextSplitter.getInstance(); String text = element.getText(); if (text.indexOf('\\') >= 0) { - for (PyStringLiteralExpression.DecodedFragment fragment : element.getDecodedFragments()) { - final String value = fragment.getValue(); - consumer.consumeToken(element, value, false, fragment.getEncodedTextRange().getStartOffset(), TextRange.allOf(value), + for (Pair fragment : element.getDecodedFragments()) { + final String value = fragment.getSecond(); + consumer.consumeToken(element, value, false, fragment.getFirst().getStartOffset(), TextRange.allOf(value), PlainTextSplitter.getInstance()); } } From e1d45302b807d53797e3947dba49fae624075be0 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Thu, 5 Dec 2013 19:14:13 +0400 Subject: [PATCH 05/10] Switched to new string value iteration in Python file references inside string literals --- .../psi/PyStringLiteralFileReferenceSet.java | 74 ++++++------------- 1 file changed, 22 insertions(+), 52 deletions(-) diff --git a/python/openapi/src/com/jetbrains/python/psi/PyStringLiteralFileReferenceSet.java b/python/openapi/src/com/jetbrains/python/psi/PyStringLiteralFileReferenceSet.java index e74cd80a3260..c11cb290574d 100644 --- a/python/openapi/src/com/jetbrains/python/psi/PyStringLiteralFileReferenceSet.java +++ b/python/openapi/src/com/jetbrains/python/psi/PyStringLiteralFileReferenceSet.java @@ -20,17 +20,19 @@ import com.intellij.openapi.util.SystemInfo; import com.intellij.openapi.util.TextRange; import com.intellij.psi.PsiReferenceProvider; import com.intellij.psi.impl.source.resolve.reference.impl.providers.FileReference; -import com.intellij.psi.impl.source.resolve.reference.impl.providers.FileReferenceSet; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import java.util.ArrayList; import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * @author traff */ public class PyStringLiteralFileReferenceSet extends RootFileReferenceSet { + public static final Pattern DELIMITERS = Pattern.compile("\\\\|/"); private final PyStringLiteralExpression myStringLiteralExpression; @@ -57,61 +59,29 @@ public class PyStringLiteralFileReferenceSet extends RootFileReferenceSet { protected void reparse() { //noinspection ConstantConditions if (myStringLiteralExpression != null) { - MyTextRangeConsumer textRangeConsumer = new MyTextRangeConsumer(this); - - myStringLiteralExpression.iterateCharacterRanges(textRangeConsumer); - textRangeConsumer.finish(); - - List referencesList = textRangeConsumer.myReferenceList; - - myReferences = referencesList.toArray(new FileReference[referencesList.size()]); + final List references = getFileReferences(myStringLiteralExpression); + myReferences = references.toArray(new FileReference[references.size()]); } } - private static class MyTextRangeConsumer implements PyStringLiteralExpression.TextRangeConsumer { - private final StringBuilder myItem = new StringBuilder(); - private int myStartOffset = -1; - private int myIndex = 0; - private int myEndOffset = -1; - private final FileReferenceSet myFileReferenceSet; - - - private final List myReferenceList = new ArrayList(); - - private MyTextRangeConsumer(FileReferenceSet set) { - myFileReferenceSet = set; + @NotNull + private List getFileReferences(@NotNull PyStringLiteralExpression expression) { + final String value = expression.getStringValue(); + final Matcher matcher = DELIMITERS.matcher(value); + int start = 0; + int index = 0; + final List results = new ArrayList(); + while (matcher.find()) { + final TextRange range = TextRange.create(expression.valueOffsetToTextOffset(start), + expression.valueOffsetToTextOffset(matcher.start())); + results.add(createFileReference(range, index++, value.substring(start, matcher.start()))); + start = matcher.end(); } - - @Override - public boolean process(int startOffset, int endOffset, String value) { - if ("\\".equals(value) || "/".equals(value)) { - addReference(startOffset); - } - else { - if (myStartOffset == -1) { - myStartOffset = startOffset; - } - myEndOffset = endOffset; - myItem.append(value); - } - return true; - } - - private void addReference(int startOffset) { - if (myStartOffset != -1) { - final FileReference ref = myFileReferenceSet.createFileReference( - new TextRange(myStartOffset, startOffset), - myIndex++, - myItem.toString()); - myReferenceList.add(ref); - myStartOffset = -1; - myItem.setLength(0); - } - } - - - public void finish() { - addReference(myEndOffset); + if (start != 0) { + final TextRange range = TextRange.create(expression.valueOffsetToTextOffset(start), + expression.valueOffsetToTextOffset(value.length())); + results.add(createFileReference(range, index, value.substring(start))); } + return results; } } From bc9e47fbbe4dc8d62d723612ac11c7a939faf4df Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Thu, 5 Dec 2013 19:16:22 +0400 Subject: [PATCH 06/10] Removed unused PyStringLiteralExpression.iterateCharacterRanges() --- .../python/psi/PyStringLiteralExpression.java | 17 ----------------- .../psi/impl/PyStringLiteralExpressionImpl.java | 8 -------- .../jetbrains/python/PyStringLiteralTest.java | 11 ++++------- 3 files changed, 4 insertions(+), 32 deletions(-) diff --git a/python/psi-api/src/com/jetbrains/python/psi/PyStringLiteralExpression.java b/python/psi-api/src/com/jetbrains/python/psi/PyStringLiteralExpression.java index 836df6de724b..f7dd122680df 100644 --- a/python/psi-api/src/com/jetbrains/python/psi/PyStringLiteralExpression.java +++ b/python/psi-api/src/com/jetbrains/python/psi/PyStringLiteralExpression.java @@ -32,21 +32,4 @@ public interface PyStringLiteralExpression extends PyLiteralExpression, StringLi @NotNull List> getDecodedFragments(); - - void iterateCharacterRanges(TextRangeConsumer consumer); - - /** - * Iterator over decoded string characters. - */ - interface TextRangeConsumer { - /** - * Process a decoded character. - * - * @param startOffset start offset in the un-decoded string - * @param endOffset end offset in the un-decoded string - * @param value decoded character value - * @return false in order to stop iteration - */ - boolean process(int startOffset, int endOffset, String value); - } } diff --git a/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java b/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java index 1c876ac23a15..622eb7e1e461 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java @@ -153,14 +153,6 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt return text.length() > 0 && Character.toUpperCase(text.charAt(0)) == 'C'; } - @Override - public void iterateCharacterRanges(PyStringLiteralExpression.TextRangeConsumer consumer) { - for (Pair fragment : getDecodedFragments()) { - final TextRange range = fragment.getFirst(); - consumer.process(range.getStartOffset(), range.getEndOffset(), fragment.getSecond()); - } - } - @Override @NotNull public List> getDecodedFragments() { diff --git a/python/testSrc/com/jetbrains/python/PyStringLiteralTest.java b/python/testSrc/com/jetbrains/python/PyStringLiteralTest.java index 016bbe5f8e0b..765cbd15afdd 100644 --- a/python/testSrc/com/jetbrains/python/PyStringLiteralTest.java +++ b/python/testSrc/com/jetbrains/python/PyStringLiteralTest.java @@ -15,6 +15,7 @@ */ package com.jetbrains.python; +import com.intellij.openapi.util.Pair; import com.intellij.openapi.util.TextRange; import com.intellij.psi.LiteralTextEscaper; import com.intellij.psi.PsiFile; @@ -123,13 +124,9 @@ public class PyStringLiteralTest extends PyTestCase { final PyStringLiteralExpression expr = createLiteralFromText(text); assertNotNull(expr); final List characters = new ArrayList(); - expr.iterateCharacterRanges(new PyStringLiteralExpression.TextRangeConsumer() { - @Override - public boolean process(int startOffset, int endOffset, String value) { - characters.add(value); - return true; - } - }); + for (Pair fragment : expr.getDecodedFragments()) { + characters.add(fragment.getSecond()); + } return characters; } } From 658b3cf1c5dff8d070f0e97c40fc8b161ac56aa7 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Thu, 5 Dec 2013 19:43:05 +0400 Subject: [PATCH 07/10] Fixed detecting file references in Python strings without delimiters --- .../psi/PyStringLiteralFileReferenceSet.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/python/openapi/src/com/jetbrains/python/psi/PyStringLiteralFileReferenceSet.java b/python/openapi/src/com/jetbrains/python/psi/PyStringLiteralFileReferenceSet.java index c11cb290574d..6c470c739c36 100644 --- a/python/openapi/src/com/jetbrains/python/psi/PyStringLiteralFileReferenceSet.java +++ b/python/openapi/src/com/jetbrains/python/psi/PyStringLiteralFileReferenceSet.java @@ -72,15 +72,19 @@ public class PyStringLiteralFileReferenceSet extends RootFileReferenceSet { int index = 0; final List results = new ArrayList(); while (matcher.find()) { - final TextRange range = TextRange.create(expression.valueOffsetToTextOffset(start), - expression.valueOffsetToTextOffset(matcher.start())); - results.add(createFileReference(range, index++, value.substring(start, matcher.start()))); + final String s = value.substring(start, matcher.start()); + if (!s.isEmpty()) { + final TextRange range = TextRange.create(expression.valueOffsetToTextOffset(start), + expression.valueOffsetToTextOffset(matcher.start())); + results.add(createFileReference(range, index++, s)); + } start = matcher.end(); } - if (start != 0) { + final String s = value.substring(start); + if (!s.isEmpty()) { final TextRange range = TextRange.create(expression.valueOffsetToTextOffset(start), expression.valueOffsetToTextOffset(value.length())); - results.add(createFileReference(range, index, value.substring(start))); + results.add(createFileReference(range, index, s)); } return results; } From 713f67177a0df4d22703cba3237c39482a7f1617 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Thu, 5 Dec 2013 19:57:11 +0400 Subject: [PATCH 08/10] Added caching of decoded fragments of string literals --- .../impl/PyStringLiteralExpressionImpl.java | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java b/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java index 622eb7e1e461..ec4b1468d7f7 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java @@ -63,6 +63,7 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt private static final Map escapeMap = initializeEscapeMap(); private String stringValue; private List valueTextRanges; + @Nullable private List> myDecodedFragments; private final DefaultRegExpPropertiesProvider myPropertiesProvider; private static Map initializeEscapeMap() { @@ -95,6 +96,7 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt super.subtreeChanged(); stringValue = null; valueTextRanges = null; + myDecodedFragments = null; } public List getStringValueTextRanges() { @@ -156,16 +158,19 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt @Override @NotNull public List> getDecodedFragments() { - final List> result = new ArrayList>(); - final int elementStart = getTextRange().getStartOffset(); - for (ASTNode node : getStringNodes()) { - final String text = node.getText(); - final TextRange textRange = getNodeTextRange(text); - final int offset = node.getTextRange().getStartOffset() - elementStart + textRange.getStartOffset(); - final String encoded = textRange.substring(text); - result.addAll(getDecodedFragments(encoded, offset, isRaw(text), isUnicode(text))); + if (myDecodedFragments == null) { + final List> result = new ArrayList>(); + final int elementStart = getTextRange().getStartOffset(); + for (ASTNode node : getStringNodes()) { + final String text = node.getText(); + final TextRange textRange = getNodeTextRange(text); + final int offset = node.getTextRange().getStartOffset() - elementStart + textRange.getStartOffset(); + final String encoded = textRange.substring(text); + result.addAll(getDecodedFragments(encoded, offset, isRaw(text), isUnicode(text))); + } + myDecodedFragments = result; } - return result; + return myDecodedFragments; } @NotNull From 71fdf968d98db2e61b946f7a94e1704a9acb5543 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Thu, 5 Dec 2013 20:42:27 +0400 Subject: [PATCH 09/10] Fixed LiteralTextEscaper.getOffsetInHost() for partial ranges inside host that contain escapes --- .../psi/impl/PyStringLiteralExpressionImpl.java | 6 ++++-- .../jetbrains/python/PyStringLiteralTest.java | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java b/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java index ec4b1468d7f7..7595fde33fe8 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java @@ -369,6 +369,7 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt if (intersection != null && !intersection.isEmpty()) { final String value = fragment.getSecond(); final int valueLength = value.length(); + final int intersectionLength = intersection.getLength(); if (valueLength == 0) { return -1; } @@ -376,14 +377,15 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt if (offset == offsetInDecoded) { return intersection.getStartOffset(); } + offset++; } else { - if (offset + valueLength >= offsetInDecoded) { + if (offset + intersectionLength >= offsetInDecoded) { final int delta = offsetInDecoded - offset; return intersection.getStartOffset() + delta; } + offset += intersectionLength; } - offset += valueLength; endOffset = intersection.getEndOffset(); } } diff --git a/python/testSrc/com/jetbrains/python/PyStringLiteralTest.java b/python/testSrc/com/jetbrains/python/PyStringLiteralTest.java index 765cbd15afdd..393358443576 100644 --- a/python/testSrc/com/jetbrains/python/PyStringLiteralTest.java +++ b/python/testSrc/com/jetbrains/python/PyStringLiteralTest.java @@ -98,6 +98,23 @@ public class PyStringLiteralTest extends PyTestCase { Arrays.asList("\nfoo.", "\\", "\nbar\n")); } + public void testEscaperOffsetInEscapedBackslash() { + final PyStringLiteralExpression expr = createLiteralFromText("'XXX foo.\\\\bar YYY'"); + assertNotNull(expr); + final LiteralTextEscaper escaper = expr.createLiteralTextEscaper(); + final TextRange range = TextRange.create(5, 14); + assertEquals(5, escaper.getOffsetInHost(0, range)); + assertEquals(6, escaper.getOffsetInHost(1, range)); + assertEquals(7, escaper.getOffsetInHost(2, range)); + assertEquals(8, escaper.getOffsetInHost(3, range)); + assertEquals(9, escaper.getOffsetInHost(4, range)); + assertEquals(11, escaper.getOffsetInHost(5, range)); + assertEquals(12, escaper.getOffsetInHost(6, range)); + assertEquals(13, escaper.getOffsetInHost(7, range)); + assertEquals(14, escaper.getOffsetInHost(8, range)); + assertEquals(-1, escaper.getOffsetInHost(9, range)); + } + private static String decodeRange(PyStringLiteralExpression expr, TextRange range) { final StringBuilder builder = new StringBuilder(); expr.createLiteralTextEscaper().decode(range, builder); From 546c532128ee525ac5d23594aae5f9f70ce0b297 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Thu, 5 Dec 2013 21:00:22 +0400 Subject: [PATCH 10/10] Detect SQL fragments only in the beginning of string literals --- python/IntelliLang-python/src/pyInjections.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/IntelliLang-python/src/pyInjections.xml b/python/IntelliLang-python/src/pyInjections.xml index a75015e02441..f561c1d33016 100644 --- a/python/IntelliLang-python/src/pyInjections.xml +++ b/python/IntelliLang-python/src/pyInjections.xml @@ -9,6 +9,6 @@ "SQL select/delete/insert/update/create" - +