diff --git a/python/psi-api/src/com/jetbrains/python/psi/PyRichStringNode.java b/python/psi-api/src/com/jetbrains/python/psi/PyRichStringNode.java index aca8a87cf20e..470ea1e9b093 100644 --- a/python/psi-api/src/com/jetbrains/python/psi/PyRichStringNode.java +++ b/python/psi-api/src/com/jetbrains/python/psi/PyRichStringNode.java @@ -1,9 +1,11 @@ package com.jetbrains.python.psi; +import com.intellij.openapi.util.Pair; import com.intellij.openapi.util.TextRange; import com.intellij.psi.PsiElement; import org.jetbrains.annotations.NotNull; +import java.util.List; import java.util.Set; /** @@ -51,6 +53,9 @@ public interface PyRichStringNode extends PsiElement { @NotNull String getContent(); + @NotNull + List> getDecodedFragments(); + @NotNull String getQuote(); diff --git a/python/src/com/jetbrains/python/inspections/PyStringFormatParser.java b/python/src/com/jetbrains/python/inspections/PyStringFormatParser.java index cecad6973f22..29b71f6b0d0b 100644 --- a/python/src/com/jetbrains/python/inspections/PyStringFormatParser.java +++ b/python/src/com/jetbrains/python/inspections/PyStringFormatParser.java @@ -21,7 +21,7 @@ import com.intellij.psi.PsiElement; import com.intellij.util.ObjectUtils; import com.jetbrains.python.PyNames; import com.jetbrains.python.psi.*; -import com.jetbrains.python.psi.impl.PyStringLiteralExpressionImpl; +import com.jetbrains.python.psi.impl.PyStringLiteralDecoder; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -625,7 +625,7 @@ public class PyStringFormatParser { @NotNull public static List getEscapeRanges(@NotNull String s) { final List ranges = new ArrayList<>(); - Matcher matcher = PyStringLiteralExpressionImpl.PATTERN_ESCAPE.matcher(s); + Matcher matcher = PyStringLiteralDecoder.PATTERN_ESCAPE.matcher(s); while (matcher.find()) { ranges.add(TextRange.create(matcher.start(), matcher.end())); } diff --git a/python/src/com/jetbrains/python/psi/impl/PyLiteralStringNodeImpl.java b/python/src/com/jetbrains/python/psi/impl/PyLiteralStringNodeImpl.java index f70b0c68460d..7a5c8f7fd08d 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyLiteralStringNodeImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyLiteralStringNodeImpl.java @@ -10,6 +10,7 @@ import com.jetbrains.python.psi.PyStringLiteralUtil; import org.jetbrains.annotations.NotNull; import java.util.EnumSet; +import java.util.List; import java.util.Set; /** @@ -37,6 +38,14 @@ public class PyLiteralStringNodeImpl extends LeafPsiElement implements PyLiteral return getContentRange().substring(getText()); } + @NotNull + @Override + public List> getDecodedFragments() { + final PyStringLiteralDecoder decoder = new PyStringLiteralDecoder(this); + decoder.decodeContent(); + return decoder.getResult(); + } + @NotNull @Override public String getTextWithoutPrefix() { @@ -46,11 +55,7 @@ public class PyLiteralStringNodeImpl extends LeafPsiElement implements PyLiteral @NotNull @Override public TextRange getContentRange() { - final Pair quotes = PyStringLiteralUtil.getQuotes(getText()); - assert quotes != null; - final String prefixWithOpeningQuote = quotes.getFirst(); - final String closingQuote = quotes.getSecond(); - return TextRange.create(prefixWithOpeningQuote.length(), getTextLength() - closingQuote.length()); + return PyStringLiteralExpressionImpl.getNodeTextRange(getText()); } @NotNull diff --git a/python/src/com/jetbrains/python/psi/impl/PyStringLiteralDecoder.java b/python/src/com/jetbrains/python/psi/impl/PyStringLiteralDecoder.java new file mode 100644 index 000000000000..15a648402c34 --- /dev/null +++ b/python/src/com/jetbrains/python/psi/impl/PyStringLiteralDecoder.java @@ -0,0 +1,157 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.jetbrains.python.psi.impl; + +import com.intellij.openapi.util.Pair; +import com.intellij.openapi.util.TextRange; +import com.intellij.psi.PsiFile; +import com.jetbrains.python.psi.FutureFeature; +import com.jetbrains.python.psi.LanguageLevel; +import com.jetbrains.python.psi.PyFile; +import com.jetbrains.python.psi.PyRichStringNode; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class PyStringLiteralDecoder { + public static final Pattern PATTERN_ESCAPE = Pattern + .compile("\\\\(\n|\\\\|'|\"|a|b|f|n|r|t|v|([0-7]{1,3})|x([0-9a-fA-F]{1,2})" + "|N(\\{.*?\\})|u([0-9a-fA-F]{4})|U([0-9a-fA-F]{8}))"); + // -> 1 -> 2 <--> 3 <- -> 4 <--> 5 <- -> 6 <-<- + + private enum EscapeRegexGroup { + WHOLE_MATCH, + ESCAPED_SUBSTRING, + OCTAL, + HEXADECIMAL, + UNICODE_NAMED, + UNICODE_16BIT, + UNICODE_32BIT + } + + private static final Map escapeMap = initializeEscapeMap(); + + @NotNull + private static Map initializeEscapeMap() { + Map map = new HashMap<>(); + map.put("\n", "\n"); + map.put("\\", "\\"); + map.put("'", "'"); + map.put("\"", "\""); + map.put("a", "\001"); + map.put("b", "\b"); + map.put("f", "\f"); + map.put("n", "\n"); + map.put("r", "\r"); + map.put("t", "\t"); + map.put("v", "\013"); + return map; + } + + private final PyRichStringNode myNode; + private final List> myResult = new ArrayList<>(); + + public PyStringLiteralDecoder(@NotNull PyRichStringNode node) { + myNode = node; + } + + public void decodeContent() { + decodeRange(myNode.getContentRange()); + } + + public void decodeRange(@NotNull TextRange range) { + myResult.addAll(decodeFragment(range.substring(myNode.getText()), range.getStartOffset())); + } + + @NotNull + public List> getResult() { + return myResult; + } + + @NotNull + private List> decodeFragment(@NotNull String encoded, int offset) { + final boolean raw = myNode.isRaw(); + final boolean unicode = myNode.isUnicode() || isUnicodeByDefault(); + + final List> result = new ArrayList<>(); + final Matcher escMatcher = PATTERN_ESCAPE.matcher(encoded); + int index = 0; + while (escMatcher.find(index)) { + if (index < escMatcher.start()) { + final TextRange range = TextRange.create(index, escMatcher.start()); + final TextRange offsetRange = range.shiftRight(offset); + result.add(Pair.create(offsetRange, range.substring(encoded))); + } + + final String octal = escapeRegexGroup(escMatcher, EscapeRegexGroup.OCTAL); + final String hex = escapeRegexGroup(escMatcher, EscapeRegexGroup.HEXADECIMAL); + // TODO: Implement unicode character name escapes: EscapeRegexGroup.UNICODE_NAMED + final String unicode16 = escapeRegexGroup(escMatcher, EscapeRegexGroup.UNICODE_16BIT); + final String unicode32 = escapeRegexGroup(escMatcher, EscapeRegexGroup.UNICODE_32BIT); + final String wholeMatch = escapeRegexGroup(escMatcher, EscapeRegexGroup.WHOLE_MATCH); + + final boolean escapedUnicode = raw && unicode || !raw; + + final String str; + if (!raw && octal != null) { + str = new String(new char[]{(char)Integer.parseInt(octal, 8)}); + } + else if (!raw && hex != null) { + str = new String(new char[]{(char)Integer.parseInt(hex, 16)}); + } + else if (escapedUnicode && unicode16 != null) { + str = unicode ? new String(new char[]{(char)Integer.parseInt(unicode16, 16)}) : wholeMatch; + } + else if (escapedUnicode && unicode32 != null) { + String s = wholeMatch; + if (unicode) { + try { + s = new String(Character.toChars((int)Long.parseLong(unicode32, 16))); + } + catch (IllegalArgumentException ignored) { + } + } + str = s; + } + else if (raw) { + str = wholeMatch; + } + else { + final String toReplace = escapeRegexGroup(escMatcher, EscapeRegexGroup.ESCAPED_SUBSTRING); + str = escapeMap.get(toReplace); + } + + if (str != null) { + final TextRange wholeMatchRange = TextRange.create(escMatcher.start(), escMatcher.end()); + result.add(Pair.create(wholeMatchRange.shiftRight(offset), str)); + } + + index = escMatcher.end(); + } + final TextRange range = TextRange.create(index, encoded.length()); + final TextRange offRange = range.shiftRight(offset); + result.add(Pair.create(offRange, range.substring(encoded))); + return result; + } + + @Nullable + private static String escapeRegexGroup(@NotNull Matcher matcher, EscapeRegexGroup group) { + return matcher.group(group.ordinal()); + } + + private boolean isUnicodeByDefault() { + if (!LanguageLevel.forElement(myNode).isPython2()) { + return true; + } + final PsiFile file = myNode.getContainingFile(); + if (file instanceof PyFile) { + final PyFile pyFile = (PyFile)file; + return pyFile.hasImportFromFuture(FutureFeature.UNICODE_LITERALS); + } + return false; + } +} diff --git a/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java b/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java index 959e1433f27c..f03c493d473d 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java @@ -45,50 +45,19 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import javax.swing.*; -import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import java.util.List; public class PyStringLiteralExpressionImpl extends PyElementImpl implements PyStringLiteralExpression, RegExpLanguageHost, PsiLiteralValue { private static final Logger LOG = Logger.getInstance(PyStringLiteralExpressionImpl.class); - public static final Pattern PATTERN_ESCAPE = Pattern - .compile("\\\\(\n|\\\\|'|\"|a|b|f|n|r|t|v|([0-7]{1,3})|x([0-9a-fA-F]{1,2})" + "|N(\\{.*?\\})|u([0-9a-fA-F]{4})|U([0-9a-fA-F]{8}))"); - // -> 1 -> 2 <--> 3 <- -> 4 <--> 5 <- -> 6 <-<- - - private enum EscapeRegexGroup { - WHOLE_MATCH, - ESCAPED_SUBSTRING, - OCTAL, - HEXADECIMAL, - UNICODE_NAMED, - UNICODE_16BIT, - UNICODE_32BIT - } - - private static final Map escapeMap = initializeEscapeMap(); @Nullable private volatile String myStringValue; @Nullable private volatile List myValueTextRanges; @Nullable private volatile List> myDecodedFragments; private final DefaultRegExpPropertiesProvider myPropertiesProvider; - @NotNull - private static Map initializeEscapeMap() { - Map map = new HashMap<>(); - map.put("\n", "\n"); - map.put("\\", "\\"); - map.put("'", "'"); - map.put("\"", "\""); - map.put("a", "\001"); - map.put("b", "\b"); - map.put("f", "\f"); - map.put("n", "\n"); - map.put("r", "\r"); - map.put("t", "\t"); - map.put("v", "\013"); - return map; - } - public PyStringLiteralExpressionImpl(ASTNode astNode) { super(astNode); myPropertiesProvider = DefaultRegExpPropertiesProvider.getInstance(); @@ -112,13 +81,13 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt public List getStringValueTextRanges() { List result = myValueTextRanges; if (result == null) { - int elStart = getTextRange().getStartOffset(); - List ranges = new ArrayList<>(); - for (ASTNode node : getStringNodes()) { - TextRange range = getNodeTextRange(node.getText()); - int nodeOffset = node.getStartOffset() - elStart; - ranges.add(TextRange.from(nodeOffset + range.getStartOffset(), range.getLength())); - } + final int elementStart = getTextRange().getStartOffset(); + final List ranges = StreamEx.of(getGluedStringNodes()) + .map(node -> { + final int nodeRelativeOffset = node.getTextRange().getStartOffset() - elementStart; + return node.getContentRange().shiftRight(nodeRelativeOffset); + }) + .toList(); myValueTextRanges = result = Collections.unmodifiableList(ranges); } return result; @@ -146,36 +115,20 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt return PyStringLiteralUtil.getPrefixEndOffset(text, 0); } - private boolean isUnicodeByDefault() { - if (!LanguageLevel.forElement(this).isPython2()) { - return true; - } - final PsiFile file = getContainingFile(); - if (file instanceof PyFile) { - final PyFile pyFile = (PyFile)file; - return pyFile.hasImportFromFuture(FutureFeature.UNICODE_LITERALS); - } - return false; - } - @Override @NotNull public List> getDecodedFragments() { + final int elementStart = getTextRange().getStartOffset(); List> result = myDecodedFragments; if (result == null) { - result = new ArrayList<>(); - final int elementStart = getTextRange().getStartOffset(); - final boolean unicodeByDefault = isUnicodeByDefault(); - for (ASTNode node : getStringNodes()) { - final String text = node.getText(); - final TextRange textRange = getNodeTextRange(text); - final int offset = node.getTextRange().getStartOffset() - elementStart + textRange.getStartOffset(); - final String encoded = textRange.substring(text); - final boolean hasRawPrefix = PyStringLiteralUtil.isRawPrefix(PyStringLiteralUtil.getPrefix(text)); - final boolean hasUnicodePrefix = PyStringLiteralUtil.isUnicodePrefix(PyStringLiteralUtil.getPrefix(text)); - result.addAll(getDecodedFragments(encoded, offset, hasRawPrefix, unicodeByDefault || hasUnicodePrefix)); - } - myDecodedFragments = result; + final List> combined = StreamEx.of(getGluedStringNodes()) + .flatMap(node -> StreamEx.of(node.getDecodedFragments()) + .map(pair -> { + final int nodeRelativeOffset = node.getTextRange().getStartOffset() - elementStart; + return Pair.create(pair.getFirst().shiftRight(nodeRelativeOffset), pair.getSecond()); + })) + .toList(); + myDecodedFragments = result = Collections.unmodifiableList(combined); } return result; } @@ -186,74 +139,6 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt return stringNodes.size() == 1 && stringNodes.get(0).getElementType() == PyTokenTypes.DOCSTRING; } - @NotNull - private static List> getDecodedFragments(@NotNull String encoded, int offset, boolean raw, boolean unicode) { - final List> result = new ArrayList<>(); - final Matcher escMatcher = PATTERN_ESCAPE.matcher(encoded); - int index = 0; - while (escMatcher.find(index)) { - if (index < escMatcher.start()) { - final TextRange range = TextRange.create(index, escMatcher.start()); - final TextRange offsetRange = range.shiftRight(offset); - result.add(Pair.create(offsetRange, range.substring(encoded))); - } - - final String octal = escapeRegexGroup(escMatcher, EscapeRegexGroup.OCTAL); - final String hex = escapeRegexGroup(escMatcher, EscapeRegexGroup.HEXADECIMAL); - // TODO: Implement unicode character name escapes: EscapeRegexGroup.UNICODE_NAMED - final String unicode16 = escapeRegexGroup(escMatcher, EscapeRegexGroup.UNICODE_16BIT); - final String unicode32 = escapeRegexGroup(escMatcher, EscapeRegexGroup.UNICODE_32BIT); - final String wholeMatch = escapeRegexGroup(escMatcher, EscapeRegexGroup.WHOLE_MATCH); - - final boolean escapedUnicode = raw && unicode || !raw; - - final String str; - if (!raw && octal != null) { - str = new String(new char[]{(char)Integer.parseInt(octal, 8)}); - } - else if (!raw && hex != null) { - str = new String(new char[]{(char)Integer.parseInt(hex, 16)}); - } - else if (escapedUnicode && unicode16 != null) { - str = unicode ? new String(new char[]{(char)Integer.parseInt(unicode16, 16)}) : wholeMatch; - } - else if (escapedUnicode && unicode32 != null) { - String s = wholeMatch; - if (unicode) { - try { - s = new String(Character.toChars((int)Long.parseLong(unicode32, 16))); - } - catch (IllegalArgumentException ignored) { - } - } - str = s; - } - else if (raw) { - str = wholeMatch; - } - else { - final String toReplace = escapeRegexGroup(escMatcher, EscapeRegexGroup.ESCAPED_SUBSTRING); - str = escapeMap.get(toReplace); - } - - if (str != null) { - final TextRange wholeMatchRange = TextRange.create(escMatcher.start(), escMatcher.end()); - result.add(Pair.create(wholeMatchRange.shiftRight(offset), str)); - } - - index = escMatcher.end(); - } - final TextRange range = TextRange.create(index, encoded.length()); - final TextRange offRange = range.shiftRight(offset); - result.add(Pair.create(offRange, range.substring(encoded))); - return result; - } - - @Nullable - private static String escapeRegexGroup(@NotNull Matcher matcher, EscapeRegexGroup group) { - return matcher.group(group.ordinal()); - } - @Override @NotNull public List getStringNodes() { @@ -321,7 +206,8 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt PyFile file = PsiTreeUtil.getParentOfType(this, PyFile.class); if (file != null) { IElementType type = PythonHighlightingLexer.convertStringType(getStringNodes().get(0).getElementType(), text, - LanguageLevel.forElement(this), file.hasImportFromFuture(FutureFeature.UNICODE_LITERALS)); + LanguageLevel.forElement(this), + file.hasImportFromFuture(FutureFeature.UNICODE_LITERALS)); if (PyTokenTypes.UNICODE_NODES.contains(type)) { return PyBuiltinCache.getInstance(this).getUnicodeType(LanguageLevel.forElement(this)); }