From d74cfa71f5df306f306bb7c417ae7619827b7d4a Mon Sep 17 00:00:00 2001 From: Mikhail Golubev Date: Mon, 20 Aug 2018 18:11:27 +0300 Subject: [PATCH] PY-31442 Methods of StringLiteralExpression delegate to those of RichStringNode First of all, getDecodedFragments() and getStringLiteralTextRanges(). Moreover, all the logic related to unescaping of string literal content is moved to the dedicated class PyStringLiteralDecoder. --- .../python/psi/PyRichStringNode.java | 5 + .../inspections/PyStringFormatParser.java | 4 +- .../psi/impl/PyLiteralStringNodeImpl.java | 15 +- .../psi/impl/PyStringLiteralDecoder.java | 157 +++++++++++++++++ .../impl/PyStringLiteralExpressionImpl.java | 158 +++--------------- 5 files changed, 196 insertions(+), 143 deletions(-) create mode 100644 python/src/com/jetbrains/python/psi/impl/PyStringLiteralDecoder.java diff --git a/python/psi-api/src/com/jetbrains/python/psi/PyRichStringNode.java b/python/psi-api/src/com/jetbrains/python/psi/PyRichStringNode.java index aca8a87cf20e..470ea1e9b093 100644 --- a/python/psi-api/src/com/jetbrains/python/psi/PyRichStringNode.java +++ b/python/psi-api/src/com/jetbrains/python/psi/PyRichStringNode.java @@ -1,9 +1,11 @@ package com.jetbrains.python.psi; +import com.intellij.openapi.util.Pair; import com.intellij.openapi.util.TextRange; import com.intellij.psi.PsiElement; import org.jetbrains.annotations.NotNull; +import java.util.List; import java.util.Set; /** @@ -51,6 +53,9 @@ public interface PyRichStringNode extends PsiElement { @NotNull String getContent(); + @NotNull + List> getDecodedFragments(); + @NotNull String getQuote(); diff --git a/python/src/com/jetbrains/python/inspections/PyStringFormatParser.java b/python/src/com/jetbrains/python/inspections/PyStringFormatParser.java index cecad6973f22..29b71f6b0d0b 100644 --- a/python/src/com/jetbrains/python/inspections/PyStringFormatParser.java +++ b/python/src/com/jetbrains/python/inspections/PyStringFormatParser.java @@ -21,7 +21,7 @@ import com.intellij.psi.PsiElement; import com.intellij.util.ObjectUtils; import com.jetbrains.python.PyNames; import com.jetbrains.python.psi.*; -import com.jetbrains.python.psi.impl.PyStringLiteralExpressionImpl; +import com.jetbrains.python.psi.impl.PyStringLiteralDecoder; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -625,7 +625,7 @@ public class PyStringFormatParser { @NotNull public static List getEscapeRanges(@NotNull String s) { final List ranges = new ArrayList<>(); - Matcher matcher = PyStringLiteralExpressionImpl.PATTERN_ESCAPE.matcher(s); + Matcher matcher = PyStringLiteralDecoder.PATTERN_ESCAPE.matcher(s); while (matcher.find()) { ranges.add(TextRange.create(matcher.start(), matcher.end())); } diff --git a/python/src/com/jetbrains/python/psi/impl/PyLiteralStringNodeImpl.java b/python/src/com/jetbrains/python/psi/impl/PyLiteralStringNodeImpl.java index f70b0c68460d..7a5c8f7fd08d 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyLiteralStringNodeImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyLiteralStringNodeImpl.java @@ -10,6 +10,7 @@ import com.jetbrains.python.psi.PyStringLiteralUtil; import org.jetbrains.annotations.NotNull; import java.util.EnumSet; +import java.util.List; import java.util.Set; /** @@ -37,6 +38,14 @@ public class PyLiteralStringNodeImpl extends LeafPsiElement implements PyLiteral return getContentRange().substring(getText()); } + @NotNull + @Override + public List> getDecodedFragments() { + final PyStringLiteralDecoder decoder = new PyStringLiteralDecoder(this); + decoder.decodeContent(); + return decoder.getResult(); + } + @NotNull @Override public String getTextWithoutPrefix() { @@ -46,11 +55,7 @@ public class PyLiteralStringNodeImpl extends LeafPsiElement implements PyLiteral @NotNull @Override public TextRange getContentRange() { - final Pair quotes = PyStringLiteralUtil.getQuotes(getText()); - assert quotes != null; - final String prefixWithOpeningQuote = quotes.getFirst(); - final String closingQuote = quotes.getSecond(); - return TextRange.create(prefixWithOpeningQuote.length(), getTextLength() - closingQuote.length()); + return PyStringLiteralExpressionImpl.getNodeTextRange(getText()); } @NotNull diff --git a/python/src/com/jetbrains/python/psi/impl/PyStringLiteralDecoder.java b/python/src/com/jetbrains/python/psi/impl/PyStringLiteralDecoder.java new file mode 100644 index 000000000000..15a648402c34 --- /dev/null +++ b/python/src/com/jetbrains/python/psi/impl/PyStringLiteralDecoder.java @@ -0,0 +1,157 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.jetbrains.python.psi.impl; + +import com.intellij.openapi.util.Pair; +import com.intellij.openapi.util.TextRange; +import com.intellij.psi.PsiFile; +import com.jetbrains.python.psi.FutureFeature; +import com.jetbrains.python.psi.LanguageLevel; +import com.jetbrains.python.psi.PyFile; +import com.jetbrains.python.psi.PyRichStringNode; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class PyStringLiteralDecoder { + public static final Pattern PATTERN_ESCAPE = Pattern + .compile("\\\\(\n|\\\\|'|\"|a|b|f|n|r|t|v|([0-7]{1,3})|x([0-9a-fA-F]{1,2})" + "|N(\\{.*?\\})|u([0-9a-fA-F]{4})|U([0-9a-fA-F]{8}))"); + // -> 1 -> 2 <--> 3 <- -> 4 <--> 5 <- -> 6 <-<- + + private enum EscapeRegexGroup { + WHOLE_MATCH, + ESCAPED_SUBSTRING, + OCTAL, + HEXADECIMAL, + UNICODE_NAMED, + UNICODE_16BIT, + UNICODE_32BIT + } + + private static final Map escapeMap = initializeEscapeMap(); + + @NotNull + private static Map initializeEscapeMap() { + Map map = new HashMap<>(); + map.put("\n", "\n"); + map.put("\\", "\\"); + map.put("'", "'"); + map.put("\"", "\""); + map.put("a", "\001"); + map.put("b", "\b"); + map.put("f", "\f"); + map.put("n", "\n"); + map.put("r", "\r"); + map.put("t", "\t"); + map.put("v", "\013"); + return map; + } + + private final PyRichStringNode myNode; + private final List> myResult = new ArrayList<>(); + + public PyStringLiteralDecoder(@NotNull PyRichStringNode node) { + myNode = node; + } + + public void decodeContent() { + decodeRange(myNode.getContentRange()); + } + + public void decodeRange(@NotNull TextRange range) { + myResult.addAll(decodeFragment(range.substring(myNode.getText()), range.getStartOffset())); + } + + @NotNull + public List> getResult() { + return myResult; + } + + @NotNull + private List> decodeFragment(@NotNull String encoded, int offset) { + final boolean raw = myNode.isRaw(); + final boolean unicode = myNode.isUnicode() || isUnicodeByDefault(); + + final List> result = new ArrayList<>(); + final Matcher escMatcher = PATTERN_ESCAPE.matcher(encoded); + int index = 0; + while (escMatcher.find(index)) { + if (index < escMatcher.start()) { + final TextRange range = TextRange.create(index, escMatcher.start()); + final TextRange offsetRange = range.shiftRight(offset); + result.add(Pair.create(offsetRange, range.substring(encoded))); + } + + final String octal = escapeRegexGroup(escMatcher, EscapeRegexGroup.OCTAL); + final String hex = escapeRegexGroup(escMatcher, EscapeRegexGroup.HEXADECIMAL); + // TODO: Implement unicode character name escapes: EscapeRegexGroup.UNICODE_NAMED + final String unicode16 = escapeRegexGroup(escMatcher, EscapeRegexGroup.UNICODE_16BIT); + final String unicode32 = escapeRegexGroup(escMatcher, EscapeRegexGroup.UNICODE_32BIT); + final String wholeMatch = escapeRegexGroup(escMatcher, EscapeRegexGroup.WHOLE_MATCH); + + final boolean escapedUnicode = raw && unicode || !raw; + + final String str; + if (!raw && octal != null) { + str = new String(new char[]{(char)Integer.parseInt(octal, 8)}); + } + else if (!raw && hex != null) { + str = new String(new char[]{(char)Integer.parseInt(hex, 16)}); + } + else if (escapedUnicode && unicode16 != null) { + str = unicode ? new String(new char[]{(char)Integer.parseInt(unicode16, 16)}) : wholeMatch; + } + else if (escapedUnicode && unicode32 != null) { + String s = wholeMatch; + if (unicode) { + try { + s = new String(Character.toChars((int)Long.parseLong(unicode32, 16))); + } + catch (IllegalArgumentException ignored) { + } + } + str = s; + } + else if (raw) { + str = wholeMatch; + } + else { + final String toReplace = escapeRegexGroup(escMatcher, EscapeRegexGroup.ESCAPED_SUBSTRING); + str = escapeMap.get(toReplace); + } + + if (str != null) { + final TextRange wholeMatchRange = TextRange.create(escMatcher.start(), escMatcher.end()); + result.add(Pair.create(wholeMatchRange.shiftRight(offset), str)); + } + + index = escMatcher.end(); + } + final TextRange range = TextRange.create(index, encoded.length()); + final TextRange offRange = range.shiftRight(offset); + result.add(Pair.create(offRange, range.substring(encoded))); + return result; + } + + @Nullable + private static String escapeRegexGroup(@NotNull Matcher matcher, EscapeRegexGroup group) { + return matcher.group(group.ordinal()); + } + + private boolean isUnicodeByDefault() { + if (!LanguageLevel.forElement(myNode).isPython2()) { + return true; + } + final PsiFile file = myNode.getContainingFile(); + if (file instanceof PyFile) { + final PyFile pyFile = (PyFile)file; + return pyFile.hasImportFromFuture(FutureFeature.UNICODE_LITERALS); + } + return false; + } +} diff --git a/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java b/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java index 959e1433f27c..f03c493d473d 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java @@ -45,50 +45,19 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import javax.swing.*; -import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import java.util.List; public class PyStringLiteralExpressionImpl extends PyElementImpl implements PyStringLiteralExpression, RegExpLanguageHost, PsiLiteralValue { private static final Logger LOG = Logger.getInstance(PyStringLiteralExpressionImpl.class); - public static final Pattern PATTERN_ESCAPE = Pattern - .compile("\\\\(\n|\\\\|'|\"|a|b|f|n|r|t|v|([0-7]{1,3})|x([0-9a-fA-F]{1,2})" + "|N(\\{.*?\\})|u([0-9a-fA-F]{4})|U([0-9a-fA-F]{8}))"); - // -> 1 -> 2 <--> 3 <- -> 4 <--> 5 <- -> 6 <-<- - - private enum EscapeRegexGroup { - WHOLE_MATCH, - ESCAPED_SUBSTRING, - OCTAL, - HEXADECIMAL, - UNICODE_NAMED, - UNICODE_16BIT, - UNICODE_32BIT - } - - private static final Map escapeMap = initializeEscapeMap(); @Nullable private volatile String myStringValue; @Nullable private volatile List myValueTextRanges; @Nullable private volatile List> myDecodedFragments; private final DefaultRegExpPropertiesProvider myPropertiesProvider; - @NotNull - private static Map initializeEscapeMap() { - Map map = new HashMap<>(); - map.put("\n", "\n"); - map.put("\\", "\\"); - map.put("'", "'"); - map.put("\"", "\""); - map.put("a", "\001"); - map.put("b", "\b"); - map.put("f", "\f"); - map.put("n", "\n"); - map.put("r", "\r"); - map.put("t", "\t"); - map.put("v", "\013"); - return map; - } - public PyStringLiteralExpressionImpl(ASTNode astNode) { super(astNode); myPropertiesProvider = DefaultRegExpPropertiesProvider.getInstance(); @@ -112,13 +81,13 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt public List getStringValueTextRanges() { List result = myValueTextRanges; if (result == null) { - int elStart = getTextRange().getStartOffset(); - List ranges = new ArrayList<>(); - for (ASTNode node : getStringNodes()) { - TextRange range = getNodeTextRange(node.getText()); - int nodeOffset = node.getStartOffset() - elStart; - ranges.add(TextRange.from(nodeOffset + range.getStartOffset(), range.getLength())); - } + final int elementStart = getTextRange().getStartOffset(); + final List ranges = StreamEx.of(getGluedStringNodes()) + .map(node -> { + final int nodeRelativeOffset = node.getTextRange().getStartOffset() - elementStart; + return node.getContentRange().shiftRight(nodeRelativeOffset); + }) + .toList(); myValueTextRanges = result = Collections.unmodifiableList(ranges); } return result; @@ -146,36 +115,20 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt return PyStringLiteralUtil.getPrefixEndOffset(text, 0); } - private boolean isUnicodeByDefault() { - if (!LanguageLevel.forElement(this).isPython2()) { - return true; - } - final PsiFile file = getContainingFile(); - if (file instanceof PyFile) { - final PyFile pyFile = (PyFile)file; - return pyFile.hasImportFromFuture(FutureFeature.UNICODE_LITERALS); - } - return false; - } - @Override @NotNull public List> getDecodedFragments() { + final int elementStart = getTextRange().getStartOffset(); List> result = myDecodedFragments; if (result == null) { - result = new ArrayList<>(); - final int elementStart = getTextRange().getStartOffset(); - final boolean unicodeByDefault = isUnicodeByDefault(); - for (ASTNode node : getStringNodes()) { - final String text = node.getText(); - final TextRange textRange = getNodeTextRange(text); - final int offset = node.getTextRange().getStartOffset() - elementStart + textRange.getStartOffset(); - final String encoded = textRange.substring(text); - final boolean hasRawPrefix = PyStringLiteralUtil.isRawPrefix(PyStringLiteralUtil.getPrefix(text)); - final boolean hasUnicodePrefix = PyStringLiteralUtil.isUnicodePrefix(PyStringLiteralUtil.getPrefix(text)); - result.addAll(getDecodedFragments(encoded, offset, hasRawPrefix, unicodeByDefault || hasUnicodePrefix)); - } - myDecodedFragments = result; + final List> combined = StreamEx.of(getGluedStringNodes()) + .flatMap(node -> StreamEx.of(node.getDecodedFragments()) + .map(pair -> { + final int nodeRelativeOffset = node.getTextRange().getStartOffset() - elementStart; + return Pair.create(pair.getFirst().shiftRight(nodeRelativeOffset), pair.getSecond()); + })) + .toList(); + myDecodedFragments = result = Collections.unmodifiableList(combined); } return result; } @@ -186,74 +139,6 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt return stringNodes.size() == 1 && stringNodes.get(0).getElementType() == PyTokenTypes.DOCSTRING; } - @NotNull - private static List> getDecodedFragments(@NotNull String encoded, int offset, boolean raw, boolean unicode) { - final List> result = new ArrayList<>(); - final Matcher escMatcher = PATTERN_ESCAPE.matcher(encoded); - int index = 0; - while (escMatcher.find(index)) { - if (index < escMatcher.start()) { - final TextRange range = TextRange.create(index, escMatcher.start()); - final TextRange offsetRange = range.shiftRight(offset); - result.add(Pair.create(offsetRange, range.substring(encoded))); - } - - final String octal = escapeRegexGroup(escMatcher, EscapeRegexGroup.OCTAL); - final String hex = escapeRegexGroup(escMatcher, EscapeRegexGroup.HEXADECIMAL); - // TODO: Implement unicode character name escapes: EscapeRegexGroup.UNICODE_NAMED - final String unicode16 = escapeRegexGroup(escMatcher, EscapeRegexGroup.UNICODE_16BIT); - final String unicode32 = escapeRegexGroup(escMatcher, EscapeRegexGroup.UNICODE_32BIT); - final String wholeMatch = escapeRegexGroup(escMatcher, EscapeRegexGroup.WHOLE_MATCH); - - final boolean escapedUnicode = raw && unicode || !raw; - - final String str; - if (!raw && octal != null) { - str = new String(new char[]{(char)Integer.parseInt(octal, 8)}); - } - else if (!raw && hex != null) { - str = new String(new char[]{(char)Integer.parseInt(hex, 16)}); - } - else if (escapedUnicode && unicode16 != null) { - str = unicode ? new String(new char[]{(char)Integer.parseInt(unicode16, 16)}) : wholeMatch; - } - else if (escapedUnicode && unicode32 != null) { - String s = wholeMatch; - if (unicode) { - try { - s = new String(Character.toChars((int)Long.parseLong(unicode32, 16))); - } - catch (IllegalArgumentException ignored) { - } - } - str = s; - } - else if (raw) { - str = wholeMatch; - } - else { - final String toReplace = escapeRegexGroup(escMatcher, EscapeRegexGroup.ESCAPED_SUBSTRING); - str = escapeMap.get(toReplace); - } - - if (str != null) { - final TextRange wholeMatchRange = TextRange.create(escMatcher.start(), escMatcher.end()); - result.add(Pair.create(wholeMatchRange.shiftRight(offset), str)); - } - - index = escMatcher.end(); - } - final TextRange range = TextRange.create(index, encoded.length()); - final TextRange offRange = range.shiftRight(offset); - result.add(Pair.create(offRange, range.substring(encoded))); - return result; - } - - @Nullable - private static String escapeRegexGroup(@NotNull Matcher matcher, EscapeRegexGroup group) { - return matcher.group(group.ordinal()); - } - @Override @NotNull public List getStringNodes() { @@ -321,7 +206,8 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt PyFile file = PsiTreeUtil.getParentOfType(this, PyFile.class); if (file != null) { IElementType type = PythonHighlightingLexer.convertStringType(getStringNodes().get(0).getElementType(), text, - LanguageLevel.forElement(this), file.hasImportFromFuture(FutureFeature.UNICODE_LITERALS)); + LanguageLevel.forElement(this), + file.hasImportFromFuture(FutureFeature.UNICODE_LITERALS)); if (PyTokenTypes.UNICODE_NODES.contains(type)) { return PyBuiltinCache.getInstance(this).getUnicodeType(LanguageLevel.forElement(this)); }