MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

PY-31442 Methods of StringLiteralExpression delegate to those of RichStringNode

Browse Source 
First of all, getDecodedFragments() and getStringLiteralTextRanges().
Moreover, all the logic related to unescaping of string literal content is moved to
the dedicated class PyStringLiteralDecoder.
This commit is contained in:
Mikhail Golubev
2018-10-01 12:46:41 +03:00
parent f335699107
commit d74cfa71f5
5 changed files with 196 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/psi-api/src/com/jetbrains/python/psi/PyRichStringNode.java
+5
View File
@@ -1,9 +1,11 @@
package com.jetbrains.python.psi;
import com.intellij.openapi.util.Pair;
import com.intellij.openapi.util.TextRange;
import com.intellij.psi.PsiElement;
import org.jetbrains.annotations.NotNull;
import java.util.List;
import java.util.Set;
/**
@@ -51,6 +53,9 @@ public interface PyRichStringNode extends PsiElement {
@NotNull
String getContent();
@NotNull
List<Pair<TextRange, String>> getDecodedFragments();
@NotNull
String getQuote();
python/src/com/jetbrains/python/inspections/PyStringFormatParser.java
+2 -2
View File
@@ -21,7 +21,7 @@ import com.intellij.psi.PsiElement;
import com.intellij.util.ObjectUtils;
import com.jetbrains.python.PyNames;
import com.jetbrains.python.psi.*;
import com.jetbrains.python.psi.impl.PyStringLiteralExpressionImpl;
import com.jetbrains.python.psi.impl.PyStringLiteralDecoder;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -625,7 +625,7 @@ public class PyStringFormatParser {
@NotNull
public static List<TextRange> getEscapeRanges(@NotNull String s) {
final List<TextRange> ranges = new ArrayList<>();
Matcher matcher = PyStringLiteralExpressionImpl.PATTERN_ESCAPE.matcher(s);
Matcher matcher = PyStringLiteralDecoder.PATTERN_ESCAPE.matcher(s);
while (matcher.find()) {
ranges.add(TextRange.create(matcher.start(), matcher.end()));
}
python/src/com/jetbrains/python/psi/impl/PyLiteralStringNodeImpl.java
+10 -5
View File
@@ -10,6 +10,7 @@ import com.jetbrains.python.psi.PyStringLiteralUtil;
import org.jetbrains.annotations.NotNull;
import java.util.EnumSet;
import java.util.List;
import java.util.Set;
/**
@@ -37,6 +38,14 @@ public class PyLiteralStringNodeImpl extends LeafPsiElement implements PyLiteral
return getContentRange().substring(getText());
}
@NotNull
@Override
public List<Pair<TextRange, String>> getDecodedFragments() {
final PyStringLiteralDecoder decoder = new PyStringLiteralDecoder(this);
decoder.decodeContent();
return decoder.getResult();
}
@NotNull
@Override
public String getTextWithoutPrefix() {
@@ -46,11 +55,7 @@ public class PyLiteralStringNodeImpl extends LeafPsiElement implements PyLiteral
@NotNull
@Override
public TextRange getContentRange() {
final Pair<String, String> quotes = PyStringLiteralUtil.getQuotes(getText());
assert quotes != null;
final String prefixWithOpeningQuote = quotes.getFirst();
final String closingQuote = quotes.getSecond();
return TextRange.create(prefixWithOpeningQuote.length(), getTextLength() - closingQuote.length());
return PyStringLiteralExpressionImpl.getNodeTextRange(getText());
}
@NotNull
python/src/com/jetbrains/python/psi/impl/PyStringLiteralDecoder.java
+157
View File
@@ -0,0 +1,157 @@
// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.jetbrains.python.psi.impl;
import com.intellij.openapi.util.Pair;
import com.intellij.openapi.util.TextRange;
import com.intellij.psi.PsiFile;
import com.jetbrains.python.psi.FutureFeature;
import com.jetbrains.python.psi.LanguageLevel;
import com.jetbrains.python.psi.PyFile;
import com.jetbrains.python.psi.PyRichStringNode;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class PyStringLiteralDecoder {
public static final Pattern PATTERN_ESCAPE = Pattern
.compile("\\\\(\n|\\\\|'|\"|a|b|f|n|r|t|v|([0-7]{1,3})|x([0-9a-fA-F]{1,2})" + "|N(\\{.*?\\})|u([0-9a-fA-F]{4})|U([0-9a-fA-F]{8}))");
// -> 1 -> 2 <--> 3 <- -> 4 <--> 5 <- -> 6 <-<-
private enum EscapeRegexGroup {
WHOLE_MATCH,
ESCAPED_SUBSTRING,
OCTAL,
HEXADECIMAL,
UNICODE_NAMED,
UNICODE_16BIT,
UNICODE_32BIT
}
private static final Map<String, String> escapeMap = initializeEscapeMap();
@NotNull
private static Map<String, String> initializeEscapeMap() {
Map<String, String> map = new HashMap<>();
map.put("\n", "\n");
map.put("\\", "\\");
map.put("'", "'");
map.put("\"", "\"");
map.put("a", "\001");
map.put("b", "\b");
map.put("f", "\f");
map.put("n", "\n");
map.put("r", "\r");
map.put("t", "\t");
map.put("v", "\013");
return map;
}
private final PyRichStringNode myNode;
private final List<Pair<TextRange, String>> myResult = new ArrayList<>();
public PyStringLiteralDecoder(@NotNull PyRichStringNode node) {
myNode = node;
}
public void decodeContent() {
decodeRange(myNode.getContentRange());
}
public void decodeRange(@NotNull TextRange range) {
myResult.addAll(decodeFragment(range.substring(myNode.getText()), range.getStartOffset()));
}
@NotNull
public List<Pair<TextRange, String>> getResult() {
return myResult;
}
@NotNull
private List<Pair<TextRange, String>> decodeFragment(@NotNull String encoded, int offset) {
final boolean raw = myNode.isRaw();
final boolean unicode = myNode.isUnicode() || isUnicodeByDefault();
final List<Pair<TextRange, String>> result = new ArrayList<>();
final Matcher escMatcher = PATTERN_ESCAPE.matcher(encoded);
int index = 0;
while (escMatcher.find(index)) {
if (index < escMatcher.start()) {
final TextRange range = TextRange.create(index, escMatcher.start());
final TextRange offsetRange = range.shiftRight(offset);
result.add(Pair.create(offsetRange, range.substring(encoded)));
}
final String octal = escapeRegexGroup(escMatcher, EscapeRegexGroup.OCTAL);
final String hex = escapeRegexGroup(escMatcher, EscapeRegexGroup.HEXADECIMAL);
// TODO: Implement unicode character name escapes: EscapeRegexGroup.UNICODE_NAMED
final String unicode16 = escapeRegexGroup(escMatcher, EscapeRegexGroup.UNICODE_16BIT);
final String unicode32 = escapeRegexGroup(escMatcher, EscapeRegexGroup.UNICODE_32BIT);
final String wholeMatch = escapeRegexGroup(escMatcher, EscapeRegexGroup.WHOLE_MATCH);
final boolean escapedUnicode = raw && unicode || !raw;
final String str;
if (!raw && octal != null) {
str = new String(new char[]{(char)Integer.parseInt(octal, 8)});
}
else if (!raw && hex != null) {
str = new String(new char[]{(char)Integer.parseInt(hex, 16)});
}
else if (escapedUnicode && unicode16 != null) {
str = unicode ? new String(new char[]{(char)Integer.parseInt(unicode16, 16)}) : wholeMatch;
}
else if (escapedUnicode && unicode32 != null) {
String s = wholeMatch;
if (unicode) {
try {
s = new String(Character.toChars((int)Long.parseLong(unicode32, 16)));
}
catch (IllegalArgumentException ignored) {
}
}
str = s;
}
else if (raw) {
str = wholeMatch;
}
else {
final String toReplace = escapeRegexGroup(escMatcher, EscapeRegexGroup.ESCAPED_SUBSTRING);
str = escapeMap.get(toReplace);
}
if (str != null) {
final TextRange wholeMatchRange = TextRange.create(escMatcher.start(), escMatcher.end());
result.add(Pair.create(wholeMatchRange.shiftRight(offset), str));
}
index = escMatcher.end();
}
final TextRange range = TextRange.create(index, encoded.length());
final TextRange offRange = range.shiftRight(offset);
result.add(Pair.create(offRange, range.substring(encoded)));
return result;
}
@Nullable
private static String escapeRegexGroup(@NotNull Matcher matcher, EscapeRegexGroup group) {
return matcher.group(group.ordinal());
}
private boolean isUnicodeByDefault() {
if (!LanguageLevel.forElement(myNode).isPython2()) {
return true;
}
final PsiFile file = myNode.getContainingFile();
if (file instanceof PyFile) {
final PyFile pyFile = (PyFile)file;
return pyFile.hasImportFromFuture(FutureFeature.UNICODE_LITERALS);
}
return false;
}
}
python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java
+22 -136
View File
@@ -45,50 +45,19 @@ import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import javax.swing.*;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.Arrays;
import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
public class PyStringLiteralExpressionImpl extends PyElementImpl implements PyStringLiteralExpression, RegExpLanguageHost, PsiLiteralValue {
private static final Logger LOG = Logger.getInstance(PyStringLiteralExpressionImpl.class);
public static final Pattern PATTERN_ESCAPE = Pattern
.compile("\\\\(\n|\\\\|'|\"|a|b|f|n|r|t|v|([0-7]{1,3})|x([0-9a-fA-F]{1,2})" + "|N(\\{.*?\\})|u([0-9a-fA-F]{4})|U([0-9a-fA-F]{8}))");
// -> 1 -> 2 <--> 3 <- -> 4 <--> 5 <- -> 6 <-<-
private enum EscapeRegexGroup {
WHOLE_MATCH,
ESCAPED_SUBSTRING,
OCTAL,
HEXADECIMAL,
UNICODE_NAMED,
UNICODE_16BIT,
UNICODE_32BIT
}
private static final Map<String, String> escapeMap = initializeEscapeMap();
@Nullable private volatile String myStringValue;
@Nullable private volatile List<TextRange> myValueTextRanges;
@Nullable private volatile List<Pair<TextRange, String>> myDecodedFragments;
private final DefaultRegExpPropertiesProvider myPropertiesProvider;
@NotNull
private static Map<String, String> initializeEscapeMap() {
Map<String, String> map = new HashMap<>();
map.put("\n", "\n");
map.put("\\", "\\");
map.put("'", "'");
map.put("\"", "\"");
map.put("a", "\001");
map.put("b", "\b");
map.put("f", "\f");
map.put("n", "\n");
map.put("r", "\r");
map.put("t", "\t");
map.put("v", "\013");
return map;
}
public PyStringLiteralExpressionImpl(ASTNode astNode) {
super(astNode);
myPropertiesProvider = DefaultRegExpPropertiesProvider.getInstance();
@@ -112,13 +81,13 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt
public List<TextRange> getStringValueTextRanges() {
List<TextRange> result = myValueTextRanges;
if (result == null) {
int elStart = getTextRange().getStartOffset();
List<TextRange> ranges = new ArrayList<>();
for (ASTNode node : getStringNodes()) {
TextRange range = getNodeTextRange(node.getText());
int nodeOffset = node.getStartOffset() - elStart;
ranges.add(TextRange.from(nodeOffset + range.getStartOffset(), range.getLength()));
}
final int elementStart = getTextRange().getStartOffset();
final List<TextRange> ranges = StreamEx.of(getGluedStringNodes())
.map(node -> {
final int nodeRelativeOffset = node.getTextRange().getStartOffset() - elementStart;
return node.getContentRange().shiftRight(nodeRelativeOffset);
})
.toList();
myValueTextRanges = result = Collections.unmodifiableList(ranges);
}
return result;
@@ -146,36 +115,20 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt
return PyStringLiteralUtil.getPrefixEndOffset(text, 0);
}
private boolean isUnicodeByDefault() {
if (!LanguageLevel.forElement(this).isPython2()) {
return true;
}
final PsiFile file = getContainingFile();
if (file instanceof PyFile) {
final PyFile pyFile = (PyFile)file;
return pyFile.hasImportFromFuture(FutureFeature.UNICODE_LITERALS);
}
return false;
}
@Override
@NotNull
public List<Pair<TextRange, String>> getDecodedFragments() {
final int elementStart = getTextRange().getStartOffset();
List<Pair<TextRange, String>> result = myDecodedFragments;
if (result == null) {
result = new ArrayList<>();
final int elementStart = getTextRange().getStartOffset();
final boolean unicodeByDefault = isUnicodeByDefault();
for (ASTNode node : getStringNodes()) {
final String text = node.getText();
final TextRange textRange = getNodeTextRange(text);
final int offset = node.getTextRange().getStartOffset() - elementStart + textRange.getStartOffset();
final String encoded = textRange.substring(text);
final boolean hasRawPrefix = PyStringLiteralUtil.isRawPrefix(PyStringLiteralUtil.getPrefix(text));
final boolean hasUnicodePrefix = PyStringLiteralUtil.isUnicodePrefix(PyStringLiteralUtil.getPrefix(text));
result.addAll(getDecodedFragments(encoded, offset, hasRawPrefix, unicodeByDefault || hasUnicodePrefix));
}
myDecodedFragments = result;
final List<Pair<TextRange, String>> combined = StreamEx.of(getGluedStringNodes())
.flatMap(node -> StreamEx.of(node.getDecodedFragments())
.map(pair -> {
final int nodeRelativeOffset = node.getTextRange().getStartOffset() - elementStart;
return Pair.create(pair.getFirst().shiftRight(nodeRelativeOffset), pair.getSecond());
}))
.toList();
myDecodedFragments = result = Collections.unmodifiableList(combined);
}
return result;
}
@@ -186,74 +139,6 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt
return stringNodes.size() == 1 && stringNodes.get(0).getElementType() == PyTokenTypes.DOCSTRING;
}
@NotNull
private static List<Pair<TextRange, String>> getDecodedFragments(@NotNull String encoded, int offset, boolean raw, boolean unicode) {
final List<Pair<TextRange, String>> result = new ArrayList<>();
final Matcher escMatcher = PATTERN_ESCAPE.matcher(encoded);
int index = 0;
while (escMatcher.find(index)) {
if (index < escMatcher.start()) {
final TextRange range = TextRange.create(index, escMatcher.start());
final TextRange offsetRange = range.shiftRight(offset);
result.add(Pair.create(offsetRange, range.substring(encoded)));
}
final String octal = escapeRegexGroup(escMatcher, EscapeRegexGroup.OCTAL);
final String hex = escapeRegexGroup(escMatcher, EscapeRegexGroup.HEXADECIMAL);
// TODO: Implement unicode character name escapes: EscapeRegexGroup.UNICODE_NAMED
final String unicode16 = escapeRegexGroup(escMatcher, EscapeRegexGroup.UNICODE_16BIT);
final String unicode32 = escapeRegexGroup(escMatcher, EscapeRegexGroup.UNICODE_32BIT);
final String wholeMatch = escapeRegexGroup(escMatcher, EscapeRegexGroup.WHOLE_MATCH);
final boolean escapedUnicode = raw && unicode || !raw;
final String str;
if (!raw && octal != null) {
str = new String(new char[]{(char)Integer.parseInt(octal, 8)});
}
else if (!raw && hex != null) {
str = new String(new char[]{(char)Integer.parseInt(hex, 16)});
}
else if (escapedUnicode && unicode16 != null) {
str = unicode ? new String(new char[]{(char)Integer.parseInt(unicode16, 16)}) : wholeMatch;
}
else if (escapedUnicode && unicode32 != null) {
String s = wholeMatch;
if (unicode) {
try {
s = new String(Character.toChars((int)Long.parseLong(unicode32, 16)));
}
catch (IllegalArgumentException ignored) {
}
}
str = s;
}
else if (raw) {
str = wholeMatch;
}
else {
final String toReplace = escapeRegexGroup(escMatcher, EscapeRegexGroup.ESCAPED_SUBSTRING);
str = escapeMap.get(toReplace);
}
if (str != null) {
final TextRange wholeMatchRange = TextRange.create(escMatcher.start(), escMatcher.end());
result.add(Pair.create(wholeMatchRange.shiftRight(offset), str));
}
index = escMatcher.end();
}
final TextRange range = TextRange.create(index, encoded.length());
final TextRange offRange = range.shiftRight(offset);
result.add(Pair.create(offRange, range.substring(encoded)));
return result;
}
@Nullable
private static String escapeRegexGroup(@NotNull Matcher matcher, EscapeRegexGroup group) {
return matcher.group(group.ordinal());
}
@Override
@NotNull
public List<ASTNode> getStringNodes() {
@@ -321,7 +206,8 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt
PyFile file = PsiTreeUtil.getParentOfType(this, PyFile.class);
if (file != null) {
IElementType type = PythonHighlightingLexer.convertStringType(getStringNodes().get(0).getElementType(), text,
LanguageLevel.forElement(this), file.hasImportFromFuture(FutureFeature.UNICODE_LITERALS));
LanguageLevel.forElement(this),
file.hasImportFromFuture(FutureFeature.UNICODE_LITERALS));
if (PyTokenTypes.UNICODE_NODES.contains(type)) {
return PyBuiltinCache.getInstance(this).getUnicodeType(LanguageLevel.forElement(this));
}