From 4a2ef4b3d079ea569812aa3c351509a1589ef677 Mon Sep 17 00:00:00 2001 From: Mikhail Golubev Date: Mon, 10 Oct 2016 17:28:24 +0300 Subject: [PATCH] Extracted methods from PythonStringUtil into dedicated util class PyStringLiteralUtil --- .../python/PyStringLiteralReference.java | 3 +- .../python/PythonFoldingBuilder.java | 2 +- .../jetbrains/python/PythonStringUtil.java | 205 +---------------- .../actions/PyFillParagraphHandler.java | 10 +- .../moveUpDown/PyStatementMover.java | 3 +- .../ConvertVariadicParamIntention.java | 5 +- ...yStringConcatenationToFormatIntention.java | 3 +- .../DocStringParameterReference.java | 3 +- .../doctest/PyDocstringLanguageInjector.java | 4 +- .../python/editor/BaseQuoteHandler.java | 6 +- .../python/lexer/PyStringLiteralLexer.java | 8 +- .../python/psi/PyStringLiteralUtil.java | 214 ++++++++++++++++++ .../src/com/jetbrains/python/psi/PyUtil.java | 9 +- .../psi/impl/PyElementGeneratorImpl.java | 3 +- .../python/psi/impl/PyNamedParameterImpl.java | 4 +- .../impl/PyStringLiteralExpressionImpl.java | 7 +- .../PyStringLiteralExpressionManipulator.java | 6 +- .../refactoring/PyReplaceExpressionUtil.java | 4 +- .../introduce/IntroduceHandler.java | 3 +- .../PythonSpellcheckerStrategy.java | 4 +- 20 files changed, 255 insertions(+), 251 deletions(-) create mode 100644 python/src/com/jetbrains/python/psi/PyStringLiteralUtil.java diff --git a/python/src/com/jetbrains/python/PyStringLiteralReference.java b/python/src/com/jetbrains/python/PyStringLiteralReference.java index 46fd61466ac9..524197817f0f 100644 --- a/python/src/com/jetbrains/python/PyStringLiteralReference.java +++ b/python/src/com/jetbrains/python/PyStringLiteralReference.java @@ -18,6 +18,7 @@ package com.jetbrains.python; import com.intellij.openapi.util.TextRange; import com.intellij.psi.PsiElement; import com.jetbrains.python.psi.PyElementGenerator; +import com.jetbrains.python.psi.PyStringLiteralUtil; import com.jetbrains.python.psi.StringLiteralExpression; import org.jetbrains.annotations.NotNull; @@ -38,7 +39,7 @@ public abstract class PyStringLiteralReference extends BaseReference { @SuppressWarnings("RefusedBequest") // 1 instead of 1 in range and "-1" at the end because we do not need quotes @Override public final TextRange getRangeInElement() { - return PythonStringUtil.getTextRange(myElement); + return PyStringLiteralUtil.getTextRange(myElement); } @Override diff --git a/python/src/com/jetbrains/python/PythonFoldingBuilder.java b/python/src/com/jetbrains/python/PythonFoldingBuilder.java index f3cffaabf67e..7978fd5deead 100644 --- a/python/src/com/jetbrains/python/PythonFoldingBuilder.java +++ b/python/src/com/jetbrains/python/PythonFoldingBuilder.java @@ -223,7 +223,7 @@ public class PythonFoldingBuilder extends CustomFoldingBuilder implements DumbAw private static String getLanguagePlaceholderForString(PyStringLiteralExpression stringLiteralExpression) { String stringText = stringLiteralExpression.getText(); - Pair quotes = PythonStringUtil.getQuotes(stringText); + Pair quotes = PyStringLiteralUtil.getQuotes(stringText); if (quotes != null) { return quotes.second + "..." + quotes.second; } diff --git a/python/src/com/jetbrains/python/PythonStringUtil.java b/python/src/com/jetbrains/python/PythonStringUtil.java index 3f0df093112d..0b4ee0e2540a 100644 --- a/python/src/com/jetbrains/python/PythonStringUtil.java +++ b/python/src/com/jetbrains/python/PythonStringUtil.java @@ -15,18 +15,15 @@ */ package com.jetbrains.python; -import com.google.common.collect.ImmutableList; import com.intellij.openapi.util.Pair; import com.intellij.openapi.util.SystemInfo; import com.intellij.openapi.util.TextRange; import com.intellij.openapi.util.io.FileUtil; import com.intellij.openapi.util.text.StringUtil; -import com.intellij.psi.PsiElement; import com.intellij.psi.util.QualifiedName; import com.intellij.util.ObjectUtils; import com.intellij.util.PathUtil; -import com.jetbrains.python.psi.PyExpression; -import com.jetbrains.python.psi.PyStringLiteralExpression; +import com.jetbrains.python.psi.PyStringLiteralUtil; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -34,49 +31,15 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -import static com.intellij.openapi.util.text.StringUtil.endsWith; -import static com.intellij.openapi.util.text.StringUtil.startsWith; - /** * @author Alexei Orischenko * @author vlan */ public class PythonStringUtil { - private static final ImmutableList QUOTES = ImmutableList.of("'''", "\"\"\"", "'", "\""); - /** - * Valid string prefix characters (lowercased) as defined in Python lexer. - */ - public static final String PREFIX_CHARACTERS = "ubcrf"; - - /** - * Maximum length of a string prefix as defined in Python lexer. - */ - public static final int MAX_PREFIX_LENGTH = 3; private PythonStringUtil() { } - /** - * 'text' => text - * "text" => text - * text => text - * "text => "text - * - * @return string without heading and trailing pair of ' or " - */ - @NotNull - public static String getStringValue(@NotNull String s) { - return getStringValueTextRange(s).substring(s); - } - - - public static TextRange getStringValueTextRange(@NotNull String s) { - final Pair quotes = getQuotes(s); - if (quotes != null) { - return TextRange.create(quotes.getFirst().length(), s.length() - quotes.getSecond().length()); - } - return TextRange.allOf(s); - } @NotNull public static String removeFirstPrefix(@Nullable String s, String separator) { @@ -159,9 +122,9 @@ public class PythonStringUtil { public static String replaceLastSuffix(String s, String separator, String newElementName) { Pair quotes = null; - if (isQuoted(s)) { - quotes = getQuotes(s); - s = stripQuotesAroundValue(s); + if (PyStringLiteralUtil.isQuoted(s)) { + quotes = PyStringLiteralUtil.getQuotes(s); + s = PyStringLiteralUtil.stripQuotesAroundValue(s); } s = removeLastSuffix(s, separator); @@ -184,112 +147,6 @@ public class PythonStringUtil { } - /** - * Handles unicode and raw strings - * - * @param text - * @return false if no quotes found, true otherwise - * sdfs -> false - * ur'x' -> true - * "string" -> true - */ - - public static boolean isQuoted(@Nullable String text) { - return text != null && getQuotes(text) != null; - } - - /** - * Handles unicode and raw strings - * - * @param text - * @return open and close quote (including raw/unicode prefixes), null if no quotes present in string - * 'string' -> (', ') - * UR"unicode raw string" -> (UR", ") - */ - @Nullable - public static Pair getQuotes(@NotNull final String text) { - final String prefix = getPrefix(text); - final String mainText = text.substring(prefix.length()); - for (String quote : QUOTES) { - final Pair quotes = getQuotes(mainText, prefix, quote); - if (quotes != null) { - return quotes; - } - } - return null; - } - - /** - * Finds the end offset of the string prefix starting from {@code startOffset} in the given char sequence. - * String prefix may contain only up to {@link #MAX_PREFIX_LENGTH} characters from {@link #PREFIX_CHARACTERS} - * (case insensitively). - * - * @return end offset of found string prefix - */ - public static int getPrefixEndOffset(@NotNull CharSequence text, int startOffset) { - int offset; - for (offset = startOffset; offset < Math.min(startOffset + MAX_PREFIX_LENGTH, text.length()); offset++) { - if (PREFIX_CHARACTERS.indexOf(Character.toLowerCase(text.charAt(offset))) < 0) { - break; - } - } - return offset; - } - - @NotNull - public static String getPrefix(@NotNull CharSequence text) { - return getPrefix(text, 0); - } - - /** - * Extracts string prefix from the given char sequence using {@link #getPrefixEndOffset(CharSequence, int)}. - * - * @return extracted string prefix - * @see #getPrefixEndOffset(CharSequence, int) - */ - @NotNull - public static String getPrefix(@NotNull CharSequence text, int startOffset) { - return text.subSequence(startOffset, getPrefixEndOffset(text, startOffset)).toString(); - } - - /** - * @return whether the given prefix contains either 'u' or 'U' character - */ - public static boolean isUnicodePrefix(@NotNull String prefix) { - return StringUtil.indexOfIgnoreCase(prefix, 'u', 0) >= 0; - } - - /** - * @return whether the given prefix contains either 'b' or 'B' character - */ - public static boolean isBytesPrefix(@NotNull String prefix) { - return StringUtil.indexOfIgnoreCase(prefix, 'b', 0) >= 0; - } - - /** - * @return whether the given prefix contains either 'r' or 'R' character - */ - public static boolean isRawPrefix(@NotNull String prefix) { - return StringUtil.indexOfIgnoreCase(prefix, 'r', 0) >= 0; - } - - /** - * @return whether the given prefix contains either 'f' or 'F' character - */ - public static boolean isFormattedPrefix(@NotNull String prefix) { - return StringUtil.indexOfIgnoreCase(prefix, 'f', 0) >= 0; - } - - @Nullable - private static Pair getQuotes(@NotNull String text, @NotNull String prefix, @NotNull String quote) { - final int length = text.length(); - final int n = quote.length(); - if (length >= 2 * n && text.startsWith(quote) && text.endsWith(quote)) { - return Pair.create(prefix + text.substring(0, n), text.substring(length - n)); - } - return null; - } - @Nullable public static String intersect(String fullName, String elementStringValue) { QualifiedName fullQName = QualifiedName.fromDottedString(fullName); @@ -323,58 +180,4 @@ public class PythonStringUtil { return null; } - - public static TextRange getTextRange(PsiElement element) { - if (element instanceof PyStringLiteralExpression) { - final List ranges = ((PyStringLiteralExpression)element).getStringValueTextRanges(); - return ranges.get(0); - } - else { - return new TextRange(0, element.getTextLength()); - } - } - - @Nullable - public static String getText(@Nullable PyExpression ex) { - if (ex == null) { - return null; - } - else { - return ex.getText(); - } - } - - @Nullable - public static String getStringValue(@Nullable PsiElement o) { - if (o == null) { - return null; - } - if (o instanceof PyStringLiteralExpression) { - PyStringLiteralExpression literalExpression = (PyStringLiteralExpression)o; - return literalExpression.getStringValue(); - } - else { - return o.getText(); - } - } - - public static String stripQuotesAroundValue(String text) { - Pair quotes = getQuotes(text); - if (quotes == null) { - return text; - } - - return text.substring(quotes.first.length(), text.length() - quotes.second.length()); - } - - public static boolean isRawString(String text) { - text = text.toLowerCase(); - text = StringUtil.trimStart(text, "u"); - return isStringPrefixedBy(text.toLowerCase(), "r"); - } - - - private static boolean isStringPrefixedBy(String text, String prefix) { - return (startsWith(text, prefix + "\"") && endsWith(text, "\"")) || (startsWith(text, prefix + "\'") && endsWith(text, "\'")); - } } diff --git a/python/src/com/jetbrains/python/actions/PyFillParagraphHandler.java b/python/src/com/jetbrains/python/actions/PyFillParagraphHandler.java index 53f58865784e..e347361a0ca4 100644 --- a/python/src/com/jetbrains/python/actions/PyFillParagraphHandler.java +++ b/python/src/com/jetbrains/python/actions/PyFillParagraphHandler.java @@ -24,11 +24,7 @@ import com.intellij.psi.PsiElement; import com.intellij.psi.PsiFile; import com.intellij.psi.PsiWhiteSpace; import com.intellij.psi.util.PsiTreeUtil; -import com.jetbrains.python.PythonStringUtil; -import com.jetbrains.python.psi.PyDocStringOwner; -import com.jetbrains.python.psi.PyFile; -import com.jetbrains.python.psi.PyStatementList; -import com.jetbrains.python.psi.PyStringLiteralExpression; +import com.jetbrains.python.psi.*; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -46,7 +42,7 @@ public class PyFillParagraphHandler extends ParagraphFillHandler { if (stringLiteralExpression != null) { final String text = stringLiteralExpression.getText(); final Pair quotes = - PythonStringUtil.getQuotes(text); + PyStringLiteralUtil.getQuotes(text); final PyDocStringOwner docStringOwner = PsiTreeUtil.getParentOfType(stringLiteralExpression, PyDocStringOwner.class); if (docStringOwner != null && stringLiteralExpression.equals(docStringOwner.getDocStringExpression())) { String indent = getIndent(stringLiteralExpression); @@ -90,7 +86,7 @@ public class PyFillParagraphHandler extends ParagraphFillHandler { if (stringLiteralExpression != null) { final String text = stringLiteralExpression.getText(); final Pair quotes = - PythonStringUtil.getQuotes(text); + PyStringLiteralUtil.getQuotes(text); final PyDocStringOwner docStringOwner = PsiTreeUtil.getParentOfType(stringLiteralExpression, PyDocStringOwner.class); if (docStringOwner != null && stringLiteralExpression.equals(docStringOwner.getDocStringExpression())) { String indent = getIndent(stringLiteralExpression); diff --git a/python/src/com/jetbrains/python/codeInsight/editorActions/moveUpDown/PyStatementMover.java b/python/src/com/jetbrains/python/codeInsight/editorActions/moveUpDown/PyStatementMover.java index fba97072271a..f9eb7311a21d 100644 --- a/python/src/com/jetbrains/python/codeInsight/editorActions/moveUpDown/PyStatementMover.java +++ b/python/src/com/jetbrains/python/codeInsight/editorActions/moveUpDown/PyStatementMover.java @@ -30,7 +30,6 @@ import com.intellij.psi.*; import com.intellij.psi.codeStyle.CodeStyleManager; import com.intellij.psi.impl.source.PostprocessReformattingAspect; import com.intellij.psi.util.PsiTreeUtil; -import com.jetbrains.python.PythonStringUtil; import com.jetbrains.python.psi.*; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -86,7 +85,7 @@ public class PyStatementMover extends LineMover { if (nearLine >= document.getLineCount() || nearLine <= 0) return false; final PyStringLiteralExpression stringLiteralExpression = PsiTreeUtil.getParentOfType(elementToMove1, PyStringLiteralExpression.class); if (stringLiteralExpression != null) { - final Pair quotes = PythonStringUtil.getQuotes(stringLiteralExpression.getText()); + final Pair quotes = PyStringLiteralUtil.getQuotes(stringLiteralExpression.getText()); if (quotes != null && (quotes.first.equals("'''") || quotes.first.equals("\"\"\""))) { final String text1 = document.getText(TextRange.create(start, end)).trim(); final String text2 = document.getText(TextRange.create(document.getLineStartOffset(nearLine), document.getLineEndOffset(nearLine))).trim(); diff --git a/python/src/com/jetbrains/python/codeInsight/intentions/ConvertVariadicParamIntention.java b/python/src/com/jetbrains/python/codeInsight/intentions/ConvertVariadicParamIntention.java index 325b102fe267..abbe391d51a4 100644 --- a/python/src/com/jetbrains/python/codeInsight/intentions/ConvertVariadicParamIntention.java +++ b/python/src/com/jetbrains/python/codeInsight/intentions/ConvertVariadicParamIntention.java @@ -26,7 +26,6 @@ import com.intellij.util.IncorrectOperationException; import com.intellij.util.containers.Stack; import com.jetbrains.python.PyBundle; import com.jetbrains.python.PyNames; -import com.jetbrains.python.PythonStringUtil; import com.jetbrains.python.psi.*; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -75,7 +74,7 @@ public class ConvertVariadicParamIntention extends BaseIntentionAction { if (function != null) { for (PyCallExpression call : findKeywordContainerCalls(function)) { final PyExpression firstArgument = ArrayUtil.getFirstElement(call.getArguments()); - final String firstArgumentValue = PythonStringUtil.getStringValue(firstArgument); + final String firstArgumentValue = PyStringLiteralUtil.getStringValue(firstArgument); if (firstArgumentValue == null || !PyNames.isIdentifierString(firstArgumentValue)) { return false; } @@ -83,7 +82,7 @@ public class ConvertVariadicParamIntention extends BaseIntentionAction { for (PySubscriptionExpression subscription : findKeywordContainerSubscriptions(function)) { final PyExpression indexExpression = subscription.getIndexExpression(); - final String indexValue = PythonStringUtil.getStringValue(indexExpression); + final String indexValue = PyStringLiteralUtil.getStringValue(indexExpression); if (indexValue == null || !PyNames.isIdentifierString(indexValue)) { return false; } diff --git a/python/src/com/jetbrains/python/codeInsight/intentions/PyStringConcatenationToFormatIntention.java b/python/src/com/jetbrains/python/codeInsight/intentions/PyStringConcatenationToFormatIntention.java index 04d5765970a1..128301f3ad40 100644 --- a/python/src/com/jetbrains/python/codeInsight/intentions/PyStringConcatenationToFormatIntention.java +++ b/python/src/com/jetbrains/python/codeInsight/intentions/PyStringConcatenationToFormatIntention.java @@ -27,7 +27,6 @@ import com.intellij.util.IncorrectOperationException; import com.intellij.util.NotNullFunction; import com.jetbrains.python.PyBundle; import com.jetbrains.python.PyTokenTypes; -import com.jetbrains.python.PythonStringUtil; import com.jetbrains.python.psi.*; import com.jetbrains.python.psi.impl.PyBuiltinCache; import com.jetbrains.python.psi.types.PyClassTypeImpl; @@ -148,7 +147,7 @@ public class PyStringConcatenationToFormatIntention extends BaseIntentionAction isUnicode = true; } if (!quotesDetected) { - quotes = PythonStringUtil.getQuotes(expression.getText()); + quotes = PyStringLiteralUtil.getQuotes(expression.getText()); quotesDetected = true; } String value = ((PyStringLiteralExpression)expression).getStringValue(); diff --git a/python/src/com/jetbrains/python/documentation/docstrings/DocStringParameterReference.java b/python/src/com/jetbrains/python/documentation/docstrings/DocStringParameterReference.java index b3907b9f1f5d..4ba9232fba72 100644 --- a/python/src/com/jetbrains/python/documentation/docstrings/DocStringParameterReference.java +++ b/python/src/com/jetbrains/python/documentation/docstrings/DocStringParameterReference.java @@ -27,7 +27,6 @@ import com.intellij.util.ArrayUtil; import com.intellij.util.IncorrectOperationException; import com.intellij.util.containers.HashSet; import com.jetbrains.python.PyNames; -import com.jetbrains.python.PythonStringUtil; import com.jetbrains.python.psi.*; import com.jetbrains.python.psi.impl.ParamHelper; import com.jetbrains.python.psi.types.TypeEvalContext; @@ -188,7 +187,7 @@ public class DocStringParameterReference extends PsiReferenceBase quotes = PythonStringUtil.getQuotes(range.substring(myElement.getText())); + Pair quotes = PyStringLiteralUtil.getQuotes(range.substring(myElement.getText())); if (quotes != null) { range = TextRange.create(range.getStartOffset() + quotes.first.length(), range.getEndOffset() - quotes.second.length()); diff --git a/python/src/com/jetbrains/python/documentation/doctest/PyDocstringLanguageInjector.java b/python/src/com/jetbrains/python/documentation/doctest/PyDocstringLanguageInjector.java index 82de48ba2f48..d78512d19dc6 100644 --- a/python/src/com/jetbrains/python/documentation/doctest/PyDocstringLanguageInjector.java +++ b/python/src/com/jetbrains/python/documentation/doctest/PyDocstringLanguageInjector.java @@ -24,10 +24,10 @@ import com.intellij.psi.InjectedLanguagePlaces; import com.intellij.psi.LanguageInjector; import com.intellij.psi.PsiLanguageInjectionHost; import com.intellij.psi.util.PsiTreeUtil; -import com.jetbrains.python.PythonStringUtil; import com.jetbrains.python.documentation.PyDocumentationSettings; import com.jetbrains.python.psi.PyDocStringOwner; import com.jetbrains.python.psi.PyStringLiteralExpression; +import com.jetbrains.python.psi.PyStringLiteralUtil; import org.jetbrains.annotations.NotNull; import java.util.List; @@ -51,7 +51,7 @@ public class PyDocstringLanguageInjector implements LanguageInjector { int end = host.getTextLength() - 1; final String text = host.getText(); - final Pair quotes = PythonStringUtil.getQuotes(text); + final Pair quotes = PyStringLiteralUtil.getQuotes(text); final List strings = StringUtil.split(text, "\n", false); boolean gotExample = false; diff --git a/python/src/com/jetbrains/python/editor/BaseQuoteHandler.java b/python/src/com/jetbrains/python/editor/BaseQuoteHandler.java index 5f3248a5fca1..4924c80ef859 100644 --- a/python/src/com/jetbrains/python/editor/BaseQuoteHandler.java +++ b/python/src/com/jetbrains/python/editor/BaseQuoteHandler.java @@ -22,7 +22,7 @@ import com.intellij.openapi.editor.highlighter.HighlighterIterator; import com.intellij.openapi.util.text.StringUtil; import com.intellij.psi.tree.IElementType; import com.intellij.psi.tree.TokenSet; -import com.jetbrains.python.PythonStringUtil; +import com.jetbrains.python.psi.PyStringLiteralUtil; import org.jetbrains.annotations.Nullable; import java.util.Arrays; @@ -61,7 +61,7 @@ public class BaseQuoteHandler extends SimpleTokenSetQuoteHandler implements Mult } if (myLiteralTokenSet.contains(iterator.getTokenType())) { int start = iterator.getStart(); - if (offset - start <= PythonStringUtil.MAX_PREFIX_LENGTH) { + if (offset - start <= PyStringLiteralUtil.MAX_PREFIX_LENGTH) { if (getLiteralStartOffset(text, start) == offset) return true; } } @@ -89,7 +89,7 @@ public class BaseQuoteHandler extends SimpleTokenSetQuoteHandler implements Mult } private static int getLiteralStartOffset(CharSequence text, int start) { - return PythonStringUtil.getPrefixEndOffset(text, start); + return PyStringLiteralUtil.getPrefixEndOffset(text, start); } @Override diff --git a/python/src/com/jetbrains/python/lexer/PyStringLiteralLexer.java b/python/src/com/jetbrains/python/lexer/PyStringLiteralLexer.java index 944f073a8e67..733146696903 100644 --- a/python/src/com/jetbrains/python/lexer/PyStringLiteralLexer.java +++ b/python/src/com/jetbrains/python/lexer/PyStringLiteralLexer.java @@ -21,7 +21,7 @@ import com.intellij.openapi.util.text.StringUtil; import com.intellij.psi.StringEscapesTokenTypes; import com.intellij.psi.tree.IElementType; import com.jetbrains.python.PyTokenTypes; -import com.jetbrains.python.PythonStringUtil; +import com.jetbrains.python.psi.PyStringLiteralUtil; import org.jetbrains.annotations.NotNull; /** @@ -70,10 +70,10 @@ public class PyStringLiteralLexer extends LexerBase { myBufferEnd = endOffset; // the following could be parsing steps if we wanted this info as tokens - final String prefix = PythonStringUtil.getPrefix(buffer, myStart); + final String prefix = PyStringLiteralUtil.getPrefix(buffer, myStart); - myIsFormatted = PythonStringUtil.isFormattedPrefix(prefix); - myIsRaw = PythonStringUtil.isRawPrefix(prefix); + myIsFormatted = PyStringLiteralUtil.isFormattedPrefix(prefix); + myIsRaw = PyStringLiteralUtil.isRawPrefix(prefix); final int quoteOffset = myStart + prefix.length(); // which quote char? diff --git a/python/src/com/jetbrains/python/psi/PyStringLiteralUtil.java b/python/src/com/jetbrains/python/psi/PyStringLiteralUtil.java new file mode 100644 index 000000000000..8c6d19b42168 --- /dev/null +++ b/python/src/com/jetbrains/python/psi/PyStringLiteralUtil.java @@ -0,0 +1,214 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.jetbrains.python.psi; + +import com.google.common.collect.ImmutableList; +import com.intellij.openapi.util.Pair; +import com.intellij.openapi.util.TextRange; +import com.intellij.openapi.util.text.StringUtil; +import com.intellij.psi.PsiElement; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.List; + +/** + * @author Mikhail Golubev + */ +public class PyStringLiteralUtil { + /** + * Valid string prefix characters (lowercased) as defined in Python lexer. + */ + public static final String PREFIX_CHARACTERS = "ubcrf"; + /** + * Maximum length of a string prefix as defined in Python lexer. + */ + public static final int MAX_PREFIX_LENGTH = 3; + private static final ImmutableList QUOTES = ImmutableList.of("'''", "\"\"\"", "'", "\""); + + private PyStringLiteralUtil() { + } + + /** + * 'text' => text + * "text" => text + * text => text + * "text => "text + * + * @return string without heading and trailing pair of ' or " + */ + @NotNull + public static String getStringValue(@NotNull String s) { + return getStringValueTextRange(s).substring(s); + } + + public static TextRange getStringValueTextRange(@NotNull String s) { + final Pair quotes = getQuotes(s); + if (quotes != null) { + return TextRange.create(quotes.getFirst().length(), s.length() - quotes.getSecond().length()); + } + return TextRange.allOf(s); + } + + /** + * Handles unicode and raw strings + * + * @param text + * @return false if no quotes found, true otherwise + * sdfs -> false + * ur'x' -> true + * "string" -> true + */ + + public static boolean isQuoted(@Nullable String text) { + return text != null && getQuotes(text) != null; + } + + /** + * Handles unicode and raw strings + * + * @param text + * @return open and close quote (including raw/unicode prefixes), null if no quotes present in string + * 'string' -> (', ') + * UR"unicode raw string" -> (UR", ") + */ + @Nullable + public static Pair getQuotes(@NotNull final String text) { + final String prefix = getPrefix(text); + final String mainText = text.substring(prefix.length()); + for (String quote : QUOTES) { + final Pair quotes = getQuotes(mainText, prefix, quote); + if (quotes != null) { + return quotes; + } + } + return null; + } + + /** + * Finds the end offset of the string prefix starting from {@code startOffset} in the given char sequence. + * String prefix may contain only up to {@link #MAX_PREFIX_LENGTH} characters from {@link #PREFIX_CHARACTERS} + * (case insensitively). + * + * @return end offset of found string prefix + */ + public static int getPrefixEndOffset(@NotNull CharSequence text, int startOffset) { + int offset; + for (offset = startOffset; offset < Math.min(startOffset + MAX_PREFIX_LENGTH, text.length()); offset++) { + if (PREFIX_CHARACTERS.indexOf(Character.toLowerCase(text.charAt(offset))) < 0) { + break; + } + } + return offset; + } + + @NotNull + public static String getPrefix(@NotNull CharSequence text) { + return getPrefix(text, 0); + } + + /** + * Extracts string prefix from the given char sequence using {@link #getPrefixEndOffset(CharSequence, int)}. + * + * @return extracted string prefix + * @see #getPrefixEndOffset(CharSequence, int) + */ + @NotNull + public static String getPrefix(@NotNull CharSequence text, int startOffset) { + return text.subSequence(startOffset, getPrefixEndOffset(text, startOffset)).toString(); + } + + /** + * @return whether the given prefix contains either 'u' or 'U' character + */ + public static boolean isUnicodePrefix(@NotNull String prefix) { + return StringUtil.indexOfIgnoreCase(prefix, 'u', 0) >= 0; + } + + /** + * @return whether the given prefix contains either 'b' or 'B' character + */ + public static boolean isBytesPrefix(@NotNull String prefix) { + return StringUtil.indexOfIgnoreCase(prefix, 'b', 0) >= 0; + } + + /** + * @return whether the given prefix contains either 'r' or 'R' character + */ + public static boolean isRawPrefix(@NotNull String prefix) { + return StringUtil.indexOfIgnoreCase(prefix, 'r', 0) >= 0; + } + + /** + * @return whether the given prefix contains either 'f' or 'F' character + */ + public static boolean isFormattedPrefix(@NotNull String prefix) { + return StringUtil.indexOfIgnoreCase(prefix, 'f', 0) >= 0; + } + + @Nullable + private static Pair getQuotes(@NotNull String text, @NotNull String prefix, @NotNull String quote) { + final int length = text.length(); + final int n = quote.length(); + if (length >= 2 * n && text.startsWith(quote) && text.endsWith(quote)) { + return Pair.create(prefix + text.substring(0, n), text.substring(length - n)); + } + return null; + } + + public static TextRange getTextRange(PsiElement element) { + if (element instanceof PyStringLiteralExpression) { + final List ranges = ((PyStringLiteralExpression)element).getStringValueTextRanges(); + return ranges.get(0); + } + else { + return new TextRange(0, element.getTextLength()); + } + } + + @Nullable + public static String getText(@Nullable PyExpression ex) { + if (ex == null) { + return null; + } + else { + return ex.getText(); + } + } + + @Nullable + public static String getStringValue(@Nullable PsiElement o) { + if (o == null) { + return null; + } + if (o instanceof PyStringLiteralExpression) { + PyStringLiteralExpression literalExpression = (PyStringLiteralExpression)o; + return literalExpression.getStringValue(); + } + else { + return o.getText(); + } + } + + public static String stripQuotesAroundValue(String text) { + Pair quotes = getQuotes(text); + if (quotes == null) { + return text; + } + + return text.substring(quotes.first.length(), text.length() - quotes.second.length()); + } +} diff --git a/python/src/com/jetbrains/python/psi/PyUtil.java b/python/src/com/jetbrains/python/psi/PyUtil.java index cc47620fcd49..296d1f23ac5e 100644 --- a/python/src/com/jetbrains/python/psi/PyUtil.java +++ b/python/src/com/jetbrains/python/psi/PyUtil.java @@ -67,7 +67,6 @@ import com.jetbrains.NotNullPredicate; import com.jetbrains.python.PyBundle; import com.jetbrains.python.PyNames; import com.jetbrains.python.PyTokenTypes; -import com.jetbrains.python.PythonStringUtil; import com.jetbrains.python.codeInsight.completion.OverwriteEqualsInsertHandler; import com.jetbrains.python.codeInsight.controlflow.ScopeOwner; import com.jetbrains.python.codeInsight.dataflow.scope.ScopeUtil; @@ -1970,28 +1969,28 @@ public class PyUtil { * @return true if given string node contains "u" or "U" prefix */ public boolean isUnicode() { - return PythonStringUtil.isUnicodePrefix(myPrefix); + return PyStringLiteralUtil.isUnicodePrefix(myPrefix); } /** * @return true if given string node contains "r" or "R" prefix */ public boolean isRaw() { - return PythonStringUtil.isRawPrefix(myPrefix); + return PyStringLiteralUtil.isRawPrefix(myPrefix); } /** * @return true if given string node contains "b" or "B" prefix */ public boolean isBytes() { - return PythonStringUtil.isBytesPrefix(myPrefix); + return PyStringLiteralUtil.isBytesPrefix(myPrefix); } /** * @return true if given string node contains "f" or "F" prefix */ public boolean isFormatted() { - return PythonStringUtil.isFormattedPrefix(myPrefix); + return PyStringLiteralUtil.isFormattedPrefix(myPrefix); } /** diff --git a/python/src/com/jetbrains/python/psi/impl/PyElementGeneratorImpl.java b/python/src/com/jetbrains/python/psi/impl/PyElementGeneratorImpl.java index 1ca42ed2e6f9..336db0a31f1c 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyElementGeneratorImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyElementGeneratorImpl.java @@ -37,7 +37,6 @@ import com.jetbrains.NotNullPredicate; import com.jetbrains.python.PyTokenTypes; import com.jetbrains.python.PythonFileType; import com.jetbrains.python.PythonLanguage; -import com.jetbrains.python.PythonStringUtil; import com.jetbrains.python.psi.*; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -107,7 +106,7 @@ public class PyElementGeneratorImpl extends PyElementGenerator { } public PyStringLiteralExpression createStringLiteral(@NotNull PyStringLiteralExpression oldElement, @NotNull String unescaped) { - Pair quotes = PythonStringUtil.getQuotes(oldElement.getText()); + Pair quotes = PyStringLiteralUtil.getQuotes(oldElement.getText()); if (quotes != null) { return createStringLiteralAlreadyEscaped(quotes.first + unescaped + quotes.second); } diff --git a/python/src/com/jetbrains/python/psi/impl/PyNamedParameterImpl.java b/python/src/com/jetbrains/python/psi/impl/PyNamedParameterImpl.java index 646670976fa2..2efd41e93e62 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyNamedParameterImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyNamedParameterImpl.java @@ -172,9 +172,9 @@ public class PyNamedParameterImpl extends PyBaseElementImpl quotes = PythonStringUtil.getQuotes(defaultValue.getText()); + final Pair quotes = PyStringLiteralUtil.getQuotes(defaultValue.getText()); if (quotes != null) { - representation = quotes.getFirst() + PythonStringUtil.getStringValue(defaultValue) + quotes.getSecond(); + representation = quotes.getFirst() + PyStringLiteralUtil.getStringValue(defaultValue) + quotes.getSecond(); } } sb.append("=").append(representation); diff --git a/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java b/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java index 5f3691f095ad..eb42c7a305ff 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionImpl.java @@ -28,7 +28,6 @@ import com.intellij.psi.impl.source.tree.LeafElement; import com.intellij.psi.tree.IElementType; import com.intellij.psi.util.PsiTreeUtil; import com.jetbrains.python.PyTokenTypes; -import com.jetbrains.python.PythonStringUtil; import com.jetbrains.python.codeInsight.regexp.PythonVerboseRegexpLanguage; import com.jetbrains.python.lexer.PythonHighlightingLexer; import com.jetbrains.python.psi.*; @@ -133,7 +132,7 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt } public static int getPrefixLength(String text) { - return PythonStringUtil.getPrefixEndOffset(text, 0); + return PyStringLiteralUtil.getPrefixEndOffset(text, 0); } private boolean isUnicodeByDefault() { @@ -160,8 +159,8 @@ public class PyStringLiteralExpressionImpl extends PyElementImpl implements PySt final TextRange textRange = getNodeTextRange(text); final int offset = node.getTextRange().getStartOffset() - elementStart + textRange.getStartOffset(); final String encoded = textRange.substring(text); - final boolean hasRawPrefix = PythonStringUtil.isRawPrefix(PythonStringUtil.getPrefix(text)); - final boolean hasUnicodePrefix = PythonStringUtil.isUnicodePrefix(PythonStringUtil.getPrefix(text)); + final boolean hasRawPrefix = PyStringLiteralUtil.isRawPrefix(PyStringLiteralUtil.getPrefix(text)); + final boolean hasUnicodePrefix = PyStringLiteralUtil.isUnicodePrefix(PyStringLiteralUtil.getPrefix(text)); result.addAll(getDecodedFragments(encoded, offset, hasRawPrefix, unicodeByDefault || hasUnicodePrefix)); } myDecodedFragments = result; diff --git a/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionManipulator.java b/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionManipulator.java index 5032d128a44d..e37ad5b47e6a 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionManipulator.java +++ b/python/src/com/jetbrains/python/psi/impl/PyStringLiteralExpressionManipulator.java @@ -18,8 +18,8 @@ package com.jetbrains.python.psi.impl; import com.intellij.openapi.util.Pair; import com.intellij.openapi.util.TextRange; import com.intellij.psi.AbstractElementManipulator; -import com.jetbrains.python.PythonStringUtil; import com.jetbrains.python.psi.PyElementGenerator; +import com.jetbrains.python.psi.PyStringLiteralUtil; import org.jetbrains.annotations.NotNull; /** @@ -29,7 +29,7 @@ public class PyStringLiteralExpressionManipulator extends AbstractElementManipul @Override public PyStringLiteralExpressionImpl handleContentChange(@NotNull PyStringLiteralExpressionImpl element, @NotNull TextRange range, String newContent) { - Pair quotes = PythonStringUtil.getQuotes(range.substring(element.getText())); + Pair quotes = PyStringLiteralUtil.getQuotes(range.substring(element.getText())); if (quotes != null) { range = TextRange.create(range.getStartOffset() + quotes.first.length(), range.getEndOffset() - quotes.second.length()); @@ -44,7 +44,7 @@ public class PyStringLiteralExpressionManipulator extends AbstractElementManipul @NotNull @Override public TextRange getRangeInElement(@NotNull PyStringLiteralExpressionImpl element) { - Pair pair = PythonStringUtil.getQuotes(element.getText()); + Pair pair = PyStringLiteralUtil.getQuotes(element.getText()); if (pair != null) { return TextRange.from(pair.first.length(), element.getTextLength() - pair.first.length() - pair.second.length()); } diff --git a/python/src/com/jetbrains/python/refactoring/PyReplaceExpressionUtil.java b/python/src/com/jetbrains/python/refactoring/PyReplaceExpressionUtil.java index a7fb0c83dd2a..f6b5dc98d130 100644 --- a/python/src/com/jetbrains/python/refactoring/PyReplaceExpressionUtil.java +++ b/python/src/com/jetbrains/python/refactoring/PyReplaceExpressionUtil.java @@ -21,9 +21,7 @@ import com.intellij.openapi.util.TextRange; import com.intellij.openapi.util.text.StringUtil; import com.intellij.psi.PsiElement; import com.intellij.psi.tree.IElementType; -import com.intellij.util.Function; import com.jetbrains.python.PyElementTypes; -import com.jetbrains.python.PythonStringUtil; import com.jetbrains.python.inspections.PyStringFormatParser; import com.jetbrains.python.psi.*; import com.jetbrains.python.psi.impl.PyBuiltinCache; @@ -125,7 +123,7 @@ public class PyReplaceExpressionUtil implements PyElementTypes { @NotNull PsiElement newExpression, @NotNull TextRange textRange) { final String fullText = oldExpression.getText(); - final Pair detectedQuotes = PythonStringUtil.getQuotes(fullText); + final Pair detectedQuotes = PyStringLiteralUtil.getQuotes(fullText); final Pair quotes = detectedQuotes != null ? detectedQuotes : Pair.create("'", "'"); final String prefix = fullText.substring(0, textRange.getStartOffset()); final String suffix = fullText.substring(textRange.getEndOffset(), oldExpression.getTextLength()); diff --git a/python/src/com/jetbrains/python/refactoring/introduce/IntroduceHandler.java b/python/src/com/jetbrains/python/refactoring/introduce/IntroduceHandler.java index 7fb1cf1043f1..457458522f35 100644 --- a/python/src/com/jetbrains/python/refactoring/introduce/IntroduceHandler.java +++ b/python/src/com/jetbrains/python/refactoring/introduce/IntroduceHandler.java @@ -43,7 +43,6 @@ import com.intellij.refactoring.util.CommonRefactoringUtil; import com.jetbrains.python.PyBundle; import com.jetbrains.python.PyNames; import com.jetbrains.python.PyTokenTypes; -import com.jetbrains.python.PythonStringUtil; import com.jetbrains.python.codeInsight.dataflow.scope.ScopeUtil; import com.jetbrains.python.psi.*; import com.jetbrains.python.psi.impl.PyPsiUtils; @@ -546,7 +545,7 @@ abstract public class IntroduceHandler implements RefactoringActionHandler { if (data != null) { final PsiElement parent = data.getFirst(); final String text = parent.getText(); - final Pair detectedQuotes = PythonStringUtil.getQuotes(text); + final Pair detectedQuotes = PyStringLiteralUtil.getQuotes(text); final Pair quotes = detectedQuotes != null ? detectedQuotes : Pair.create("'", "'"); final TextRange range = data.getSecond(); final String substring = range.substring(text); diff --git a/python/src/com/jetbrains/python/spellchecker/PythonSpellcheckerStrategy.java b/python/src/com/jetbrains/python/spellchecker/PythonSpellcheckerStrategy.java index 2cc3ff78a921..82b27dadbed5 100644 --- a/python/src/com/jetbrains/python/spellchecker/PythonSpellcheckerStrategy.java +++ b/python/src/com/jetbrains/python/spellchecker/PythonSpellcheckerStrategy.java @@ -27,10 +27,10 @@ import com.intellij.spellchecker.tokenizer.TokenConsumer; import com.intellij.spellchecker.tokenizer.Tokenizer; import com.intellij.util.containers.ContainerUtil; import com.jetbrains.python.PyTokenTypes; -import com.jetbrains.python.PythonStringUtil; import com.jetbrains.python.inspections.PyStringFormatParser; import com.jetbrains.python.psi.PyBinaryExpression; import com.jetbrains.python.psi.PyStringLiteralExpression; +import com.jetbrains.python.psi.PyStringLiteralUtil; import org.jetbrains.annotations.NotNull; import java.util.List; @@ -48,7 +48,7 @@ public class PythonSpellcheckerStrategy extends SpellcheckingStrategy { final List strNodes = element.getStringNodes(); final List prefixes = ContainerUtil.mapNotNull(strNodes, n -> StringUtil.nullize(new StringNodeInfo(n).getPrefix())); - if (element.textContains('\\') && !prefixes.stream().anyMatch(PythonStringUtil::isRawPrefix)) { + if (element.textContains('\\') && !prefixes.stream().anyMatch(PyStringLiteralUtil::isRawPrefix)) { for (Pair fragment : element.getDecodedFragments()) { final String value = fragment.getSecond(); final int startOffset = fragment.getFirst().getStartOffset();