From b8f2aa3a7d4763c5611ebbc69589fa55c4404c8b Mon Sep 17 00:00:00 2001 From: Mikhail Golubev Date: Tue, 21 Aug 2018 13:40:42 +0300 Subject: [PATCH] PY-31442 Start incorporating f-string specific tokens into the lexer --- .../com/jetbrains/python/PyTokenTypes.java | 2 + .../python/psi/PyFStringFragment.java | 22 ++++ .../python/psi/PyFormattedStringNode.java | 11 ++ .../com/jetbrains/python/PyElementTypes.java | 3 + .../psi/impl/PyFStringFragmentImpl.java | 43 +++++++ .../psi/impl/PyFormattedStringNodeImpl.java | 115 ++++++++++++++++++ .../psi/impl/PyLiteralStringNodeImpl.java | 8 +- 7 files changed, 199 insertions(+), 5 deletions(-) create mode 100644 python/psi-api/src/com/jetbrains/python/psi/PyFStringFragment.java create mode 100644 python/psi-api/src/com/jetbrains/python/psi/PyFormattedStringNode.java create mode 100644 python/src/com/jetbrains/python/psi/impl/PyFStringFragmentImpl.java create mode 100644 python/src/com/jetbrains/python/psi/impl/PyFormattedStringNodeImpl.java diff --git a/python/psi-api/src/com/jetbrains/python/PyTokenTypes.java b/python/psi-api/src/com/jetbrains/python/PyTokenTypes.java index 63b80160a75b..cee9fc3437bb 100644 --- a/python/psi-api/src/com/jetbrains/python/PyTokenTypes.java +++ b/python/psi-api/src/com/jetbrains/python/PyTokenTypes.java @@ -178,4 +178,6 @@ public class PyTokenTypes { public static final PyElementType INDENT = new PyElementType("INDENT"); public static final PyElementType DEDENT = new PyElementType("DEDENT"); + + PyElementType FSTRING_TEXT = new PyElementType("FSTRING_FRAGMENT"); } diff --git a/python/psi-api/src/com/jetbrains/python/psi/PyFStringFragment.java b/python/psi-api/src/com/jetbrains/python/psi/PyFStringFragment.java new file mode 100644 index 000000000000..f6e81e860ca2 --- /dev/null +++ b/python/psi-api/src/com/jetbrains/python/psi/PyFStringFragment.java @@ -0,0 +1,22 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.jetbrains.python.psi; + +import com.intellij.psi.PsiElement; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.List; + +public interface PyFStringFragment extends PsiElement { + @Nullable + PyExpression getMainExpression(); + + @NotNull + List getFormatFragments(); + + @Nullable + PsiElement getColon(); + + @Nullable + PsiElement getClosingBrace(); +} diff --git a/python/psi-api/src/com/jetbrains/python/psi/PyFormattedStringNode.java b/python/psi-api/src/com/jetbrains/python/psi/PyFormattedStringNode.java new file mode 100644 index 000000000000..5269a5721e28 --- /dev/null +++ b/python/psi-api/src/com/jetbrains/python/psi/PyFormattedStringNode.java @@ -0,0 +1,11 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.jetbrains.python.psi; + +import org.jetbrains.annotations.NotNull; + +import java.util.List; + +public interface PyFormattedStringNode extends PyRichStringNode { + @NotNull + List getFragments(); +} diff --git a/python/src/com/jetbrains/python/PyElementTypes.java b/python/src/com/jetbrains/python/PyElementTypes.java index 82292eb0d50b..321a76c3284f 100644 --- a/python/src/com/jetbrains/python/PyElementTypes.java +++ b/python/src/com/jetbrains/python/PyElementTypes.java @@ -150,4 +150,7 @@ public interface PyElementTypes { TokenSet STAR_PARAMETERS = TokenSet.create(NAMED_PARAMETER, STAR_ARGUMENT_EXPRESSION, STAR_EXPRESSION, DOUBLE_STAR_EXPRESSION); TokenSet CLASS_OR_FUNCTION = TokenSet.create(CLASS_DECLARATION, FUNCTION_DECLARATION); TokenSet IMPORT_STATEMENTS = TokenSet.create(IMPORT_STATEMENT, FROM_IMPORT_STATEMENT); + + PyElementType FSTRING_NODE = new PyElementType("FSTRING_NODE", PyFormattedStringNodeImpl.class); + PyElementType FSTRING_FRAGMENT = new PyElementType("FSTRING_FRAGMENT", PyFStringFragmentImpl.class); } diff --git a/python/src/com/jetbrains/python/psi/impl/PyFStringFragmentImpl.java b/python/src/com/jetbrains/python/psi/impl/PyFStringFragmentImpl.java new file mode 100644 index 000000000000..180fbf2a44b7 --- /dev/null +++ b/python/src/com/jetbrains/python/psi/impl/PyFStringFragmentImpl.java @@ -0,0 +1,43 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.jetbrains.python.psi.impl; + +import com.intellij.lang.ASTNode; +import com.intellij.psi.PsiElement; +import com.jetbrains.python.PyElementTypes; +import com.jetbrains.python.PyTokenTypes; +import com.jetbrains.python.psi.PyExpression; +import com.jetbrains.python.psi.PyFStringFragment; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.List; + +public class PyFStringFragmentImpl extends PyElementImpl implements PyFStringFragment { + public PyFStringFragmentImpl(ASTNode astNode) { + super(astNode); + } + + @Nullable + @Override + public PyExpression getMainExpression() { + return findChildByClass(PyExpression.class); + } + + @NotNull + @Override + public List getFormatFragments() { + return findChildrenByType(PyElementTypes.FSTRING_FRAGMENT); + } + + @Nullable + @Override + public PsiElement getColon() { + return findChildByType(PyTokenTypes.COLON); + } + + @Nullable + @Override + public PsiElement getClosingBrace() { + return findChildByType(PyTokenTypes.RBRACE); + } +} diff --git a/python/src/com/jetbrains/python/psi/impl/PyFormattedStringNodeImpl.java b/python/src/com/jetbrains/python/psi/impl/PyFormattedStringNodeImpl.java new file mode 100644 index 000000000000..d0c8289b78d7 --- /dev/null +++ b/python/src/com/jetbrains/python/psi/impl/PyFormattedStringNodeImpl.java @@ -0,0 +1,115 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.jetbrains.python.psi.impl; + +import com.intellij.lang.ASTNode; +import com.intellij.openapi.util.Pair; +import com.intellij.openapi.util.TextRange; +import com.jetbrains.python.PyElementTypes; +import com.jetbrains.python.psi.PyFStringFragment; +import com.jetbrains.python.psi.PyFormattedStringNode; +import com.jetbrains.python.psi.PyStringLiteralUtil; +import org.jetbrains.annotations.NotNull; + +import java.util.List; +import java.util.Set; + +public class PyFormattedStringNodeImpl extends PyElementImpl implements PyFormattedStringNode { + + public PyFormattedStringNodeImpl(ASTNode astNode) { + super(astNode); + } + + @NotNull + @Override + public List getFragments() { + return findChildrenByType(PyElementTypes.FSTRING_FRAGMENT); + } + + @NotNull + @Override + public String getPrefix() { + return PyStringLiteralUtil.getPrefix(getText()); + } + + @Override + public int getPrefixLength() { + return PyStringLiteralUtil.getPrefixLength(getText()); + } + + @NotNull + @Override + public String getTextWithoutPrefix() { + return null; + } + + @NotNull + @Override + public TextRange getContentRange() { + return null; + } + + @NotNull + @Override + public TextRange getAbsoluteContentRange() { + return null; + } + + @NotNull + @Override + public String getContent() { + return null; + } + + @NotNull + @Override + public List> getDecodedFragments() { + return null; + } + + @NotNull + @Override + public String getQuote() { + return null; + } + + @Override + public char getQuoteChar() { + return 0; + } + + @Override + public boolean isTripleQuoted() { + return false; + } + + @Override + public boolean isTerminated() { + return false; + } + + @NotNull + @Override + public Set getModifiers() { + return null; + } + + @Override + public boolean isUnicode() { + return false; + } + + @Override + public boolean isRaw() { + return false; + } + + @Override + public boolean isBytes() { + return false; + } + + @Override + public final boolean isFormatted() { + return true; + } +} diff --git a/python/src/com/jetbrains/python/psi/impl/PyLiteralStringNodeImpl.java b/python/src/com/jetbrains/python/psi/impl/PyLiteralStringNodeImpl.java index 81315c556d59..7a81e417eed6 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyLiteralStringNodeImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyLiteralStringNodeImpl.java @@ -72,9 +72,7 @@ public class PyLiteralStringNodeImpl extends LeafPsiElement implements PyLiteral @NotNull @Override public String getQuote() { - final Pair quotes = PyStringLiteralUtil.getQuotes(getText()); - assert quotes != null; - return quotes.getFirst(); + return getText().substring(getPrefixLength(), getContentRange().getStartOffset()); } @Override @@ -133,7 +131,7 @@ public class PyLiteralStringNodeImpl extends LeafPsiElement implements PyLiteral } @Override - public boolean isFormatted() { - return StringUtil.containsIgnoreCase(getPrefix(), "f"); + public final boolean isFormatted() { + return false; } }