PY-31442 Start incorporating f-string specific tokens into the lexer

python/psi-api/src/com/jetbrains/python/PyTokenTypes.java

@@ -178,4 +178,6 @@ public class PyTokenTypes {
 
   public static final PyElementType INDENT = new PyElementType("INDENT");
   public static final PyElementType DEDENT = new PyElementType("DEDENT");
+
+  PyElementType FSTRING_TEXT = new PyElementType("FSTRING_FRAGMENT");
 }

python/psi-api/src/com/jetbrains/python/psi/PyFStringFragment.java

@@ -0,0 +1,22 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.jetbrains.python.psi;
+
+import com.intellij.psi.PsiElement;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.List;
+
+public interface PyFStringFragment extends PsiElement {
+  @Nullable
+  PyExpression getMainExpression();
+
+  @NotNull
+  List<PyFStringFragment> getFormatFragments();
+
+  @Nullable
+  PsiElement getColon();
+
+  @Nullable
+  PsiElement getClosingBrace();
+}

python/psi-api/src/com/jetbrains/python/psi/PyFormattedStringNode.java

@@ -0,0 +1,11 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.jetbrains.python.psi;
+
+import org.jetbrains.annotations.NotNull;
+
+import java.util.List;
+
+public interface PyFormattedStringNode extends PyRichStringNode {
+  @NotNull
+  List<PyFStringFragment> getFragments();
+}

python/src/com/jetbrains/python/PyElementTypes.java

@@ -150,4 +150,7 @@ public interface PyElementTypes {
   TokenSet STAR_PARAMETERS = TokenSet.create(NAMED_PARAMETER, STAR_ARGUMENT_EXPRESSION, STAR_EXPRESSION, DOUBLE_STAR_EXPRESSION);
   TokenSet CLASS_OR_FUNCTION = TokenSet.create(CLASS_DECLARATION, FUNCTION_DECLARATION);
   TokenSet IMPORT_STATEMENTS = TokenSet.create(IMPORT_STATEMENT, FROM_IMPORT_STATEMENT);
+
+  PyElementType FSTRING_NODE = new PyElementType("FSTRING_NODE", PyFormattedStringNodeImpl.class);
+  PyElementType FSTRING_FRAGMENT = new PyElementType("FSTRING_FRAGMENT", PyFStringFragmentImpl.class);
 }

python/src/com/jetbrains/python/psi/impl/PyFStringFragmentImpl.java

@@ -0,0 +1,43 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.jetbrains.python.psi.impl;
+
+import com.intellij.lang.ASTNode;
+import com.intellij.psi.PsiElement;
+import com.jetbrains.python.PyElementTypes;
+import com.jetbrains.python.PyTokenTypes;
+import com.jetbrains.python.psi.PyExpression;
+import com.jetbrains.python.psi.PyFStringFragment;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.List;
+
+public class PyFStringFragmentImpl extends PyElementImpl implements PyFStringFragment {
+  public PyFStringFragmentImpl(ASTNode astNode) {
+    super(astNode);
+  }
+
+  @Nullable
+  @Override
+  public PyExpression getMainExpression() {
+    return findChildByClass(PyExpression.class);
+  }
+
+  @NotNull
+  @Override
+  public List<PyFStringFragment> getFormatFragments() {
+    return findChildrenByType(PyElementTypes.FSTRING_FRAGMENT);
+  }
+
+  @Nullable
+  @Override
+  public PsiElement getColon() {
+    return findChildByType(PyTokenTypes.COLON);
+  }
+
+  @Nullable
+  @Override
+  public PsiElement getClosingBrace() {
+    return findChildByType(PyTokenTypes.RBRACE);
+  }
+}

python/src/com/jetbrains/python/psi/impl/PyFormattedStringNodeImpl.java

@@ -0,0 +1,115 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.jetbrains.python.psi.impl;
+
+import com.intellij.lang.ASTNode;
+import com.intellij.openapi.util.Pair;
+import com.intellij.openapi.util.TextRange;
+import com.jetbrains.python.PyElementTypes;
+import com.jetbrains.python.psi.PyFStringFragment;
+import com.jetbrains.python.psi.PyFormattedStringNode;
+import com.jetbrains.python.psi.PyStringLiteralUtil;
+import org.jetbrains.annotations.NotNull;
+
+import java.util.List;
+import java.util.Set;
+
+public class PyFormattedStringNodeImpl extends PyElementImpl implements PyFormattedStringNode {
+
+  public PyFormattedStringNodeImpl(ASTNode astNode) {
+    super(astNode);
+  }
+
+  @NotNull
+  @Override
+  public List<PyFStringFragment> getFragments() {
+    return findChildrenByType(PyElementTypes.FSTRING_FRAGMENT);
+  }
+
+  @NotNull
+  @Override
+  public String getPrefix() {
+    return PyStringLiteralUtil.getPrefix(getText());
+  }
+
+  @Override
+  public int getPrefixLength() {
+    return PyStringLiteralUtil.getPrefixLength(getText());
+  }
+
+  @NotNull
+  @Override
+  public String getTextWithoutPrefix() {
+    return null;
+  }
+
+  @NotNull
+  @Override
+  public TextRange getContentRange() {
+    return null;
+  }
+
+  @NotNull
+  @Override
+  public TextRange getAbsoluteContentRange() {
+    return null;
+  }
+
+  @NotNull
+  @Override
+  public String getContent() {
+    return null;
+  }
+
+  @NotNull
+  @Override
+  public List<Pair<TextRange, String>> getDecodedFragments() {
+    return null;
+  }
+
+  @NotNull
+  @Override
+  public String getQuote() {
+    return null;
+  }
+
+  @Override
+  public char getQuoteChar() {
+    return 0;
+  }
+
+  @Override
+  public boolean isTripleQuoted() {
+    return false;
+  }
+
+  @Override
+  public boolean isTerminated() {
+    return false;
+  }
+
+  @NotNull
+  @Override
+  public Set<Modifier> getModifiers() {
+    return null;
+  }
+
+  @Override
+  public boolean isUnicode() {
+    return false;
+  }
+
+  @Override
+  public boolean isRaw() {
+    return false;
+  }
+
+  @Override
+  public boolean isBytes() {
+    return false;
+  }
+
+  @Override
+  public final boolean isFormatted() {
+    return true;
+  }
+}

python/src/com/jetbrains/python/psi/impl/PyLiteralStringNodeImpl.java

@@ -72,9 +72,7 @@ public class PyLiteralStringNodeImpl extends LeafPsiElement implements PyLiteral
   @NotNull
   @Override
   public String getQuote() {
-    final Pair<String, String> quotes = PyStringLiteralUtil.getQuotes(getText());
-    assert quotes != null;
-    return quotes.getFirst();
+    return getText().substring(getPrefixLength(), getContentRange().getStartOffset());
   }
 
   @Override
@@ -133,7 +131,7 @@ public class PyLiteralStringNodeImpl extends LeafPsiElement implements PyLiteral
   }
 
   @Override
-  public boolean isFormatted() {
-    return StringUtil.containsIgnoreCase(getPrefix(), "f");
+  public final boolean isFormatted() {
+    return false;
   }
 }