PY-31758 Highlight escape sequences inside literal parts of f-strings

We can't suppress such highlighting for raw f-strings, though, as we do
for ordinary string literals, since it's not possible to keep track of
"r" prefixes in FSTRING_START tokens with the current API of LayeredLexer:
layer lexers are supposed to be stateless and can't distinguish between
f" as a start of a new literal and when it occurs as a textual part of,
say, f'f"'.

python/src/com/jetbrains/python/highlighting/PyHighlighter.java

@@ -10,6 +10,7 @@ import com.intellij.psi.StringEscapesTokenTypes;
 import com.intellij.psi.tree.IElementType;
 import com.jetbrains.python.PyTokenTypes;
 import com.jetbrains.python.PythonDialectsTokenSetProvider;
+import com.jetbrains.python.lexer.PyFStringLiteralLexer;
 import com.jetbrains.python.lexer.PyStringLiteralLexer;
 import com.jetbrains.python.lexer.PythonHighlightingLexer;
 import com.jetbrains.python.psi.LanguageLevel;
@@ -47,6 +48,10 @@ public class PyHighlighter extends SyntaxHighlighterBase {
       new PyStringLiteralLexer(PyTokenTypes.TRIPLE_QUOTED_UNICODE),
       PyTokenTypes.TRIPLE_QUOTED_UNICODE
     );
+    ret.registerLayer(
+      new PyFStringLiteralLexer(), 
+      PyTokenTypes.FSTRING_TEXT
+    );
 
     return ret;
   }

python/src/com/jetbrains/python/lexer/PyFStringLiteralLexer.kt

@@ -0,0 +1,28 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.jetbrains.python.lexer
+
+import com.intellij.util.text.CharArrayUtil
+import com.jetbrains.python.PyTokenTypes
+
+class PyFStringLiteralLexer: PyStringLiteralLexerBase(PyTokenTypes.FSTRING_TEXT) {
+  override fun locateToken(start: Int): Int {
+    if (start >= myBufferEnd) {
+      return myBufferEnd
+    }
+    
+    if (myBuffer[start] == '\\') {
+      return locateEscapeSequence(start)
+    }
+    else {
+      val nextBackslashOffset = CharArrayUtil.indexOf(myBuffer, "\\", start + 1, myBufferEnd)
+      return if (nextBackslashOffset >= 0) nextBackslashOffset else myBufferEnd
+    }
+  }
+
+  // TODO actually keep track of "raw" prefixes of f-strings somehow
+  override fun isRaw(): Boolean = false
+
+  override fun isUnicodeMode(): Boolean = true
+
+  override fun getState(): Int = myBaseLexerState
+}

python/src/com/jetbrains/python/lexer/PyStringLiteralLexer.java

@@ -1,10 +1,6 @@
 // Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
 package com.jetbrains.python.lexer;
 
-import com.intellij.lexer.LexerBase;
-import com.intellij.openapi.diagnostic.Logger;
-import com.intellij.openapi.util.text.StringUtil;
-import com.intellij.psi.StringEscapesTokenTypes;
 import com.intellij.psi.tree.IElementType;
 import com.jetbrains.python.PyTokenTypes;
 import com.jetbrains.python.psi.PyStringLiteralUtil;
@@ -17,48 +13,33 @@ import org.jetbrains.annotations.NotNull;
  * Lexes the entire string, with u/b/r designator, quotes, and content, thus self-adjusts for the format.
  * User: dcheryasov
  */
-public class PyStringLiteralLexer extends LexerBase {
-  private static final Logger LOG = Logger.getInstance("#com.jetbrains.python.lexer.PyStringLiteralLexer");
+public class PyStringLiteralLexer extends PyStringLiteralLexerBase {
 
   private static final short BEFORE_FIRST_QUOTE = 0; // the initial state; may last during 'u' and 'r' prefixes.
   private static final short AFTER_FIRST_QUOTE = 1;
   private static final short AFTER_LAST_QUOTE = 2;
 
-  private CharSequence myBuffer;
-  private int myStart;
-  private int myEnd;
   private int myState;
   private int myLastState;
-  private int myBufferEnd;
   private char myQuoteChar;
-
   private boolean myIsRaw;
   private boolean myIsTriple;
-  private boolean myIsFormatted;
-  private final IElementType myOriginalLiteralToken;
-  private boolean mySeenEscapedSpacesOnly;
-
 
   /**
    * @param originalLiteralToken the AST node we're layering over.
    */
   public PyStringLiteralLexer(final IElementType originalLiteralToken) {
-    myOriginalLiteralToken = originalLiteralToken;
+    super(originalLiteralToken);
   }
 
   @Override
-  public void start(@NotNull CharSequence buffer, int startOffset, int endOffset, int initialState) {
-    myBuffer = buffer;
-    myStart = startOffset;
+  protected void handleStart(@NotNull CharSequence buffer, int initialState) {
     myState = initialState;
     myLastState = initialState;
-    mySeenEscapedSpacesOnly = true;
-    myBufferEnd = endOffset;
-
+    
     // the following could be parsing steps if we wanted this info as tokens
     final String prefix = PyStringLiteralUtil.getPrefix(buffer, myStart);
 
-    myIsFormatted = PyStringLiteralUtil.isFormattedPrefix(prefix);
     myIsRaw = PyStringLiteralUtil.isRawPrefix(prefix);
 
     final int quoteOffset = myStart + prefix.length();
@@ -68,9 +49,27 @@ public class PyStringLiteralLexer extends LexerBase {
     myQuoteChar = c;
 
     myIsTriple = (buffer.length() > quoteOffset + 2) && (buffer.charAt(quoteOffset + 1) == c) && (buffer.charAt(quoteOffset + 2) == c);
+  }
 
-    // calculate myEnd at last
-    myEnd = locateToken(myStart);
+  @Override
+  protected boolean isRaw() {
+    return myIsRaw;
+  }
+
+  @Override
+  protected boolean isUnicodeMode() {
+    return PyTokenTypes.UNICODE_NODES.contains(myOriginalLiteralToken);
+  }
+
+  @Override
+  protected boolean isEscapeEnd(int offset) {
+    return super.isEscapeEnd(offset) || myBuffer.charAt(offset) == myQuoteChar;
+  }
+
+  @Override
+  public void advance() {
+    myLastState = myState;
+    super.advance();
   }
 
   @Override
@@ -79,183 +78,27 @@ public class PyStringLiteralLexer extends LexerBase {
   }
 
   @Override
-  public IElementType getTokenType() {
-    if (myStart >= myEnd) return null;
-
-    // skip non-escapes immediately
-    if (myBuffer.charAt(myStart) != '\\' || (myIsRaw && (!isUnicodeMode() || !nextIsUnicodeEscape()))) {
-      mySeenEscapedSpacesOnly = false;
-      return myOriginalLiteralToken;
-    }
-
-    // from here on, only escapes
-    if (myStart + 1 >= myEnd) return StringEscapesTokenTypes.INVALID_CHARACTER_ESCAPE_TOKEN; // escape ends too early
-    char nextChar = myBuffer.charAt(myStart + 1);
-    mySeenEscapedSpacesOnly &= nextChar == ' ';
-    if ((nextChar == '\n' || nextChar == ' ' && (mySeenEscapedSpacesOnly || isTrailingSpace(myStart+2)))) {
-      return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN; // escaped EOL
-    }
-    if (nextChar == 'u' || nextChar == 'U') {
-      if (isUnicodeMode()) {
-        final int width = nextChar == 'u'? 4 : 8; // is it uNNNN or Unnnnnnnn
-        for(int i = myStart + 2; i < myStart + width + 2; i++) {
-          if (i >= myEnd || !StringUtil.isHexDigit(myBuffer.charAt(i))) return StringEscapesTokenTypes.INVALID_UNICODE_ESCAPE_TOKEN;
-        }
-        return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN;
-      }
-      else return myOriginalLiteralToken; // b"\u1234" is just b"\\u1234", nothing gets escaped
-    }
-
-    if (nextChar == 'x') { // \xNN is allowed both in bytes and unicode.
-      for(int i = myStart + 2; i < myStart + 4; i++) {
-        if (i >= myEnd || !StringUtil.isHexDigit(myBuffer.charAt(i))) return StringEscapesTokenTypes.INVALID_UNICODE_ESCAPE_TOKEN;
-      }
-      return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN;
-    }
-
-    if (nextChar == 'N' && isUnicodeMode()) {
-      int i = myStart+2;
-      if (i >= myEnd || myBuffer.charAt(i) != '{') return StringEscapesTokenTypes.INVALID_UNICODE_ESCAPE_TOKEN;
-      i++;
-      while(i < myEnd && myBuffer.charAt(i) != '}') i++;
-      if (i >= myEnd) return StringEscapesTokenTypes.INVALID_UNICODE_ESCAPE_TOKEN;
-      return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN;
-    }
-
-    switch (nextChar) {
-      case 'a':
-      case 'b':
-      case 'f':
-      case 'n':
-      case 'r':
-      case 't':
-      case 'v':
-      case '\'':
-      case '\"':
-      case '\\':
-      case '0':
-      case '1':
-      case '2':
-      case '3':
-      case '4':
-      case '5':
-      case '6':
-      case '7': return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN;
-    }
-
-    // other unrecognized escapes are just part of string, not an error
-    return myOriginalLiteralToken;
-  }
-
-  private boolean nextIsUnicodeEscape() {
-    if (myStart + 1 < myEnd) {
-      char nextChar = myBuffer.charAt(myStart + 1);
-      return nextChar == 'u' || nextChar == 'U';
-    }
-    return false;
-  }
-
-  private boolean isUnicodeMode() {
-    return PyTokenTypes.UNICODE_NODES.contains(myOriginalLiteralToken);
-  }
-
-  // all subsequent chars are escaped spaces
-  private boolean isTrailingSpace(final int start) {
-    for (int i=start; i<myBufferEnd; i+=2) {
-      final char c = myBuffer.charAt(i);
-      if (c != '\\') return false;
-      if (i == myBufferEnd-1) return false;
-      if (myBuffer.charAt(i+1) != ' ') return false;
-    }
-    return true;
-  }
-
-  @Override
-  public int getTokenStart() {
-    assert myStart < myEnd || (myStart == myEnd && myEnd == myBufferEnd);
-    return myStart;
-  }
-
-  @Override
-  public int getTokenEnd() {
-    if (!(myStart < myEnd || (myStart == myEnd && myEnd == myBufferEnd))) {
-      LOG.error("myStart=" + myStart + " myEnd="+ myEnd + " myBufferEnd=" + myBufferEnd + " text=" + myBuffer.subSequence(myStart, myBufferEnd));
-    }
-    return myEnd;
-  }
-
-  private int locateToken(int start) {
+  protected int locateToken(int start) {
     if (start == myBufferEnd) {
       myState = AFTER_LAST_QUOTE;
     }
     if (myState == AFTER_LAST_QUOTE) return start; // exhausted
+    
     int i = start;
     if (myBuffer.charAt(i) == '\\') {
       LOG.assertTrue(myState == AFTER_FIRST_QUOTE);
-      i++;
-      if (myIsRaw) return i;
-      if (i == myBufferEnd) {
+      final int end = locateEscapeSequence(i);
+      if (end == myBufferEnd) {
         myState = AFTER_LAST_QUOTE;
-        return i;
-      }
-
-      // is octal?
-      if (myBuffer.charAt(i) >= '0' && myBuffer.charAt(i) <= '7') {
-        char first = myBuffer.charAt(i);
-        i++;
-        if (i < myBufferEnd && myBuffer.charAt(i) >= '0' && myBuffer.charAt(i) <= '7') {
-          i++;
-          if (i < myBufferEnd && first <= '3' && myBuffer.charAt(i) >= '0' && myBuffer.charAt(i) <= '7') {
-            i++;
-          }
-        }
-        return i;
-      }
-
-      // \xNN byte escape
-      if (myBuffer.charAt(i) == 'x') {
-        i++;
-        for (; i < start + 4; i++) {
-          if (i == myBufferEnd || myBuffer.charAt(i) == '\n' || myBuffer.charAt(i) == myQuoteChar || myBuffer.charAt(i) == '\\') {
-            return i;
-          }
-        }
-        return i;
-      }
-
-      // unicode escape
-      if (myBuffer.charAt(i) == 'u' || myBuffer.charAt(i) == 'U') {
-        final int width = myBuffer.charAt(i) == 'u'? 4 : 8; // is it uNNNN or Unnnnnnnn
-        i++;
-        for (; i < start + width + 2; i++) {
-          if (i == myBufferEnd || myBuffer.charAt(i) == '\n' || myBuffer.charAt(i) == myQuoteChar || myBuffer.charAt(i) == '\\') {
-            return i;
-          }
-        }
-        return i;
-      }
-
-      if (myBuffer.charAt(i) == 'N' && isUnicodeMode()) {
-        i++;
-        while(i < myBufferEnd && myBuffer.charAt(i) != '}' && myBuffer.charAt(i) != '\\') {
-          i++;
-        }
-        if (i < myBufferEnd && myBuffer.charAt(i) == '}') {
-          i++;
-        }
-        return i;
-      }
-
-      else {
-        return i + 1;
       }
+      return end;
     }
     else { // not a \something
       //LOG.assertTrue(myState == AFTER_FIRST_QUOTE || myBuffer.charAt(i) == myQuoteChar);
       final int quote_limit = myIsTriple ? 3 : 1;
       int qcnt = 0; // count consequent quotes
       while (i < myBufferEnd) { // scan to next \something
-        if (myBuffer.charAt(i) == '\\' && !myIsRaw) {
+        if (myBuffer.charAt(i) == '\\' && !isRaw()) {
           return i;
         }
         if (myState == BEFORE_FIRST_QUOTE && myBuffer.charAt(i) == myQuoteChar) {
@@ -265,7 +108,7 @@ public class PyStringLiteralLexer extends LexerBase {
             qcnt = 0; // for last quote detection in the same pass
           }
         }
-        else if (myState == AFTER_FIRST_QUOTE && myBuffer.charAt(i) == myQuoteChar && (!myIsRaw || myBuffer.charAt(i-1) != '\\')) { // done?
+        else if (myState == AFTER_FIRST_QUOTE && myBuffer.charAt(i) == myQuoteChar && (!isRaw() || myBuffer.charAt(i-1) != '\\')) { // done?
           qcnt += 1;
           if (qcnt == quote_limit) {
             myState = AFTER_LAST_QUOTE;
@@ -281,26 +124,4 @@ public class PyStringLiteralLexer extends LexerBase {
 
     return i;
   }
-
-  @Override
-  public void advance() {
-    myLastState = myState;
-    myStart = myEnd;
-    myEnd = locateToken(myStart);
-    if (! (myStart < myEnd || (myStart == myEnd && myEnd == myBufferEnd))) {
-      LOG.warn("Inconsistent: start " + myStart + ", end " + myEnd + ", buf end " + myBufferEnd);
-    }
-    //assert myStart < myEnd || (myStart == myEnd && myEnd == myBufferEnd) : "Inconsistent: start " + myStart + ", end " + myEnd + ", buf end " + myBufferEnd;
-  }
-
-  @Override
-  @NotNull
-  public CharSequence getBufferSequence() {
-    return myBuffer;
-  }
-
-  @Override
-  public int getBufferEnd() {
-    return myBufferEnd;
-  }
 }

python/src/com/jetbrains/python/lexer/PyStringLiteralLexerBase.java

@@ -0,0 +1,240 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.jetbrains.python.lexer;
+
+import com.intellij.lexer.LexerBase;
+import com.intellij.openapi.diagnostic.Logger;
+import com.intellij.openapi.util.text.StringUtil;
+import com.intellij.psi.StringEscapesTokenTypes;
+import com.intellij.psi.tree.IElementType;
+import org.jetbrains.annotations.NotNull;
+
+public abstract class PyStringLiteralLexerBase extends LexerBase {
+  protected static final Logger LOG = Logger.getInstance("#com.jetbrains.python.lexer.PyStringLiteralLexer");
+  protected final IElementType myOriginalLiteralToken;
+  protected CharSequence myBuffer;
+  protected int myBufferEnd;
+  protected int myStart;
+  protected int myEnd;
+  protected int myBaseLexerState;
+  private boolean mySeenEscapedSpacesOnly;
+
+  public PyStringLiteralLexerBase(final IElementType originalLiteralToken) {
+    myOriginalLiteralToken = originalLiteralToken;
+  }
+
+  @Override
+  public final void start(@NotNull CharSequence buffer, int startOffset, int endOffset, int initialState) {
+    myBuffer = buffer;
+    myStart = startOffset;
+    mySeenEscapedSpacesOnly = true;
+    myBufferEnd = endOffset;
+    myBaseLexerState = initialState;
+
+    handleStart(buffer, initialState);
+
+    // calculate myEnd at last
+    myEnd = locateToken(myStart);
+  }
+
+  protected void handleStart(@NotNull CharSequence buffer, int initialState) {}
+
+  protected abstract boolean isRaw();
+
+  protected abstract boolean isUnicodeMode();
+
+  @Override
+  public IElementType getTokenType() {
+    if (myStart >= myEnd) return null;
+
+    // skip non-escapes immediately
+    if (myBuffer.charAt(myStart) != '\\' || (isRaw() && !(isUnicodeMode() && nextIsUnicodeEscape()))) {
+      mySeenEscapedSpacesOnly = false;
+      return myOriginalLiteralToken;
+    }
+
+    // from here on, only escapes
+    return getEscapeSequenceType();
+  }
+
+  @NotNull
+  public final IElementType getEscapeSequenceType() {
+    if (myStart + 1 >= myEnd) return StringEscapesTokenTypes.INVALID_CHARACTER_ESCAPE_TOKEN; // escape ends too early
+    char nextChar = myBuffer.charAt(myStart + 1);
+    mySeenEscapedSpacesOnly &= nextChar == ' ';
+    if ((nextChar == '\n' || nextChar == ' ' && (mySeenEscapedSpacesOnly || isTrailingSpace(myStart + 2)))) {
+      return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN; // escaped EOL
+    }
+    if (nextChar == 'u' || nextChar == 'U') {
+      if (isUnicodeMode()) {
+        final int width = nextChar == 'u' ? 4 : 8; // is it uNNNN or Unnnnnnnn
+        for (int i = myStart + 2; i < myStart + width + 2; i++) {
+          if (i >= myEnd || !StringUtil.isHexDigit(myBuffer.charAt(i))) return StringEscapesTokenTypes.INVALID_UNICODE_ESCAPE_TOKEN;
+        }
+        return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN;
+      }
+      else {
+        return myOriginalLiteralToken; // b"\u1234" is just b"\\u1234", nothing gets escaped
+      }
+    }
+
+    if (nextChar == 'x') { // \xNN is allowed both in bytes and unicode.
+      for (int i = myStart + 2; i < myStart + 4; i++) {
+        if (i >= myEnd || !StringUtil.isHexDigit(myBuffer.charAt(i))) return StringEscapesTokenTypes.INVALID_UNICODE_ESCAPE_TOKEN;
+      }
+      return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN;
+    }
+
+    if (nextChar == 'N' && isUnicodeMode()) {
+      int i = myStart + 2;
+      if (i >= myEnd || myBuffer.charAt(i) != '{') return StringEscapesTokenTypes.INVALID_UNICODE_ESCAPE_TOKEN;
+      i++;
+      while (i < myEnd && myBuffer.charAt(i) != '}') i++;
+      if (i >= myEnd) return StringEscapesTokenTypes.INVALID_UNICODE_ESCAPE_TOKEN;
+      return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN;
+    }
+
+    switch (nextChar) {
+      case 'a':
+      case 'b':
+      case 'f':
+      case 'n':
+      case 'r':
+      case 't':
+      case 'v':
+      case '\'':
+      case '\"':
+      case '\\':
+      case '0':
+      case '1':
+      case '2':
+      case '3':
+      case '4':
+      case '5':
+      case '6':
+      case '7':
+        return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN;
+    }
+
+    // other unrecognized escapes are just part of string, not an error
+    return myOriginalLiteralToken;
+  }
+
+  private boolean nextIsUnicodeEscape() {
+    if (myStart + 1 < myEnd) {
+      char nextChar = myBuffer.charAt(myStart + 1);
+      return nextChar == 'u' || nextChar == 'U';
+    }
+    return false;
+  }
+
+  // all subsequent chars are escaped spaces
+  private boolean isTrailingSpace(final int start) {
+    for (int i = start; i < myBufferEnd; i += 2) {
+      final char c = myBuffer.charAt(i);
+      if (c != '\\') return false;
+      if (i == myBufferEnd - 1) return false;
+      if (myBuffer.charAt(i + 1) != ' ') return false;
+    }
+    return true;
+  }
+
+  @Override
+  public final int getTokenStart() {
+    assert myStart < myEnd || (myStart == myEnd && myEnd == myBufferEnd);
+    return myStart;
+  }
+
+  @Override
+  public final int getTokenEnd() {
+    if (!(myStart < myEnd || (myStart == myEnd && myEnd == myBufferEnd))) {
+      LOG.error(
+        "myStart=" + myStart + " myEnd=" + myEnd + " myBufferEnd=" + myBufferEnd + " text=" + myBuffer.subSequence(myStart, myBufferEnd));
+    }
+    return myEnd;
+  }
+
+  @Override
+  public final int getBufferEnd() {
+    return myBufferEnd;
+  }
+
+  @Override
+  @NotNull
+  public final CharSequence getBufferSequence() {
+    return myBuffer;
+  }
+
+  protected abstract int locateToken(int start);
+
+  protected final int locateEscapeSequence(int start) {
+    assert myBuffer.charAt(start) == '\\';
+    int i = start;
+    i++;
+    if (isRaw()) return i;
+    if (i == myBufferEnd) {
+      return i;
+    }
+
+    // is octal?
+    if (myBuffer.charAt(i) >= '0' && myBuffer.charAt(i) <= '7') {
+      char first = myBuffer.charAt(i);
+      i++;
+      if (i < myBufferEnd && myBuffer.charAt(i) >= '0' && myBuffer.charAt(i) <= '7') {
+        i++;
+        if (i < myBufferEnd && first <= '3' && myBuffer.charAt(i) >= '0' && myBuffer.charAt(i) <= '7') {
+          i++;
+        }
+      }
+      return i;
+    }
+
+    // \xNN byte escape
+    if (myBuffer.charAt(i) == 'x') {
+      i++;
+      for (; i < start + 4; i++) {
+        if (isEscapeEnd(i)) {
+          return i;
+        }
+      }
+      return i;
+    }
+
+    // unicode escape
+    if (myBuffer.charAt(i) == 'u' || myBuffer.charAt(i) == 'U') {
+      final int width = myBuffer.charAt(i) == 'u' ? 4 : 8; // is it uNNNN or Unnnnnnnn
+      i++;
+      for (; i < start + width + 2; i++) {
+        if (isEscapeEnd(i)) {
+          return i;
+        }
+      }
+      return i;
+    }
+
+    if (myBuffer.charAt(i) == 'N' && isUnicodeMode()) {
+      i++;
+      while (i < myBufferEnd && myBuffer.charAt(i) != '}' && myBuffer.charAt(i) != '\\') {
+        i++;
+      }
+      if (i < myBufferEnd && myBuffer.charAt(i) == '}') {
+        i++;
+      }
+      return i;
+    }
+    return i + 1;
+  }
+
+  protected boolean isEscapeEnd(int offset) {
+    return offset == myBufferEnd || myBuffer.charAt(offset) == '\n' || myBuffer.charAt(offset) == '\\';
+  }
+
+  @Override
+  public void advance() {
+    myStart = myEnd;
+    myEnd = locateToken(myStart);
+    if (!(myStart < myEnd || (myStart == myEnd && myEnd == myBufferEnd))) {
+      LOG.warn("Inconsistent: start " + myStart + ", end " + myEnd + ", buf end " + myBufferEnd);
+    }
+    //assert myStart < myEnd || (myStart == myEnd && myEnd == myBufferEnd) : "Inconsistent: start " + myStart + ", end " + myEnd + ", buf end " + myBufferEnd;
+  }
+}

python/testData/highlighting/escapedWhitespacesHighlighting.py

@@ -0,0 +1 @@
+s = f"\ \ "

python/testSrc/com/jetbrains/python/PythonHighlightingLexerTest.java

@@ -16,6 +16,7 @@
 package com.jetbrains.python;
 
 import com.jetbrains.python.fixtures.PyLexerTestCase;
+import com.jetbrains.python.highlighting.PyHighlighter;
 import com.jetbrains.python.lexer.PythonHighlightingLexer;
 import com.jetbrains.python.psi.LanguageLevel;
 
@@ -177,7 +178,49 @@ public class PythonHighlightingLexerTest extends PyLexerTestCase {
     doTest(LanguageLevel.PYTHON34, "expr = br'raw bytes'", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_STRING");
   }
 
+  // PY-31758
+  public void testFStringEscapeSequences() {
+    doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\nbar'", 
+                             "Py:FSTRING_START", "Py:FSTRING_TEXT", "VALID_STRING_ESCAPE_TOKEN", "Py:FSTRING_TEXT", "Py:FSTRING_END");
+    doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\\nbar'", 
+                             "Py:FSTRING_START", "Py:FSTRING_TEXT", "VALID_STRING_ESCAPE_TOKEN", "Py:FSTRING_TEXT", "Py:FSTRING_END");
+    doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\u0041bar'", 
+                             "Py:FSTRING_START", "Py:FSTRING_TEXT", "VALID_STRING_ESCAPE_TOKEN", "Py:FSTRING_TEXT", "Py:FSTRING_END");
+    doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\x41bar'", 
+                             "Py:FSTRING_START", "Py:FSTRING_TEXT", "VALID_STRING_ESCAPE_TOKEN", "Py:FSTRING_TEXT", "Py:FSTRING_END");
+    doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\101bar'", 
+                             "Py:FSTRING_START", "Py:FSTRING_TEXT", "VALID_STRING_ESCAPE_TOKEN", "Py:FSTRING_TEXT", "Py:FSTRING_END");
+    doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\N{GREEK SMALL LETTER ALPHA}bar'", 
+                             "Py:FSTRING_START", "Py:FSTRING_TEXT", "VALID_STRING_ESCAPE_TOKEN", "Py:FSTRING_TEXT", "Py:FSTRING_END");
+
+    doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\",
+                             "Py:FSTRING_START", "Py:FSTRING_TEXT", "INVALID_CHARACTER_ESCAPE_TOKEN");
+
+    doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\u00'", 
+                             "Py:FSTRING_START", "Py:FSTRING_TEXT", "INVALID_UNICODE_ESCAPE_TOKEN", "Py:FSTRING_END");
+    doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\uZZZZbar'", 
+                             "Py:FSTRING_START", "Py:FSTRING_TEXT", "INVALID_UNICODE_ESCAPE_TOKEN", "Py:FSTRING_TEXT", "Py:FSTRING_END");
+
+    doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\x0'", 
+                             "Py:FSTRING_START", "Py:FSTRING_TEXT", "INVALID_UNICODE_ESCAPE_TOKEN", "Py:FSTRING_END");
+    doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\xZZbar'", 
+                             "Py:FSTRING_START", "Py:FSTRING_TEXT", "INVALID_UNICODE_ESCAPE_TOKEN", "Py:FSTRING_TEXT", "Py:FSTRING_END");
+
+    doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\10'",
+                             "Py:FSTRING_START", "Py:FSTRING_TEXT", "VALID_STRING_ESCAPE_TOKEN", "Py:FSTRING_END");
+    doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\777'",
+                             "Py:FSTRING_START", "Py:FSTRING_TEXT", "VALID_STRING_ESCAPE_TOKEN", "Py:FSTRING_TEXT", "Py:FSTRING_END");
+    
+    doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\N{GREEK SMALL LETTER ALPHA'", 
+                             "Py:FSTRING_START", "Py:FSTRING_TEXT", "INVALID_UNICODE_ESCAPE_TOKEN", "Py:FSTRING_END");
+
+  }
+
   private static void doTest(LanguageLevel languageLevel, String text, String... expectedTokens) {
     PyLexerTestCase.doLexerTest(text, new PythonHighlightingLexer(languageLevel), expectedTokens);
   }
+
+  private static void doTestStringHighlighting(LanguageLevel languageLevel, String text, String... expectedTokens) {
+    PyLexerTestCase.doLexerTest(text, new PyHighlighter(languageLevel).getHighlightingLexer(), expectedTokens);
+  }
 }