diff --git a/python/src/com/jetbrains/python/highlighting/PyHighlighter.java b/python/src/com/jetbrains/python/highlighting/PyHighlighter.java index 3ef0def6e036..28a5e0581c97 100644 --- a/python/src/com/jetbrains/python/highlighting/PyHighlighter.java +++ b/python/src/com/jetbrains/python/highlighting/PyHighlighter.java @@ -10,6 +10,7 @@ import com.intellij.psi.StringEscapesTokenTypes; import com.intellij.psi.tree.IElementType; import com.jetbrains.python.PyTokenTypes; import com.jetbrains.python.PythonDialectsTokenSetProvider; +import com.jetbrains.python.lexer.PyFStringLiteralLexer; import com.jetbrains.python.lexer.PyStringLiteralLexer; import com.jetbrains.python.lexer.PythonHighlightingLexer; import com.jetbrains.python.psi.LanguageLevel; @@ -47,6 +48,10 @@ public class PyHighlighter extends SyntaxHighlighterBase { new PyStringLiteralLexer(PyTokenTypes.TRIPLE_QUOTED_UNICODE), PyTokenTypes.TRIPLE_QUOTED_UNICODE ); + ret.registerLayer( + new PyFStringLiteralLexer(), + PyTokenTypes.FSTRING_TEXT + ); return ret; } diff --git a/python/src/com/jetbrains/python/lexer/PyFStringLiteralLexer.kt b/python/src/com/jetbrains/python/lexer/PyFStringLiteralLexer.kt new file mode 100644 index 000000000000..2ee57838126b --- /dev/null +++ b/python/src/com/jetbrains/python/lexer/PyFStringLiteralLexer.kt @@ -0,0 +1,28 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.jetbrains.python.lexer + +import com.intellij.util.text.CharArrayUtil +import com.jetbrains.python.PyTokenTypes + +class PyFStringLiteralLexer: PyStringLiteralLexerBase(PyTokenTypes.FSTRING_TEXT) { + override fun locateToken(start: Int): Int { + if (start >= myBufferEnd) { + return myBufferEnd + } + + if (myBuffer[start] == '\\') { + return locateEscapeSequence(start) + } + else { + val nextBackslashOffset = CharArrayUtil.indexOf(myBuffer, "\\", start + 1, myBufferEnd) + return if (nextBackslashOffset >= 0) nextBackslashOffset else myBufferEnd + } + } + + // TODO actually keep track of "raw" prefixes of f-strings somehow + override fun isRaw(): Boolean = false + + override fun isUnicodeMode(): Boolean = true + + override fun getState(): Int = myBaseLexerState +} diff --git a/python/src/com/jetbrains/python/lexer/PyStringLiteralLexer.java b/python/src/com/jetbrains/python/lexer/PyStringLiteralLexer.java index 79f78dd40144..0699e6d48463 100644 --- a/python/src/com/jetbrains/python/lexer/PyStringLiteralLexer.java +++ b/python/src/com/jetbrains/python/lexer/PyStringLiteralLexer.java @@ -1,10 +1,6 @@ // Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. package com.jetbrains.python.lexer; -import com.intellij.lexer.LexerBase; -import com.intellij.openapi.diagnostic.Logger; -import com.intellij.openapi.util.text.StringUtil; -import com.intellij.psi.StringEscapesTokenTypes; import com.intellij.psi.tree.IElementType; import com.jetbrains.python.PyTokenTypes; import com.jetbrains.python.psi.PyStringLiteralUtil; @@ -17,48 +13,33 @@ import org.jetbrains.annotations.NotNull; * Lexes the entire string, with u/b/r designator, quotes, and content, thus self-adjusts for the format. * User: dcheryasov */ -public class PyStringLiteralLexer extends LexerBase { - private static final Logger LOG = Logger.getInstance("#com.jetbrains.python.lexer.PyStringLiteralLexer"); +public class PyStringLiteralLexer extends PyStringLiteralLexerBase { private static final short BEFORE_FIRST_QUOTE = 0; // the initial state; may last during 'u' and 'r' prefixes. private static final short AFTER_FIRST_QUOTE = 1; private static final short AFTER_LAST_QUOTE = 2; - private CharSequence myBuffer; - private int myStart; - private int myEnd; private int myState; private int myLastState; - private int myBufferEnd; private char myQuoteChar; - private boolean myIsRaw; private boolean myIsTriple; - private boolean myIsFormatted; - private final IElementType myOriginalLiteralToken; - private boolean mySeenEscapedSpacesOnly; - /** * @param originalLiteralToken the AST node we're layering over. */ public PyStringLiteralLexer(final IElementType originalLiteralToken) { - myOriginalLiteralToken = originalLiteralToken; + super(originalLiteralToken); } @Override - public void start(@NotNull CharSequence buffer, int startOffset, int endOffset, int initialState) { - myBuffer = buffer; - myStart = startOffset; + protected void handleStart(@NotNull CharSequence buffer, int initialState) { myState = initialState; myLastState = initialState; - mySeenEscapedSpacesOnly = true; - myBufferEnd = endOffset; - + // the following could be parsing steps if we wanted this info as tokens final String prefix = PyStringLiteralUtil.getPrefix(buffer, myStart); - myIsFormatted = PyStringLiteralUtil.isFormattedPrefix(prefix); myIsRaw = PyStringLiteralUtil.isRawPrefix(prefix); final int quoteOffset = myStart + prefix.length(); @@ -68,9 +49,27 @@ public class PyStringLiteralLexer extends LexerBase { myQuoteChar = c; myIsTriple = (buffer.length() > quoteOffset + 2) && (buffer.charAt(quoteOffset + 1) == c) && (buffer.charAt(quoteOffset + 2) == c); + } - // calculate myEnd at last - myEnd = locateToken(myStart); + @Override + protected boolean isRaw() { + return myIsRaw; + } + + @Override + protected boolean isUnicodeMode() { + return PyTokenTypes.UNICODE_NODES.contains(myOriginalLiteralToken); + } + + @Override + protected boolean isEscapeEnd(int offset) { + return super.isEscapeEnd(offset) || myBuffer.charAt(offset) == myQuoteChar; + } + + @Override + public void advance() { + myLastState = myState; + super.advance(); } @Override @@ -79,183 +78,27 @@ public class PyStringLiteralLexer extends LexerBase { } @Override - public IElementType getTokenType() { - if (myStart >= myEnd) return null; - - // skip non-escapes immediately - if (myBuffer.charAt(myStart) != '\\' || (myIsRaw && (!isUnicodeMode() || !nextIsUnicodeEscape()))) { - mySeenEscapedSpacesOnly = false; - return myOriginalLiteralToken; - } - - // from here on, only escapes - if (myStart + 1 >= myEnd) return StringEscapesTokenTypes.INVALID_CHARACTER_ESCAPE_TOKEN; // escape ends too early - char nextChar = myBuffer.charAt(myStart + 1); - mySeenEscapedSpacesOnly &= nextChar == ' '; - if ((nextChar == '\n' || nextChar == ' ' && (mySeenEscapedSpacesOnly || isTrailingSpace(myStart+2)))) { - return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN; // escaped EOL - } - if (nextChar == 'u' || nextChar == 'U') { - if (isUnicodeMode()) { - final int width = nextChar == 'u'? 4 : 8; // is it uNNNN or Unnnnnnnn - for(int i = myStart + 2; i < myStart + width + 2; i++) { - if (i >= myEnd || !StringUtil.isHexDigit(myBuffer.charAt(i))) return StringEscapesTokenTypes.INVALID_UNICODE_ESCAPE_TOKEN; - } - return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN; - } - else return myOriginalLiteralToken; // b"\u1234" is just b"\\u1234", nothing gets escaped - } - - if (nextChar == 'x') { // \xNN is allowed both in bytes and unicode. - for(int i = myStart + 2; i < myStart + 4; i++) { - if (i >= myEnd || !StringUtil.isHexDigit(myBuffer.charAt(i))) return StringEscapesTokenTypes.INVALID_UNICODE_ESCAPE_TOKEN; - } - return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN; - } - - if (nextChar == 'N' && isUnicodeMode()) { - int i = myStart+2; - if (i >= myEnd || myBuffer.charAt(i) != '{') return StringEscapesTokenTypes.INVALID_UNICODE_ESCAPE_TOKEN; - i++; - while(i < myEnd && myBuffer.charAt(i) != '}') i++; - if (i >= myEnd) return StringEscapesTokenTypes.INVALID_UNICODE_ESCAPE_TOKEN; - return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN; - } - - switch (nextChar) { - case 'a': - case 'b': - case 'f': - case 'n': - case 'r': - case 't': - case 'v': - case '\'': - case '\"': - case '\\': - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN; - } - - // other unrecognized escapes are just part of string, not an error - return myOriginalLiteralToken; - } - - private boolean nextIsUnicodeEscape() { - if (myStart + 1 < myEnd) { - char nextChar = myBuffer.charAt(myStart + 1); - return nextChar == 'u' || nextChar == 'U'; - } - return false; - } - - private boolean isUnicodeMode() { - return PyTokenTypes.UNICODE_NODES.contains(myOriginalLiteralToken); - } - - // all subsequent chars are escaped spaces - private boolean isTrailingSpace(final int start) { - for (int i=start; i= myEnd) return null; + + // skip non-escapes immediately + if (myBuffer.charAt(myStart) != '\\' || (isRaw() && !(isUnicodeMode() && nextIsUnicodeEscape()))) { + mySeenEscapedSpacesOnly = false; + return myOriginalLiteralToken; + } + + // from here on, only escapes + return getEscapeSequenceType(); + } + + @NotNull + public final IElementType getEscapeSequenceType() { + if (myStart + 1 >= myEnd) return StringEscapesTokenTypes.INVALID_CHARACTER_ESCAPE_TOKEN; // escape ends too early + char nextChar = myBuffer.charAt(myStart + 1); + mySeenEscapedSpacesOnly &= nextChar == ' '; + if ((nextChar == '\n' || nextChar == ' ' && (mySeenEscapedSpacesOnly || isTrailingSpace(myStart + 2)))) { + return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN; // escaped EOL + } + if (nextChar == 'u' || nextChar == 'U') { + if (isUnicodeMode()) { + final int width = nextChar == 'u' ? 4 : 8; // is it uNNNN or Unnnnnnnn + for (int i = myStart + 2; i < myStart + width + 2; i++) { + if (i >= myEnd || !StringUtil.isHexDigit(myBuffer.charAt(i))) return StringEscapesTokenTypes.INVALID_UNICODE_ESCAPE_TOKEN; + } + return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN; + } + else { + return myOriginalLiteralToken; // b"\u1234" is just b"\\u1234", nothing gets escaped + } + } + + if (nextChar == 'x') { // \xNN is allowed both in bytes and unicode. + for (int i = myStart + 2; i < myStart + 4; i++) { + if (i >= myEnd || !StringUtil.isHexDigit(myBuffer.charAt(i))) return StringEscapesTokenTypes.INVALID_UNICODE_ESCAPE_TOKEN; + } + return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN; + } + + if (nextChar == 'N' && isUnicodeMode()) { + int i = myStart + 2; + if (i >= myEnd || myBuffer.charAt(i) != '{') return StringEscapesTokenTypes.INVALID_UNICODE_ESCAPE_TOKEN; + i++; + while (i < myEnd && myBuffer.charAt(i) != '}') i++; + if (i >= myEnd) return StringEscapesTokenTypes.INVALID_UNICODE_ESCAPE_TOKEN; + return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN; + } + + switch (nextChar) { + case 'a': + case 'b': + case 'f': + case 'n': + case 'r': + case 't': + case 'v': + case '\'': + case '\"': + case '\\': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN; + } + + // other unrecognized escapes are just part of string, not an error + return myOriginalLiteralToken; + } + + private boolean nextIsUnicodeEscape() { + if (myStart + 1 < myEnd) { + char nextChar = myBuffer.charAt(myStart + 1); + return nextChar == 'u' || nextChar == 'U'; + } + return false; + } + + // all subsequent chars are escaped spaces + private boolean isTrailingSpace(final int start) { + for (int i = start; i < myBufferEnd; i += 2) { + final char c = myBuffer.charAt(i); + if (c != '\\') return false; + if (i == myBufferEnd - 1) return false; + if (myBuffer.charAt(i + 1) != ' ') return false; + } + return true; + } + + @Override + public final int getTokenStart() { + assert myStart < myEnd || (myStart == myEnd && myEnd == myBufferEnd); + return myStart; + } + + @Override + public final int getTokenEnd() { + if (!(myStart < myEnd || (myStart == myEnd && myEnd == myBufferEnd))) { + LOG.error( + "myStart=" + myStart + " myEnd=" + myEnd + " myBufferEnd=" + myBufferEnd + " text=" + myBuffer.subSequence(myStart, myBufferEnd)); + } + return myEnd; + } + + @Override + public final int getBufferEnd() { + return myBufferEnd; + } + + @Override + @NotNull + public final CharSequence getBufferSequence() { + return myBuffer; + } + + protected abstract int locateToken(int start); + + protected final int locateEscapeSequence(int start) { + assert myBuffer.charAt(start) == '\\'; + int i = start; + i++; + if (isRaw()) return i; + if (i == myBufferEnd) { + return i; + } + + // is octal? + if (myBuffer.charAt(i) >= '0' && myBuffer.charAt(i) <= '7') { + char first = myBuffer.charAt(i); + i++; + if (i < myBufferEnd && myBuffer.charAt(i) >= '0' && myBuffer.charAt(i) <= '7') { + i++; + if (i < myBufferEnd && first <= '3' && myBuffer.charAt(i) >= '0' && myBuffer.charAt(i) <= '7') { + i++; + } + } + return i; + } + + // \xNN byte escape + if (myBuffer.charAt(i) == 'x') { + i++; + for (; i < start + 4; i++) { + if (isEscapeEnd(i)) { + return i; + } + } + return i; + } + + // unicode escape + if (myBuffer.charAt(i) == 'u' || myBuffer.charAt(i) == 'U') { + final int width = myBuffer.charAt(i) == 'u' ? 4 : 8; // is it uNNNN or Unnnnnnnn + i++; + for (; i < start + width + 2; i++) { + if (isEscapeEnd(i)) { + return i; + } + } + return i; + } + + if (myBuffer.charAt(i) == 'N' && isUnicodeMode()) { + i++; + while (i < myBufferEnd && myBuffer.charAt(i) != '}' && myBuffer.charAt(i) != '\\') { + i++; + } + if (i < myBufferEnd && myBuffer.charAt(i) == '}') { + i++; + } + return i; + } + return i + 1; + } + + protected boolean isEscapeEnd(int offset) { + return offset == myBufferEnd || myBuffer.charAt(offset) == '\n' || myBuffer.charAt(offset) == '\\'; + } + + @Override + public void advance() { + myStart = myEnd; + myEnd = locateToken(myStart); + if (!(myStart < myEnd || (myStart == myEnd && myEnd == myBufferEnd))) { + LOG.warn("Inconsistent: start " + myStart + ", end " + myEnd + ", buf end " + myBufferEnd); + } + //assert myStart < myEnd || (myStart == myEnd && myEnd == myBufferEnd) : "Inconsistent: start " + myStart + ", end " + myEnd + ", buf end " + myBufferEnd; + } +} diff --git a/python/testData/highlighting/escapedWhitespacesHighlighting.py b/python/testData/highlighting/escapedWhitespacesHighlighting.py new file mode 100644 index 000000000000..a1083e8064e6 --- /dev/null +++ b/python/testData/highlighting/escapedWhitespacesHighlighting.py @@ -0,0 +1 @@ +s = f"\ \ " diff --git a/python/testSrc/com/jetbrains/python/PythonHighlightingLexerTest.java b/python/testSrc/com/jetbrains/python/PythonHighlightingLexerTest.java index 9333c54c51d8..b00fbc560a86 100644 --- a/python/testSrc/com/jetbrains/python/PythonHighlightingLexerTest.java +++ b/python/testSrc/com/jetbrains/python/PythonHighlightingLexerTest.java @@ -16,6 +16,7 @@ package com.jetbrains.python; import com.jetbrains.python.fixtures.PyLexerTestCase; +import com.jetbrains.python.highlighting.PyHighlighter; import com.jetbrains.python.lexer.PythonHighlightingLexer; import com.jetbrains.python.psi.LanguageLevel; @@ -177,7 +178,49 @@ public class PythonHighlightingLexerTest extends PyLexerTestCase { doTest(LanguageLevel.PYTHON34, "expr = br'raw bytes'", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_STRING"); } + // PY-31758 + public void testFStringEscapeSequences() { + doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\nbar'", + "Py:FSTRING_START", "Py:FSTRING_TEXT", "VALID_STRING_ESCAPE_TOKEN", "Py:FSTRING_TEXT", "Py:FSTRING_END"); + doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\\nbar'", + "Py:FSTRING_START", "Py:FSTRING_TEXT", "VALID_STRING_ESCAPE_TOKEN", "Py:FSTRING_TEXT", "Py:FSTRING_END"); + doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\u0041bar'", + "Py:FSTRING_START", "Py:FSTRING_TEXT", "VALID_STRING_ESCAPE_TOKEN", "Py:FSTRING_TEXT", "Py:FSTRING_END"); + doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\x41bar'", + "Py:FSTRING_START", "Py:FSTRING_TEXT", "VALID_STRING_ESCAPE_TOKEN", "Py:FSTRING_TEXT", "Py:FSTRING_END"); + doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\101bar'", + "Py:FSTRING_START", "Py:FSTRING_TEXT", "VALID_STRING_ESCAPE_TOKEN", "Py:FSTRING_TEXT", "Py:FSTRING_END"); + doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\N{GREEK SMALL LETTER ALPHA}bar'", + "Py:FSTRING_START", "Py:FSTRING_TEXT", "VALID_STRING_ESCAPE_TOKEN", "Py:FSTRING_TEXT", "Py:FSTRING_END"); + + doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\", + "Py:FSTRING_START", "Py:FSTRING_TEXT", "INVALID_CHARACTER_ESCAPE_TOKEN"); + + doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\u00'", + "Py:FSTRING_START", "Py:FSTRING_TEXT", "INVALID_UNICODE_ESCAPE_TOKEN", "Py:FSTRING_END"); + doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\uZZZZbar'", + "Py:FSTRING_START", "Py:FSTRING_TEXT", "INVALID_UNICODE_ESCAPE_TOKEN", "Py:FSTRING_TEXT", "Py:FSTRING_END"); + + doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\x0'", + "Py:FSTRING_START", "Py:FSTRING_TEXT", "INVALID_UNICODE_ESCAPE_TOKEN", "Py:FSTRING_END"); + doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\xZZbar'", + "Py:FSTRING_START", "Py:FSTRING_TEXT", "INVALID_UNICODE_ESCAPE_TOKEN", "Py:FSTRING_TEXT", "Py:FSTRING_END"); + + doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\10'", + "Py:FSTRING_START", "Py:FSTRING_TEXT", "VALID_STRING_ESCAPE_TOKEN", "Py:FSTRING_END"); + doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\777'", + "Py:FSTRING_START", "Py:FSTRING_TEXT", "VALID_STRING_ESCAPE_TOKEN", "Py:FSTRING_TEXT", "Py:FSTRING_END"); + + doTestStringHighlighting(LanguageLevel.PYTHON36, "f'foo\\N{GREEK SMALL LETTER ALPHA'", + "Py:FSTRING_START", "Py:FSTRING_TEXT", "INVALID_UNICODE_ESCAPE_TOKEN", "Py:FSTRING_END"); + + } + private static void doTest(LanguageLevel languageLevel, String text, String... expectedTokens) { PyLexerTestCase.doLexerTest(text, new PythonHighlightingLexer(languageLevel), expectedTokens); } + + private static void doTestStringHighlighting(LanguageLevel languageLevel, String text, String... expectedTokens) { + PyLexerTestCase.doLexerTest(text, new PyHighlighter(languageLevel).getHighlightingLexer(), expectedTokens); + } }