From b3aaaca70a2ccdd326683d065366ddf8817cc884 Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Tue, 10 Apr 2018 19:05:22 +0200 Subject: [PATCH] [java] raw string literals: lexer/parser (IDEA-189075) --- .../src/com/intellij/psi/JavaTokenType.java | 17 ++----------- .../intellij/lang/java/lexer/JavaLexer.java | 25 +++++++++++++++++++ .../lang/java/parser/ExpressionParser.java | 6 ++--- .../expressions/RawLiteral0.txt | 3 +++ .../intellij/java/lexer/JavaLexerTest.java | 24 ++++++------------ .../parser/partial/ExpressionParserTest.java | 2 ++ 6 files changed, 42 insertions(+), 35 deletions(-) create mode 100644 java/java-tests/testData/psi/parser-partial/expressions/RawLiteral0.txt diff --git a/java/java-psi-api/src/com/intellij/psi/JavaTokenType.java b/java/java-psi-api/src/com/intellij/psi/JavaTokenType.java index 03c33d0572ad..255c1806bb85 100644 --- a/java/java-psi-api/src/com/intellij/psi/JavaTokenType.java +++ b/java/java-psi-api/src/com/intellij/psi/JavaTokenType.java @@ -1,18 +1,4 @@ -/* - * Copyright 2000-2017 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. package com.intellij.psi; import com.intellij.psi.tree.IElementType; @@ -31,6 +17,7 @@ public interface JavaTokenType extends TokenType { IElementType DOUBLE_LITERAL = new IJavaElementType("DOUBLE_LITERAL"); IElementType CHARACTER_LITERAL = new IJavaElementType("CHARACTER_LITERAL"); IElementType STRING_LITERAL = new IJavaElementType("STRING_LITERAL"); + IElementType RAW_STRING_LITERAL = new IJavaElementType("RAW_STRING_LITERAL"); IElementType TRUE_KEYWORD = new IKeywordElementType("TRUE_KEYWORD"); IElementType FALSE_KEYWORD = new IKeywordElementType("FALSE_KEYWORD"); diff --git a/java/java-psi-impl/src/com/intellij/lang/java/lexer/JavaLexer.java b/java/java-psi-impl/src/com/intellij/lang/java/lexer/JavaLexer.java index 71667fa8b9aa..8cfb42de8591 100644 --- a/java/java-psi-impl/src/com/intellij/lang/java/lexer/JavaLexer.java +++ b/java/java-psi-impl/src/com/intellij/lang/java/lexer/JavaLexer.java @@ -149,6 +149,11 @@ public class JavaLexer extends LexerBase { myTokenEndOffset = getClosingQuote(myBufferIndex + 1, c); break; + case '`': + myTokenType = JavaTokenType.RAW_STRING_LITERAL; + myTokenEndOffset = getRawLiteralEnd(myBufferIndex); + break; + default: flexLocateToken(); } @@ -246,6 +251,26 @@ public class JavaLexer extends LexerBase { return pos; } + private int getRawLiteralEnd(int offset) { + int pos = offset; + + while (pos < myBufferEndOffset && charAt(pos) == '`') pos++; + int quoteLen = pos - offset; + + outer: + while (pos < myBufferEndOffset) { + while (pos < myBufferEndOffset && charAt(pos) != '`') pos++; + + int left = quoteLen; + while (pos < myBufferEndOffset && charAt(pos) == '`') { + pos++; + if (--left == 0) break outer; + } + } + + return pos; + } + private char charAt(int position) { return myBufferArray != null ? myBufferArray[position] : myBuffer.charAt(position); } diff --git a/java/java-psi-impl/src/com/intellij/lang/java/parser/ExpressionParser.java b/java/java-psi-impl/src/com/intellij/lang/java/parser/ExpressionParser.java index 3c51536d9d24..bf378b12cd9b 100644 --- a/java/java-psi-impl/src/com/intellij/lang/java/parser/ExpressionParser.java +++ b/java/java-psi-impl/src/com/intellij/lang/java/parser/ExpressionParser.java @@ -1,6 +1,4 @@ -// Copyright 2000-2017 JetBrains s.r.o. -// Use of this source code is governed by the Apache 2.0 license that can be -// found in the LICENSE file. +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. package com.intellij.lang.java.parser; import com.intellij.codeInsight.daemon.JavaErrorMessages; @@ -36,7 +34,7 @@ public class ExpressionParser { private static final TokenSet LITERALS = TokenSet.create( JavaTokenType.TRUE_KEYWORD, JavaTokenType.FALSE_KEYWORD, JavaTokenType.NULL_KEYWORD, JavaTokenType.INTEGER_LITERAL, JavaTokenType.LONG_LITERAL, JavaTokenType.FLOAT_LITERAL, JavaTokenType.DOUBLE_LITERAL, JavaTokenType.CHARACTER_LITERAL, - JavaTokenType.STRING_LITERAL); + JavaTokenType.STRING_LITERAL, JavaTokenType.RAW_STRING_LITERAL); private static final TokenSet CONDITIONAL_OR_OPS = TokenSet.create(JavaTokenType.OROR); private static final TokenSet CONDITIONAL_AND_OPS = TokenSet.create(JavaTokenType.ANDAND); private static final TokenSet OR_OPS = TokenSet.create(JavaTokenType.OR); diff --git a/java/java-tests/testData/psi/parser-partial/expressions/RawLiteral0.txt b/java/java-tests/testData/psi/parser-partial/expressions/RawLiteral0.txt new file mode 100644 index 000000000000..08d52f5e0acd --- /dev/null +++ b/java/java-tests/testData/psi/parser-partial/expressions/RawLiteral0.txt @@ -0,0 +1,3 @@ +PsiJavaFile:RawLiteral0.java + PsiLiteralExpression:`.` + PsiJavaToken:RAW_STRING_LITERAL('`.`') \ No newline at end of file diff --git a/java/java-tests/testSrc/com/intellij/java/lexer/JavaLexerTest.java b/java/java-tests/testSrc/com/intellij/java/lexer/JavaLexerTest.java index 73834417d8f2..37b0f67d8850 100644 --- a/java/java-tests/testSrc/com/intellij/java/lexer/JavaLexerTest.java +++ b/java/java-tests/testSrc/com/intellij/java/lexer/JavaLexerTest.java @@ -1,18 +1,4 @@ -/* - * Copyright 2000-2017 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. package com.intellij.java.lexer; import com.intellij.lang.java.JavaParserDefinition; @@ -161,6 +147,12 @@ public class JavaLexerTest extends LexerTestCase { "IDENTIFIER ('Ɐ')\nWHITE_SPACE (' ')\nIDENTIFIER ('Σx')\nWHITE_SPACE (' ')\nIDENTIFIER ('dΦ')"); } + public void testRawLiterals() { + doTest(" ``.`.`` ", "WHITE_SPACE (' ')\nRAW_STRING_LITERAL ('``.`.``')\nWHITE_SPACE (' ')"); + doTest(" ``.`.` ", "WHITE_SPACE (' ')\nRAW_STRING_LITERAL ('``.`.` ')"); + doTest(" ``.`.``` ", "WHITE_SPACE (' ')\nRAW_STRING_LITERAL ('``.`.``')\nRAW_STRING_LITERAL ('` ')"); + } + @Override protected Lexer createLexer() { return JavaParserDefinition.createLexer(LanguageLevel.HIGHEST); @@ -170,4 +162,4 @@ public class JavaLexerTest extends LexerTestCase { protected String getDirPath() { return ""; } -} +} \ No newline at end of file diff --git a/java/java-tests/testSrc/com/intellij/java/parser/partial/ExpressionParserTest.java b/java/java-tests/testSrc/com/intellij/java/parser/partial/ExpressionParserTest.java index 6ed647fe71c4..f9a6311c86af 100644 --- a/java/java-tests/testSrc/com/intellij/java/parser/partial/ExpressionParserTest.java +++ b/java/java-tests/testSrc/com/intellij/java/parser/partial/ExpressionParserTest.java @@ -144,6 +144,8 @@ public class ExpressionParserTest extends JavaParsingTestCase { public void testLambdaExpression26() { doParserTest("(var var) -> var"); } public void testAmbiguousLambdaExpression() { doParserTest("f( (x) < y , z > (w) -> v )"); } + public void testRawLiteral0() { doParserTest("`.`"); } + private void doParserTest(String text) { doParserTest(text, builder -> JavaParser.INSTANCE.getExpressionParser().parse(builder)); }