[java] raw string literals: lexer/parser (IDEA-189075)

java/java-psi-api/src/com/intellij/psi/JavaTokenType.java

@@ -1,18 +1,4 @@
-/*
- * Copyright 2000-2017 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
 package com.intellij.psi;
 
 import com.intellij.psi.tree.IElementType;
@@ -31,6 +17,7 @@ public interface JavaTokenType extends TokenType {
   IElementType DOUBLE_LITERAL = new IJavaElementType("DOUBLE_LITERAL");
   IElementType CHARACTER_LITERAL = new IJavaElementType("CHARACTER_LITERAL");
   IElementType STRING_LITERAL = new IJavaElementType("STRING_LITERAL");
+  IElementType RAW_STRING_LITERAL = new IJavaElementType("RAW_STRING_LITERAL");
 
   IElementType TRUE_KEYWORD = new IKeywordElementType("TRUE_KEYWORD");
   IElementType FALSE_KEYWORD = new IKeywordElementType("FALSE_KEYWORD");

java/java-psi-impl/src/com/intellij/lang/java/lexer/JavaLexer.java

@@ -149,6 +149,11 @@ public class JavaLexer extends LexerBase {
         myTokenEndOffset = getClosingQuote(myBufferIndex + 1, c);
         break;
 
+      case '`':
+        myTokenType = JavaTokenType.RAW_STRING_LITERAL;
+        myTokenEndOffset = getRawLiteralEnd(myBufferIndex);
+        break;
+
       default:
         flexLocateToken();
     }
@@ -246,6 +251,26 @@ public class JavaLexer extends LexerBase {
     return pos;
   }
 
+  private int getRawLiteralEnd(int offset) {
+    int pos = offset;
+
+    while (pos < myBufferEndOffset && charAt(pos) == '`') pos++;
+    int quoteLen = pos - offset;
+
+    outer:
+    while (pos < myBufferEndOffset) {
+      while (pos < myBufferEndOffset && charAt(pos) != '`') pos++;
+
+      int left = quoteLen;
+      while (pos < myBufferEndOffset && charAt(pos) == '`') {
+        pos++;
+        if (--left == 0) break outer;
+      }
+    }
+
+    return pos;
+  }
+
   private char charAt(int position) {
     return myBufferArray != null ? myBufferArray[position] : myBuffer.charAt(position);
   }

java/java-psi-impl/src/com/intellij/lang/java/parser/ExpressionParser.java

@@ -1,6 +1,4 @@
-// Copyright 2000-2017 JetBrains s.r.o.
-// Use of this source code is governed by the Apache 2.0 license that can be
-// found in the LICENSE file.
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
 package com.intellij.lang.java.parser;
 
 import com.intellij.codeInsight.daemon.JavaErrorMessages;
@@ -36,7 +34,7 @@ public class ExpressionParser {
   private static final TokenSet LITERALS = TokenSet.create(
     JavaTokenType.TRUE_KEYWORD, JavaTokenType.FALSE_KEYWORD, JavaTokenType.NULL_KEYWORD, JavaTokenType.INTEGER_LITERAL,
     JavaTokenType.LONG_LITERAL, JavaTokenType.FLOAT_LITERAL, JavaTokenType.DOUBLE_LITERAL, JavaTokenType.CHARACTER_LITERAL,
-    JavaTokenType.STRING_LITERAL);
+    JavaTokenType.STRING_LITERAL, JavaTokenType.RAW_STRING_LITERAL);
   private static final TokenSet CONDITIONAL_OR_OPS = TokenSet.create(JavaTokenType.OROR);
   private static final TokenSet CONDITIONAL_AND_OPS = TokenSet.create(JavaTokenType.ANDAND);
   private static final TokenSet OR_OPS = TokenSet.create(JavaTokenType.OR);

java/java-tests/testData/psi/parser-partial/expressions/RawLiteral0.txt

@@ -0,0 +1,3 @@
+PsiJavaFile:RawLiteral0.java
+  PsiLiteralExpression:`.`
+    PsiJavaToken:RAW_STRING_LITERAL('`.`')

java/java-tests/testSrc/com/intellij/java/lexer/JavaLexerTest.java

@@ -1,18 +1,4 @@
-/*
- * Copyright 2000-2017 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
 package com.intellij.java.lexer;
 
 import com.intellij.lang.java.JavaParserDefinition;
@@ -161,6 +147,12 @@ public class JavaLexerTest extends LexerTestCase {
            "IDENTIFIER ('Ɐ')\nWHITE_SPACE (' ')\nIDENTIFIER ('Σx')\nWHITE_SPACE (' ')\nIDENTIFIER ('dΦ')");
   }
 
+  public void testRawLiterals() {
+    doTest(" ``.`.`` ", "WHITE_SPACE (' ')\nRAW_STRING_LITERAL ('``.`.``')\nWHITE_SPACE (' ')");
+    doTest(" ``.`.` ", "WHITE_SPACE (' ')\nRAW_STRING_LITERAL ('``.`.` ')");
+    doTest(" ``.`.``` ", "WHITE_SPACE (' ')\nRAW_STRING_LITERAL ('``.`.``')\nRAW_STRING_LITERAL ('` ')");
+  }
+
   @Override
   protected Lexer createLexer() {
     return JavaParserDefinition.createLexer(LanguageLevel.HIGHEST);
@@ -170,4 +162,4 @@ public class JavaLexerTest extends LexerTestCase {
   protected String getDirPath() {
     return "";
   }
-}
+}

java/java-tests/testSrc/com/intellij/java/parser/partial/ExpressionParserTest.java

@@ -144,6 +144,8 @@ public class ExpressionParserTest extends JavaParsingTestCase {
   public void testLambdaExpression26() { doParserTest("(var var) -> var"); }
   public void testAmbiguousLambdaExpression() { doParserTest("f( (x) < y , z > (w) -> v )"); }
 
+  public void testRawLiteral0() { doParserTest("`.`"); }
+
   private void doParserTest(String text) {
     doParserTest(text, builder -> JavaParser.INSTANCE.getExpressionParser().parse(builder));
   }