Java part of IDEA-50496

java/java-psi-impl/src/com/intellij/psi/impl/source/tree/java/PsiLiteralExpressionImpl.java

@@ -50,7 +50,7 @@ public class PsiLiteralExpressionImpl
 
   @Override
   public PsiType getType() {
-    final IElementType type = getFirstChildNode().getElementType();
+    final IElementType type = getLiteralElementType();
     if (type == JavaTokenType.INTEGER_LITERAL) {
       return PsiType.INT;
     }
@@ -80,6 +80,10 @@ public class PsiLiteralExpressionImpl
     return null;
   }
 
+  public IElementType getLiteralElementType() {
+    return getFirstChildNode().getElementType();
+  }
+
   public String getCanonicalText() {
     final TreeElement literal = getFirstChildNode();
     final IElementType type = literal.getElementType();
@@ -88,8 +92,7 @@ public class PsiLiteralExpressionImpl
 
   @Override
   public Object getValue() {
-    final TreeElement literal = getFirstChildNode();
-    final IElementType type = literal.getElementType();
+    final IElementType type = getLiteralElementType();
     String text = NUMERIC_LITERALS.contains(type) ? getCanonicalText().toLowerCase() : getCanonicalText();
     final int textLength = text.length();
 
@@ -172,19 +175,8 @@ public class PsiLiteralExpressionImpl
       return Character.valueOf(chars.charAt(0));
     }
     if (type == JavaTokenType.STRING_LITERAL) {
-      if (StringUtil.endsWithChar(text, '\"')) {
-        if (textLength == 1) return null;
-        text = text.substring(1, textLength - 1);
-      }
-      else {
-        if (text.startsWith(QUOT) && text.endsWith(QUOT) && textLength > QUOT.length()) {
-          text = text.substring(QUOT.length(), textLength - QUOT.length());
-        }
-        else {
-          return null;
-        }
-      }
-      return internedParseStringCharacters(text);
+      String innerText = getInnerText();
+      return innerText == null ? null : internedParseStringCharacters(text);
     }
     if (type == JavaTokenType.TRUE_KEYWORD) {
       return Boolean.TRUE;
@@ -196,6 +188,25 @@ public class PsiLiteralExpressionImpl
     return null;
   }
 
+  @Nullable
+  public String getInnerText() {
+    String text = getCanonicalText();
+    int textLength = text.length();
+    if (StringUtil.endsWithChar(text, '\"')) {
+      if (textLength == 1) return null;
+      text = text.substring(1, textLength - 1);
+    }
+    else {
+      if (text.startsWith(QUOT) && text.endsWith(QUOT) && textLength > QUOT.length()) {
+        text = text.substring(QUOT.length(), textLength - QUOT.length());
+      }
+      else {
+        return null;
+      }
+    }
+    return text;
+  }
+
   // convert text to number according to radix specified
   // if number is more than maxBits bits long, throws NumberFormatException
   private static long parseDigits(final String text, final int bitsInRadix, final int maxBits) throws NumberFormatException {

plugins/java-i18n/src/com/intellij/spellchecker/LiteralExpressionTokenizer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2009 JetBrains s.r.o.
+ * Copyright 2000-2012 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,11 +16,13 @@
 package com.intellij.spellchecker;
 
 import com.intellij.codeInsight.AnnotationUtil;
-import com.intellij.psi.PsiClassType;
+import com.intellij.psi.JavaTokenType;
 import com.intellij.psi.PsiLiteralExpression;
 import com.intellij.psi.PsiModifierListOwner;
+import com.intellij.psi.impl.source.tree.java.PsiLiteralExpressionImpl;
 import com.intellij.psi.util.PsiTreeUtil;
 import com.intellij.spellchecker.inspections.PlainTextSplitter;
+import com.intellij.spellchecker.tokenizer.EscapeSequenceTokenizer;
 import com.intellij.spellchecker.tokenizer.TokenConsumer;
 import com.intellij.spellchecker.tokenizer.Tokenizer;
 import org.jetbrains.annotations.NotNull;
@@ -35,7 +37,8 @@ import java.util.Collections;
 public class LiteralExpressionTokenizer extends Tokenizer<PsiLiteralExpression> {
   @Override
   public void tokenize(@NotNull PsiLiteralExpression element, TokenConsumer consumer) {
-    if (!(element.getType() instanceof PsiClassType)) {
+    PsiLiteralExpressionImpl literalExpression = (PsiLiteralExpressionImpl) element;
+    if (literalExpression.getLiteralElementType() != JavaTokenType.STRING_LITERAL) {
       return;  // not a string literal
     }
 
@@ -44,6 +47,23 @@ public class LiteralExpressionTokenizer extends Tokenizer<PsiLiteralExpression>
       return;
     }
 
-    consumer.consumeToken(element, PlainTextSplitter.getInstance());
+    String text = literalExpression.getInnerText();
+    if (text == null) {
+      return;
+    }
+    if (!text.contains("\\")) {
+      consumer.consumeToken(element, PlainTextSplitter.getInstance());
+    }
+    else {
+      processTextWithEscapeSequences(element, text, consumer);
+    }
+  }
+
+  public static void processTextWithEscapeSequences(PsiLiteralExpression element, String text, TokenConsumer consumer) {
+    StringBuilder unescapedText = new StringBuilder();
+    int[] offsets = new int[text.length()+1];
+    PsiLiteralExpressionImpl.parseStringCharacters(text, unescapedText, offsets);
+
+    EscapeSequenceTokenizer.processTextWithOffsets(element, consumer, unescapedText, offsets);
   }
 }

plugins/java-i18n/testData/inspection/stringWithMistakes/idea50496.java

@@ -0,0 +1,19 @@
+/*
+ * Copyright 2000-2012 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+class Idea50496 {
+  public static final String FOO = "\nclass";
+  public static final String s2 = "\n<TYPO descr="Typo: In word 'fooquux'">fooquux</TYPO>";
+}

plugins/java-i18n/testSrc/com/intellij/spellchecker/inspection/ClassNameWithMistakesInspectionTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2009 JetBrains s.r.o.
+ * Copyright 2000-2012 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -26,7 +26,7 @@ public class ClassNameWithMistakesInspectionTest extends JavaSpellcheckerInspect
   }
 
   public void testJava() throws Throwable {
-    doTest("TestUpgade.java", SpellcheckerInspectionTestCase.getInspectionTools());
+    doTest("TestUpgade.java");
   }
 
   

plugins/java-i18n/testSrc/com/intellij/spellchecker/inspection/DocCommentWithMistakesInspectionTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2009 JetBrains s.r.o.
+ * Copyright 2000-2012 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -28,6 +28,6 @@ public class DocCommentWithMistakesInspectionTest extends JavaSpellcheckerInspec
   }
 
   public void testJava() throws Throwable {
-    doTest("SPITest6.java", SpellcheckerInspectionTestCase.getInspectionTools());
+    doTest("SPITest6.java");
   }
 }

plugins/java-i18n/testSrc/com/intellij/spellchecker/inspection/FieldNameWithMistakesInspectionTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2009 JetBrains s.r.o.
+ * Copyright 2000-2012 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -28,7 +28,7 @@ public class FieldNameWithMistakesInspectionTest extends JavaSpellcheckerInspect
   }
 
   public void testJava() throws Throwable {
-    doTest("SPITest2.java", SpellcheckerInspectionTestCase.getInspectionTools());
+    doTest("SPITest2.java");
   }
 
 }

plugins/java-i18n/testSrc/com/intellij/spellchecker/inspection/JavaSpellcheckerInspectionTestCase.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2010 JetBrains s.r.o.
+ * Copyright 2000-2012 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,7 +15,6 @@
  */
 package com.intellij.spellchecker.inspection;
 
-import com.intellij.codeInspection.LocalInspectionTool;
 import com.intellij.openapi.application.PathManager;
 import com.intellij.openapi.application.PluginPathManager;
 import com.intellij.openapi.util.io.FileUtil;
@@ -30,8 +29,8 @@ public abstract class JavaSpellcheckerInspectionTestCase extends JavaCodeInsight
   @NonNls
   protected String DATA_PATH = FileUtil.toSystemIndependentName(PathManager.getHomePath()) + "/plugins/spellchecker/core/tests/testData";
 
-  protected void doTest(String file, LocalInspectionTool... tools) throws Throwable {
-    myFixture.enableInspections(tools);
+  protected void doTest(String file) {
+    myFixture.enableInspections(SpellcheckerInspectionTestCase.getInspectionTools());
     myFixture.testHighlighting(false, false, true, file);
   }
 

plugins/java-i18n/testSrc/com/intellij/spellchecker/inspection/LiteralExpressionTokenizerTest.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright 2000-2012 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.spellchecker.inspection;
+
+import com.intellij.openapi.util.TextRange;
+import com.intellij.psi.PsiElement;
+import com.intellij.spellchecker.LiteralExpressionTokenizer;
+import com.intellij.spellchecker.inspections.Splitter;
+import com.intellij.spellchecker.tokenizer.TokenConsumer;
+import com.intellij.testFramework.UsefulTestCase;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author yole
+ */
+public class LiteralExpressionTokenizerTest extends UsefulTestCase {
+  private static class TokenCollector extends TokenConsumer {
+    private List<String> myTokenTexts = new ArrayList<String>();
+
+    @Override
+    public void consumeToken(PsiElement element, String text, boolean useRename, int offset, TextRange rangeToCheck, Splitter splitter) {
+      myTokenTexts.add(text);
+    }
+
+    public List<String> getTokenTexts() {
+      return myTokenTexts;
+    }
+  }
+
+  public void testEscapeSequences() {
+    doTest("hello\\nworld", "hello", "world");
+  }
+
+  public void testEscapeSequences2() {
+    doTest("\\nhello\\nworld\\n", "hello", "world");
+  }
+
+  private static void doTest(final String text, final String... expected) {
+    TokenCollector collector = new TokenCollector();
+    LiteralExpressionTokenizer.processTextWithEscapeSequences(null, text, collector);
+    assertOrderedEquals(collector.getTokenTexts(), expected);
+  }
+}

plugins/java-i18n/testSrc/com/intellij/spellchecker/inspection/LocalVariableWithMistakesInspectionTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2009 JetBrains s.r.o.
+ * Copyright 2000-2012 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -27,7 +27,7 @@ public class LocalVariableWithMistakesInspectionTest extends JavaSpellcheckerIns
   }
 
   public void testJava() throws Throwable {
-    doTest("SPITest3.java", SpellcheckerInspectionTestCase.getInspectionTools());
+    doTest("SPITest3.java");
   }
 
 }

plugins/java-i18n/testSrc/com/intellij/spellchecker/inspection/MethodNameWithMistakesInspectionTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2009 JetBrains s.r.o.
+ * Copyright 2000-2012 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -27,7 +27,7 @@ public class MethodNameWithMistakesInspectionTest extends JavaSpellcheckerInspec
   }
 
   public void testJava() throws Throwable {
-    doTest("SPITest4.java", SpellcheckerInspectionTestCase.getInspectionTools());
+    doTest("SPITest4.java");
   }
 
 }

plugins/java-i18n/testSrc/com/intellij/spellchecker/inspection/StringWithMistakesInspectionTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2009 JetBrains s.r.o.
+ * Copyright 2000-2012 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -28,9 +28,11 @@ public class StringWithMistakesInspectionTest extends JavaSpellcheckerInspection
   }
 
 
-  public void testJava() throws Throwable {
-    doTest("SPITest5.java", SpellcheckerInspectionTestCase.getInspectionTools());
+  public void testJava()  {
+    doTest("SPITest5.java");
   }
 
-  
-}
+  public void testEscapeSequence() {
+    doTest("idea50496.java");
+  }
+}

plugins/spellchecker/src/com/intellij/spellchecker/tokenizer/EscapeSequenceTokenizer.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2000-2012 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.spellchecker.tokenizer;
+
+import com.intellij.openapi.util.TextRange;
+import com.intellij.psi.PsiElement;
+import com.intellij.spellchecker.inspections.PlainTextSplitter;
+
+/**
+ * @author yole
+ */
+public class EscapeSequenceTokenizer {
+  public static void processTextWithOffsets(PsiElement element, TokenConsumer consumer, StringBuilder unescapedText,
+                                            int[] offsets) {
+    StringBuilder currentToken = new StringBuilder();
+    int currentTokenStart = 0;
+    for (int i = 0; i < unescapedText.length(); i++) {
+      if (offsets[i+1]-offsets[i] == 1) {
+        if (currentToken.length() == 0) {
+          currentTokenStart = offsets[i];
+        }
+        currentToken.append(unescapedText.charAt(i));
+      }
+      else {
+        if (currentToken.length() > 0) {
+          processCurrentToken(element, currentToken, currentTokenStart, consumer);
+          currentToken.setLength(0);
+        }
+      }
+    }
+    if (currentToken.length() > 0) {
+      processCurrentToken(element, currentToken, currentTokenStart, consumer);
+    }
+  }
+
+  private static void processCurrentToken(PsiElement element,
+                                          StringBuilder currentToken,
+                                          int currentTokenStart, TokenConsumer consumer) {
+    final String token = currentToken.toString();
+    // +1 for the starting quote of the string literal
+    consumer.consumeToken(element, token, false, currentTokenStart+1, TextRange.allOf(token), PlainTextSplitter.getInstance());
+  }
+}

plugins/spellchecker/src/com/intellij/spellchecker/tokenizer/TokenConsumer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2011 JetBrains s.r.o.
+ * Copyright 2000-2012 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -27,11 +27,6 @@ public abstract class TokenConsumer {
     consumeToken(element, false, splitter);
   }
 
-  public void consumeToken(PsiElement element, String value, int offset, Splitter splitter) {
-    String text = element.getText();
-    consumeToken(element, value, false, offset, TextRange.allOf(text), splitter);
-  }
-
   public void consumeToken(PsiElement element, boolean useRename, Splitter splitter) {
     String text = element.getText();
     consumeToken(element, text, useRename, 0, TextRange.allOf(text), splitter);