add -> keyword to cpp and make sure it's lexed correctly (IDEA-121906)

community-resources/src/CommunityFileTypes.xml

@@ -60,6 +60,7 @@
         </options>
         <keywords keywords="#define;#else;#endif;#error;#if;#ifdef;#ifndef;#include;#line;#undef;auto;bool;break;case;char;const;continue;default;delete;do;double;else;enum;extern;false;float;for;goto;if;int;long;register;return;short;signed;sizeof;static;struct;switch;typedef;union;unsigned;void;volatile;while" ignore_case="false" />
         <keywords2 keywords="asm;catch;class;const_cast;dynamic_cast;explicit;friend;inline;mutable;namespace;new;operator;private;protected;public;reinterpret_cast;static_cast;this;throw;true;try;typeid;typename;using;virtual;wchar_t" />
+        <keywords3 keywords="->" />
       </highlighting>
     </filetype>
 

java/java-tests/testSrc/com/intellij/codeInsight/completion/HippieCompletionTest.groovy

@@ -189,7 +189,16 @@ class Xoo {
 class Xoo {
 }
 '''
+  }
 
+  public void "test cpp indirection"() {
+    myFixture.configureByText "a.c", '''f<caret>
+foo->bar
+'''
+    complete()
+    myFixture.checkResult '''foo<caret>
+foo->bar
+'''
   }
 
   private void complete() {

platform/indexing-impl/src/com/intellij/ide/highlighter/custom/CustomFileTypeLexer.java

@@ -50,7 +50,7 @@ public final class CustomFileTypeLexer extends AbstractCustomLexer {
       }
     };
     
-    final IdentifierParser identifierParser = new IdentifierParser();
+    final IdentifierParser identifierParser = new IdentifierParser(parser);
 
     final QuotedStringParser quotedStringParser = new QuotedStringParser("\"", CustomHighlighterTokenType.STRING, table.isHasStringEscapes());
 

platform/indexing-impl/src/com/intellij/ide/highlighter/custom/tokens/IdentifierParser.java

@@ -23,7 +23,10 @@ import com.intellij.psi.tree.IElementType;
  * @author dsl
  */
 public class IdentifierParser extends TokenParser {
-  public IdentifierParser() {
+  private final KeywordParser myKeywordParser;
+
+  public IdentifierParser(KeywordParser keywordParser) {
+    myKeywordParser = keywordParser;
   }
 
   @Override
@@ -31,15 +34,15 @@ public class IdentifierParser extends TokenParser {
     if (!Character.isJavaIdentifierStart(myBuffer.charAt(position))) return false;
     final int start = position;
     for (position++; position < myEndOffset; position++) {
-      final char c = myBuffer.charAt(position);
-      if (!isIdentifierPart(c)) break;
+      if (!isIdentifierPart(position)) break;
     }
     IElementType tokenType = CustomHighlighterTokenType.IDENTIFIER;
     myTokenInfo.updateData(start, position, tokenType);
     return true;
   }
 
-  protected boolean isIdentifierPart(final char c) {
-    return Character.isJavaIdentifierPart(c) || c == '-';
+  private boolean isIdentifierPart(int position) {
+    if (myBuffer.charAt(position) == '-') return !myKeywordParser.hasToken(position, myBuffer, null);
+    return KeywordParser.isWordPart(position, myBuffer);
   }
 }

platform/indexing-impl/src/com/intellij/ide/highlighter/custom/tokens/KeywordParser.java

@@ -23,6 +23,7 @@ import com.intellij.psi.tree.IElementType;
 import com.intellij.util.containers.CharTrie;
 import gnu.trove.THashSet;
 import gnu.trove.TIntHashSet;
+import org.jetbrains.annotations.Nullable;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -63,10 +64,9 @@ public class KeywordParser {
     return result;
   }
 
-  public boolean hasToken(int position, CharSequence myBuffer, TokenInfo myTokenInfo) {
+  public boolean hasToken(int position, CharSequence myBuffer, @Nullable TokenInfo tokenInfo) {
     int index = 0;
     int offset = position;
-    boolean found = false;
     while (offset < myBuffer.length()) {
       char c = myBuffer.charAt(offset++);
       int nextIndex = myTrie.findSubNode(index, myIgnoreCase ? Character.toUpperCase(c) : c);
@@ -79,24 +79,29 @@ public class KeywordParser {
         String testKeyword = myIgnoreCase ? StringUtil.toUpperCase(keyword) : keyword;
         for (int i = 0; i < CustomHighlighterTokenType.KEYWORD_TYPE_COUNT; i++) {
           if (myKeywordSets.get(i).contains(testKeyword)) {
-            myTokenInfo.updateData(position, position + keyword.length(), getToken(i));
-            found = true;
-            break;
+            if (tokenInfo != null) {
+              tokenInfo.updateData(position, position + keyword.length(), getToken(i));
+            }
+            return true;
           }
         }
       }
     }
 
-    return found;
+    return false;
   }
 
-  private static boolean isWordEnd(int offset, CharSequence myBuffer) {
-    if (offset == myBuffer.length()) {
+  private static boolean isWordEnd(int offset, CharSequence sequence) {
+    if (offset == sequence.length()) {
       return true;
     }
+    
+    return !isWordPart(offset - 1, sequence) || !isWordPart(offset, sequence);
+  }
 
-    char ch = myBuffer.charAt(offset);
-    return ch != '-' && ch != '_' && !Character.isLetterOrDigit(ch);
+  static boolean isWordPart(int offset, CharSequence sequence) {
+    char ch = sequence.charAt(offset);
+    return ch == '-' || Character.isJavaIdentifierPart(ch);
   }
 
   private static IElementType getToken(int keywordSetIndex) {

platform/platform-resources/src/FileTypeManager.xml

@@ -60,6 +60,7 @@
         </options>
         <keywords keywords="#define;#else;#endif;#error;#if;#ifdef;#ifndef;#include;#line;#undef;auto;bool;break;case;char;const;continue;default;delete;do;double;else;enum;extern;false;float;for;goto;if;int;long;register;return;short;signed;sizeof;static;struct;switch;typedef;union;unsigned;void;volatile;while" ignore_case="false" />
         <keywords2 keywords="asm;catch;class;const_cast;dynamic_cast;explicit;friend;inline;mutable;namespace;new;operator;private;protected;public;reinterpret_cast;static_cast;this;throw;true;try;typeid;typename;using;virtual;wchar_t" />
+        <keywords3 keywords="->" />
       </highlighting>
     </filetype>
 

platform/platform-tests/testSrc/com/intellij/ide/highlighter/custom/CustomFileTypeLexerTest.groovy

@@ -283,6 +283,16 @@ IDENTIFIER ('k')
 '''
   }
 
+  public void testCpp() {
+    SyntaxTable table = new SyntaxTable()
+    table.addKeyword1('->')
+    doTest table, "foo->bar", '''\
+IDENTIFIER ('foo')
+KEYWORD_1 ('->')
+IDENTIFIER ('bar')
+'''
+  }
+
   public void testNumber() {
     doTest createPropTable(), "1.23=1.24", '''\
 NUMBER ('1.23')