From 97d8c2937ed17e28e87a6fb845ca8691ab69d902 Mon Sep 17 00:00:00 2001
From: peter <peter@jetbrains.com>
Date: Wed, 3 Aug 2016 17:37:44 +0200
Subject: [PATCH] IDEA-159360 Custom file type: inconsistent highlighting of
 symbols added as keywords

---
 .../custom/tokens/IdentifierParser.java       |  6 +-
 .../custom/tokens/KeywordParser.java          |  7 +-
 .../custom/CustomFileTypeLexerTest.groovy     | 66 ++++++++++++++++++-
 3 files changed, 69 insertions(+), 10 deletions(-)

diff --git a/platform/indexing-impl/src/com/intellij/ide/highlighter/custom/tokens/IdentifierParser.java b/platform/indexing-impl/src/com/intellij/ide/highlighter/custom/tokens/IdentifierParser.java
index 8fc92e722e0f..70b38783cb78 100644
--- a/platform/indexing-impl/src/com/intellij/ide/highlighter/custom/tokens/IdentifierParser.java
+++ b/platform/indexing-impl/src/com/intellij/ide/highlighter/custom/tokens/IdentifierParser.java
@@ -42,7 +42,9 @@ public class IdentifierParser extends TokenParser {
   }
 
   private boolean isIdentifierPart(int position) {
-    if (myBuffer.charAt(position) == '-') return !myKeywordParser.hasToken(position, myBuffer, null);
-    return KeywordParser.isWordPart(position, myBuffer);
+    if (myBuffer.charAt(position) == '-') {
+      return !myKeywordParser.hasToken(position, myBuffer, null);
+    }
+    return Character.isJavaIdentifierPart(myBuffer.charAt(position));
   }
 }
diff --git a/platform/indexing-impl/src/com/intellij/ide/highlighter/custom/tokens/KeywordParser.java b/platform/indexing-impl/src/com/intellij/ide/highlighter/custom/tokens/KeywordParser.java
index b210305c2489..45b48234dcee 100644
--- a/platform/indexing-impl/src/com/intellij/ide/highlighter/custom/tokens/KeywordParser.java
+++ b/platform/indexing-impl/src/com/intellij/ide/highlighter/custom/tokens/KeywordParser.java
@@ -96,12 +96,7 @@ public class KeywordParser {
       return true;
     }
     
-    return !isWordPart(offset - 1, sequence) || !isWordPart(offset, sequence);
-  }
-
-  static boolean isWordPart(int offset, CharSequence sequence) {
-    char ch = sequence.charAt(offset);
-    return ch == '-' || Character.isJavaIdentifierPart(ch);
+    return !Character.isJavaIdentifierPart(sequence.charAt(offset - 1)) || !Character.isJavaIdentifierPart(sequence.charAt(offset));
   }
 
   private static IElementType getToken(int keywordSetIndex) {
diff --git a/platform/platform-tests/testSrc/com/intellij/ide/highlighter/custom/CustomFileTypeLexerTest.groovy b/platform/platform-tests/testSrc/com/intellij/ide/highlighter/custom/CustomFileTypeLexerTest.groovy
index 6a50715e0138..a0d8f6fd3cfd 100644
--- a/platform/platform-tests/testSrc/com/intellij/ide/highlighter/custom/CustomFileTypeLexerTest.groovy
+++ b/platform/platform-tests/testSrc/com/intellij/ide/highlighter/custom/CustomFileTypeLexerTest.groovy
@@ -33,7 +33,7 @@ import org.jetbrains.annotations.Nullable
  */
 class CustomFileTypeLexerTest extends TestCase {
 
-  private void doTest(SyntaxTable table, @NonNls String text, @Nullable String expected) {
+  private static void doTest(SyntaxTable table, @NonNls String text, @Nullable String expected) {
     def lexer = new CustomFileTypeLexer(table)
     doTest(lexer, text, expected);
   }
@@ -97,6 +97,67 @@ WHITESPACE ('\\n')
 '''
   }
 
+  public void "test punctuation keywords"() {
+    def table = createGenericTable()
+    table.addKeyword4("+")
+    table.addKeyword4("-")
+    table.addKeyword4("=")
+    doTest table, 'i++; i--; i-100; i -100; i- 100; a=-1; a= -1; a+=1; a= +1;', '''\
+IDENTIFIER ('i')
+KEYWORD_4 ('+')
+KEYWORD_4 ('+')
+PUNCTUATION (';')
+WHITESPACE (' ')
+IDENTIFIER ('i')
+KEYWORD_4 ('-')
+KEYWORD_4 ('-')
+PUNCTUATION (';')
+WHITESPACE (' ')
+IDENTIFIER ('i')
+KEYWORD_4 ('-')
+NUMBER ('100')
+PUNCTUATION (';')
+WHITESPACE (' ')
+IDENTIFIER ('i')
+WHITESPACE (' ')
+KEYWORD_4 ('-')
+NUMBER ('100')
+PUNCTUATION (';')
+WHITESPACE (' ')
+IDENTIFIER ('i')
+KEYWORD_4 ('-')
+WHITESPACE (' ')
+NUMBER ('100')
+PUNCTUATION (';')
+WHITESPACE (' ')
+IDENTIFIER ('a')
+KEYWORD_4 ('=')
+KEYWORD_4 ('-')
+NUMBER ('1')
+PUNCTUATION (';')
+WHITESPACE (' ')
+IDENTIFIER ('a')
+KEYWORD_4 ('=')
+WHITESPACE (' ')
+KEYWORD_4 ('-')
+NUMBER ('1')
+PUNCTUATION (';')
+WHITESPACE (' ')
+IDENTIFIER ('a')
+KEYWORD_4 ('+')
+KEYWORD_4 ('=')
+NUMBER ('1')
+PUNCTUATION (';')
+WHITESPACE (' ')
+IDENTIFIER ('a')
+KEYWORD_4 ('=')
+WHITESPACE (' ')
+KEYWORD_4 ('+')
+NUMBER ('1')
+PUNCTUATION (';')
+'''
+  }
+
   private static SyntaxTable createJavaSyntaxTable() {
     SyntaxTable table = new SyntaxTable();
 
@@ -341,7 +402,8 @@ KEYWORD_1 ('b-c')
 WHITESPACE (' ')
 KEYWORD_2 ('d#')
 WHITESPACE (' ')
-IDENTIFIER ('e-')
+KEYWORD_2 ('e')
+CHARACTER ('-')
 WHITESPACE (' ')
 KEYWORD_2 ('e')
 WHITESPACE (' ')