share and reuse the expensive automaton in custom file type lexing IDEA-95553

platform/indexing-impl/src/com/intellij/ide/highlighter/custom/CustomFileTypeLexer.java

@@ -21,7 +21,6 @@ import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.psi.CustomHighlighterTokenType;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 
 /**
@@ -42,8 +41,15 @@ public final class CustomFileTypeLexer extends AbstractCustomLexer {
             MultilineCommentParser.create(table.getStartComment(), table.getEndComment());
     final NumberParser numberParser = new NumberParser(table.getNumPostfixChars(), table.isIgnoreCase());
     final HexNumberParser hexNumberParser = HexNumberParser.create(table.getHexPrefix());
-    final KeywordParser keywordParser = new KeywordParser(Arrays.asList(table.getKeywords1(), table.getKeywords2(), table.getKeywords3(), table.getKeywords4()),
-                    table.isIgnoreCase());
+    
+    final KeywordParser parser = table.getKeywordParser();
+    final TokenParser keywordParser = new TokenParser() {
+      @Override
+      public boolean hasToken(int position) {
+        return parser.hasToken(position, myBuffer, myTokenInfo);
+      }
+    };
+    
     final IdentifierParser identifierParser = new IdentifierParser();
 
     final QuotedStringParser quotedStringParser = new QuotedStringParser("\"", CustomHighlighterTokenType.STRING, table.isHasStringEscapes());

platform/indexing-impl/src/com/intellij/ide/highlighter/custom/SyntaxTable.java

@@ -15,6 +15,9 @@
  */
 package com.intellij.ide.highlighter.custom;
 
+import com.intellij.ide.highlighter.custom.tokens.KeywordParser;
+
+import java.util.Arrays;
 import java.util.Set;
 import java.util.TreeSet;
 
@@ -41,6 +44,7 @@ public class SyntaxTable implements Cloneable {
   private boolean myHasBrackets;
   private boolean myHasParens;
   private boolean myHasStringEscapes;
+  private volatile KeywordParser myKeywordParser;
 
   // -------------------------------------------------------------------------
   // Constructor
@@ -53,13 +57,26 @@ public class SyntaxTable implements Cloneable {
     myKeywords4 = new TreeSet<String>();
   }
 
+  KeywordParser getKeywordParser() {
+    KeywordParser parser = myKeywordParser;
+    if (parser == null) {
+      synchronized (this) {
+        parser = myKeywordParser;
+        if (parser == null) {
+          myKeywordParser = parser = new KeywordParser(Arrays.asList(myKeywords1, myKeywords2, myKeywords3, myKeywords4), myIgnoreCase);
+        }
+      }
+    }
+    return parser;
+  }
+
   protected Object clone() throws CloneNotSupportedException {
     SyntaxTable cl = (SyntaxTable) super.clone();
     cl.myKeywords1 = new TreeSet<String>(myKeywords1);
     cl.myKeywords2 = new TreeSet<String>(myKeywords2);
     cl.myKeywords3 = new TreeSet<String>(myKeywords3);
     cl.myKeywords4 = new TreeSet<String>(myKeywords4);
-
+    cl.myKeywordParser = null;
     return cl;
   }
 
@@ -69,6 +86,7 @@ public class SyntaxTable implements Cloneable {
 
   public void addKeyword1(String keyword) {
     myKeywords1.add(keyword);
+    myKeywordParser = null;
   }
 
   public Set<String> getKeywords1() {
@@ -77,6 +95,7 @@ public class SyntaxTable implements Cloneable {
 
   public void addKeyword2(String keyword) {
     myKeywords2.add(keyword);
+    myKeywordParser = null;
   }
 
   public Set<String> getKeywords2() {
@@ -85,6 +104,7 @@ public class SyntaxTable implements Cloneable {
 
   public void addKeyword3(String keyword) {
     myKeywords3.add(keyword);
+    myKeywordParser = null;
   }
 
   public Set<String> getKeywords3() {
@@ -93,6 +113,7 @@ public class SyntaxTable implements Cloneable {
 
   public void addKeyword4(String keyword) {
     myKeywords4.add(keyword);
+    myKeywordParser = null;
   }
 
   public Set<String> getKeywords4() {
@@ -145,6 +166,7 @@ public class SyntaxTable implements Cloneable {
 
   public void setIgnoreCase(boolean ignoreCase) {
     myIgnoreCase = ignoreCase;
+    myKeywordParser = null;
   }
 
   public boolean isHasBraces() {

platform/indexing-impl/src/com/intellij/ide/highlighter/custom/tokens/KeywordParser.java

@@ -30,8 +30,9 @@ import java.util.Set;
 
 /**
  * @author dsl
+ * @author peter
  */
-public class KeywordParser extends TokenParser {
+public class KeywordParser {
   private static final Logger LOG = Logger.getInstance("#com.intellij.ide.highlighter.custom.tokens.KeywordParser");
   private final List<Set<String>> myKeywordSets = new ArrayList<Set<String>>();
   private final CharTrie myTrie = new CharTrie();
@@ -62,7 +63,7 @@ public class KeywordParser extends TokenParser {
     return result;
   }
 
-  public boolean hasToken(int position) {
+  public boolean hasToken(int position, CharSequence myBuffer, TokenInfo myTokenInfo) {
     int index = 0;
     int offset = position;
     boolean found = false;
@@ -73,7 +74,7 @@ public class KeywordParser extends TokenParser {
         break;
       }
       index = nextIndex;
-      if (myHashCodes.contains(index) && isWordEnd(offset)) {
+      if (myHashCodes.contains(index) && isWordEnd(offset, myBuffer)) {
         String keyword = myBuffer.subSequence(position, offset).toString();
         String testKeyword = myIgnoreCase ? StringUtil.toUpperCase(keyword) : keyword;
         for (int i = 0; i < CustomHighlighterTokenType.KEYWORD_TYPE_COUNT; i++) {
@@ -89,7 +90,7 @@ public class KeywordParser extends TokenParser {
     return found;
   }
 
-  private boolean isWordEnd(int offset) {
+  private static boolean isWordEnd(int offset, CharSequence myBuffer) {
     if (offset == myBuffer.length()) {
       return true;
     }