Splitting IdTableBuilding

2026-09-27 10:03:11 +07:00 · 2012-05-11 18:05:10 +02:00
parent 89de93f609
commit 0a7f10796f
49 changed files with 237 additions and 190 deletions
@@ -26,7 +26,7 @@ import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.psi.JspPsiUtil;
 import com.intellij.psi.PsiDocumentManager;
 import com.intellij.psi.PsiFile;
-import com.intellij.psi.impl.cache.impl.id.IdTableBuilding;
+import com.intellij.psi.impl.cache.impl.id.PlatformIdTableBuilding;
 import com.intellij.psi.impl.source.tree.StdTokenSets;
 import com.intellij.psi.jsp.JspFile;
 import com.intellij.psi.jsp.JspTokenType;
@@ -47,7 +47,7 @@ public class JspIndexPatternBuilder implements IndexPatternBuilder {
      boolean alreadyInitializedHighlighter = false;

      if ((cachedEditorHighlighter = EditorHighlighterCache.getEditorHighlighterForCachesBuilding(document)) != null &&
-          IdTableBuilding.checkCanUseCachedEditorHighlighter(file.getText(), cachedEditorHighlighter)) {
+          PlatformIdTableBuilding.checkCanUseCachedEditorHighlighter(file.getText(), cachedEditorHighlighter)) {
        highlighter = cachedEditorHighlighter;
        alreadyInitializedHighlighter = true;
      }
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2009 JetBrains s.r.o.
+ * Copyright 2000-2012 JetBrains s.r.o.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2009 JetBrains s.r.o.
+ * Copyright 2000-2012 JetBrains s.r.o.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2009 JetBrains s.r.o.
+ * Copyright 2000-2012 JetBrains s.r.o.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2009 JetBrains s.r.o.
+ * Copyright 2000-2012 JetBrains s.r.o.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2009 JetBrains s.r.o.
+ * Copyright 2000-2012 JetBrains s.r.o.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2009 JetBrains s.r.o.
+ * Copyright 2000-2012 JetBrains s.r.o.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2009 JetBrains s.r.o.
+ * Copyright 2000-2012 JetBrains s.r.o.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
@@ -8,6 +8,7 @@
    <orderEntry type="inheritedJdk" />
    <orderEntry type="sourceFolder" forTests="false" />
    <orderEntry type="module" module-name="core-impl" />
+    <orderEntry type="module" module-name="indexing-api" exported="" />
  </component>
 </module>

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2009 JetBrains s.r.o.
+ * Copyright 2000-2012 JetBrains s.r.o.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2009 JetBrains s.r.o.
+ * Copyright 2000-2012 JetBrains s.r.o.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2009 JetBrains s.r.o.
+ * Copyright 2000-2012 JetBrains s.r.o.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2009 JetBrains s.r.o.
+ * Copyright 2000-2012 JetBrains s.r.o.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
@@ -22,7 +22,6 @@ import gnu.trove.THashMap;
 import gnu.trove.TIntIntHashMap;
 import gnu.trove.TIntIntProcedure;

-import java.util.HashMap;
 import java.util.Map;

 /**
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2009 JetBrains s.r.o.
+ * Copyright 2000-2012 JetBrains s.r.o.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
@@ -22,7 +22,7 @@
    <orderEntry type="library" exported="" name="cli-parser" level="project" />
    <orderEntry type="library" name="http-client-3.1" level="project" />
    <orderEntry type="module" module-name="indexing-api" />
-    <orderEntry type="module" module-name="indexing-impl" />
+    <orderEntry type="module" module-name="indexing-impl" exported="" />
  </component>
 </module>

@@ -16,7 +16,6 @@
 package com.intellij.find.findUsages;

 import com.intellij.psi.PsiElement;
-import com.intellij.psi.PsiFile;
 import com.intellij.lang.findUsages.LanguageFindUsages;
 import com.intellij.psi.PsiFileSystemItem;
 import org.jetbrains.annotations.NotNull;
@@ -20,13 +20,11 @@
 package com.intellij.openapi.fileTypes;

 import com.intellij.ide.highlighter.custom.AbstractCustomLexer;
-import com.intellij.ide.highlighter.custom.CustomHighlighterColors;
 import com.intellij.ide.highlighter.custom.tokens.*;
 import com.intellij.lexer.Lexer;
 import com.intellij.openapi.editor.colors.TextAttributesKey;
 import com.intellij.openapi.project.Project;
 import com.intellij.openapi.vfs.VirtualFile;
-import com.intellij.psi.CustomHighlighterTokenType;
 import com.intellij.psi.tree.IElementType;
 import org.jetbrains.annotations.NotNull;

@@ -21,7 +21,6 @@ import com.intellij.openapi.vfs.LocalFileSystem;
 import com.intellij.psi.impl.cache.impl.id.IdIndexEntry;
 import com.intellij.psi.impl.cache.impl.id.IdTableBuilding;
 import com.intellij.psi.impl.cache.impl.id.LexerBasedIdIndexer;
-import com.intellij.psi.impl.cache.impl.todo.LexerBasedTodoIndexer;
 import com.intellij.psi.impl.cache.impl.todo.TodoIndexEntry;
 import com.intellij.psi.search.IndexPattern;
 import com.intellij.util.indexing.FileContent;
@@ -29,7 +28,6 @@ import com.intellij.util.indexing.IdDataConsumer;
 import gnu.trove.THashMap;

 import java.util.Collections;
-import java.util.HashMap;
 import java.util.Map;

 public class BaseFilterLexerUtil {
@@ -16,39 +16,26 @@

 package com.intellij.psi.impl.cache.impl.id;

-import com.intellij.ide.highlighter.HighlighterFactory;
 import com.intellij.ide.highlighter.custom.CustomFileTypeLexer;
-import com.intellij.idea.LoggerFactory;
 import com.intellij.lang.Language;
-import com.intellij.lang.LanguageParserDefinitions;
-import com.intellij.lang.ParserDefinition;
 import com.intellij.lang.cacheBuilder.*;
 import com.intellij.lang.findUsages.FindUsagesProvider;
 import com.intellij.lang.findUsages.LanguageFindUsages;
-import com.intellij.openapi.diagnostic.Logger;
-import com.intellij.openapi.editor.ex.util.LexerEditorHighlighter;
-import com.intellij.openapi.editor.highlighter.EditorHighlighter;
-import com.intellij.openapi.editor.highlighter.HighlighterIterator;
 import com.intellij.openapi.fileTypes.FileType;
-import com.intellij.openapi.fileTypes.FileTypes;
 import com.intellij.openapi.fileTypes.LanguageFileType;
-import com.intellij.openapi.fileTypes.StdFileTypes;
 import com.intellij.openapi.fileTypes.impl.AbstractFileType;
-import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.psi.CustomHighlighterTokenType;
-import com.intellij.psi.impl.cache.impl.BaseFilterLexer;
-import com.intellij.psi.impl.cache.impl.CacheUtil;
 import com.intellij.psi.impl.cache.impl.IndexPatternUtil;
 import com.intellij.psi.impl.cache.impl.OccurrenceConsumer;
 import com.intellij.psi.impl.cache.impl.todo.TodoIndexEntry;
 import com.intellij.psi.impl.cache.impl.todo.TodoIndexers;
 import com.intellij.psi.search.IndexPattern;
 import com.intellij.psi.search.UsageSearchContext;
-import com.intellij.psi.tree.IElementType;
 import com.intellij.psi.tree.TokenSet;
 import com.intellij.util.Processor;
-import com.intellij.util.containers.CollectionFactory;
-import com.intellij.util.indexing.*;
+import com.intellij.util.indexing.DataIndexer;
+import com.intellij.util.indexing.FileContent;
+import com.intellij.util.indexing.IdDataConsumer;
 import com.intellij.util.text.CharArrayUtil;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
@@ -123,8 +110,6 @@ public class IdTableBuilding {
  }

  private static final HashMap<FileType, FileTypeIdIndexer> ourIdIndexers = new HashMap<FileType, FileTypeIdIndexer>();
-  private static final HashMap<FileType, DataIndexer<TodoIndexEntry, Integer, FileContent>> ourTodoIndexers =
-    new HashMap<FileType, DataIndexer<TodoIndexEntry, Integer, FileContent>>();

  @Deprecated
  public static void registerIdIndexer(FileType fileType, FileTypeIdIndexer indexer) {
@@ -133,7 +118,7 @@ public class IdTableBuilding {

  @Deprecated
  public static void registerTodoIndexer(FileType fileType, DataIndexer<TodoIndexEntry, Integer, FileContent> indexer) {
-    ourTodoIndexers.put(fileType, indexer);
+    PlatformIdTableBuilding.ourTodoIndexers.put(fileType, indexer);
  }

  public static boolean isIdIndexerRegistered(FileType fileType) {
@@ -144,18 +129,7 @@ public class IdTableBuilding {
    return ourIdIndexers.containsKey(fileType) || TodoIndexers.INSTANCE.forFileType(fileType) != null;
  }

-  static {
-    registerIdIndexer(FileTypes.PLAIN_TEXT, new PlainTextIndexer());
-    registerTodoIndexer(FileTypes.PLAIN_TEXT, new PlainTextTodoIndexer());

-    registerIdIndexer(StdFileTypes.IDEA_MODULE, null);
-    registerIdIndexer(StdFileTypes.IDEA_WORKSPACE, null);
-    registerIdIndexer(StdFileTypes.IDEA_PROJECT, null);
-
-    registerTodoIndexer(StdFileTypes.IDEA_MODULE, null);
-    registerTodoIndexer(StdFileTypes.IDEA_WORKSPACE, null);
-    registerTodoIndexer(StdFileTypes.IDEA_PROJECT, null);
-  }

  @Nullable
  public static FileTypeIdIndexer getFileTypeIndexer(FileType fileType) {
@@ -201,46 +175,6 @@ public class IdTableBuilding {

  }

-  private static final TokenSet ABSTRACT_FILE_COMMENT_TOKENS =
-    TokenSet.create(CustomHighlighterTokenType.LINE_COMMENT, CustomHighlighterTokenType.MULTI_LINE_COMMENT);
-
-  @Nullable
-  public static DataIndexer<TodoIndexEntry, Integer, FileContent> getTodoIndexer(FileType fileType, final VirtualFile virtualFile) {
-    final DataIndexer<TodoIndexEntry, Integer, FileContent> indexer = ourTodoIndexers.get(fileType);
-
-    if (indexer != null) {
-      return indexer;
-    }
-
-    final DataIndexer<TodoIndexEntry, Integer, FileContent> extIndexer;
-    if (fileType instanceof SubstitutedFileType) {
-      SubstitutedFileType sft = (SubstitutedFileType)fileType;
-      extIndexer =
-        new CompositeTodoIndexer(getTodoIndexer(sft.getOriginalFileType(), virtualFile), getTodoIndexer(sft.getFileType(), virtualFile));
-    }
-    else {
-      extIndexer = TodoIndexers.INSTANCE.forFileType(fileType);
-    }
-    if (extIndexer != null) {
-      return extIndexer;
-    }
-
-    if (fileType instanceof LanguageFileType) {
-      final Language lang = ((LanguageFileType)fileType).getLanguage();
-      final ParserDefinition parserDef = LanguageParserDefinitions.INSTANCE.forLanguage(lang);
-      final TokenSet commentTokens = parserDef != null ? parserDef.getCommentTokens() : null;
-      if (commentTokens != null) {
-        return new TokenSetTodoIndexer(commentTokens, virtualFile);
-      }
-    }
-
-    if (fileType instanceof AbstractFileType) {
-      return new TokenSetTodoIndexer(ABSTRACT_FILE_COMMENT_TOKENS, virtualFile);
-    }
-
-    return null;
-  }
-
  private static class WordsScannerFileTypeIdIndexerAdapter extends FileTypeIdIndexer {
    private final WordsScanner myScanner;

@@ -279,102 +213,6 @@ public class IdTableBuilding {
    }
  }

-  private static class CompositeTodoIndexer implements DataIndexer<TodoIndexEntry, Integer, FileContent> {
-
-    private final DataIndexer<TodoIndexEntry, Integer, FileContent>[] indexers;
-
-    public CompositeTodoIndexer(DataIndexer<TodoIndexEntry, Integer, FileContent>... indexers) {
-      this.indexers = indexers;
-    }
-
-    @NotNull
-    @Override
-    public Map<TodoIndexEntry, Integer> map(FileContent inputData) {
-      Map<TodoIndexEntry, Integer> result = CollectionFactory.newTroveMap();
-      for (DataIndexer<TodoIndexEntry, Integer, FileContent> indexer : indexers) {
-        for (Map.Entry<TodoIndexEntry, Integer> entry : indexer.map(inputData).entrySet()) {
-          TodoIndexEntry key = entry.getKey();
-          if (result.containsKey(key)) {
-            result.put(key, result.get(key) + entry.getValue());
-          } else {
-            result.put(key, entry.getValue());
-          }
-        }
-      }
-      return result;
-    }
-  }
-
-  private static class TokenSetTodoIndexer implements DataIndexer<TodoIndexEntry, Integer, FileContent> {
-    @NotNull private final TokenSet myCommentTokens;
-    private final VirtualFile myFile;
-
-    public TokenSetTodoIndexer(@NotNull final TokenSet commentTokens, @NotNull final VirtualFile file) {
-      myCommentTokens = commentTokens;
-      myFile = file;
-    }
-
-    @Override
-    @NotNull
-    public Map<TodoIndexEntry, Integer> map(final FileContent inputData) {
-      if (IndexPatternUtil.getIndexPatternCount() > 0) {
-        final CharSequence chars = inputData.getContentAsText();
-        final OccurrenceConsumer occurrenceConsumer = new OccurrenceConsumer(null, true);
-        EditorHighlighter highlighter;
-
-        final EditorHighlighter editorHighlighter = inputData.getUserData(FileBasedIndexImpl.EDITOR_HIGHLIGHTER);
-        if (editorHighlighter != null && checkCanUseCachedEditorHighlighter(chars, editorHighlighter)) {
-          highlighter = editorHighlighter;
-        }
-        else {
-          highlighter = HighlighterFactory.createHighlighter(null, myFile);
-          highlighter.setText(chars);
-        }
-
-        final int documentLength = chars.length();
-        BaseFilterLexer.TodoScanningData[] todoScanningDatas = null;
-        final HighlighterIterator iterator = highlighter.createIterator(0);
-
-        while (!iterator.atEnd()) {
-          final IElementType token = iterator.getTokenType();
-
-          if (myCommentTokens.contains(token) || CacheUtil.isInComments(token)) {
-            int start = iterator.getStart();
-            if (start >= documentLength) break;            
-            int end = iterator.getEnd();
-
-            todoScanningDatas = BaseFilterLexer.advanceTodoItemsCount(
-              chars.subSequence(start, Math.min(end, documentLength)), 
-              occurrenceConsumer, 
-              todoScanningDatas
-            );
-            if (end > documentLength) break;
-          }
-          iterator.advance();
-        }
-        final Map<TodoIndexEntry, Integer> map = new HashMap<TodoIndexEntry, Integer>();
-        for (IndexPattern pattern : IndexPatternUtil.getIndexPatterns()) {
-          final int count = occurrenceConsumer.getOccurrenceCount(pattern);
-          if (count > 0) {
-            map.put(new TodoIndexEntry(pattern.getPatternString(), pattern.isCaseSensitive()), count);
-          }
-        }
-        return map;
-      }
-      return Collections.emptyMap();
-    }
-  }
-
-  public static boolean checkCanUseCachedEditorHighlighter(final CharSequence chars, final EditorHighlighter editorHighlighter) {
-    assert editorHighlighter instanceof LexerEditorHighlighter;
-    final boolean b = ((LexerEditorHighlighter)editorHighlighter).checkContentIsEqualTo(chars);
-    if (!b) {
-      final Logger logger = LoggerFactory.getInstance().getLoggerInstance(IdTableBuilding.class.getName());
-      logger.warn("Unexpected mismatch of editor highlighter content with indexing content");
-    }
-    return b;
-  }
-
  public static void scanWords(final ScanWordProcessor processor, final CharSequence chars, final int startOffset, final int endOffset) {
    scanWords(processor, chars, CharArrayUtil.fromSequenceWithoutCopying(chars), startOffset, endOffset, false);
  }
@@ -0,0 +1,211 @@
+/*
+ * Copyright 2000-2012 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.psi.impl.cache.impl.id;
+
+import com.intellij.ide.highlighter.HighlighterFactory;
+import com.intellij.lang.Language;
+import com.intellij.lang.LanguageParserDefinitions;
+import com.intellij.lang.ParserDefinition;
+import com.intellij.openapi.diagnostic.Logger;
+import com.intellij.openapi.editor.ex.util.LexerEditorHighlighter;
+import com.intellij.openapi.editor.highlighter.EditorHighlighter;
+import com.intellij.openapi.editor.highlighter.HighlighterIterator;
+import com.intellij.openapi.fileTypes.FileType;
+import com.intellij.openapi.fileTypes.FileTypes;
+import com.intellij.openapi.fileTypes.LanguageFileType;
+import com.intellij.openapi.fileTypes.StdFileTypes;
+import com.intellij.openapi.fileTypes.impl.AbstractFileType;
+import com.intellij.openapi.vfs.VirtualFile;
+import com.intellij.psi.CustomHighlighterTokenType;
+import com.intellij.psi.impl.cache.impl.BaseFilterLexer;
+import com.intellij.psi.impl.cache.impl.CacheUtil;
+import com.intellij.psi.impl.cache.impl.IndexPatternUtil;
+import com.intellij.psi.impl.cache.impl.OccurrenceConsumer;
+import com.intellij.psi.impl.cache.impl.todo.TodoIndexEntry;
+import com.intellij.psi.impl.cache.impl.todo.TodoIndexers;
+import com.intellij.psi.search.IndexPattern;
+import com.intellij.psi.tree.IElementType;
+import com.intellij.psi.tree.TokenSet;
+import com.intellij.util.containers.CollectionFactory;
+import com.intellij.util.indexing.DataIndexer;
+import com.intellij.util.indexing.FileBasedIndexImpl;
+import com.intellij.util.indexing.FileContent;
+import com.intellij.util.indexing.SubstitutedFileType;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Author: dmitrylomov
+ */
+public abstract class PlatformIdTableBuilding {
+  static final HashMap<FileType, DataIndexer<TodoIndexEntry, Integer, FileContent>> ourTodoIndexers =
+    new HashMap<FileType, DataIndexer<TodoIndexEntry, Integer, FileContent>>();
+  private static final TokenSet ABSTRACT_FILE_COMMENT_TOKENS =
+    TokenSet.create(CustomHighlighterTokenType.LINE_COMMENT, CustomHighlighterTokenType.MULTI_LINE_COMMENT);
+
+  private PlatformIdTableBuilding() {}
+
+  @Nullable
+  public static DataIndexer<TodoIndexEntry, Integer, FileContent> getTodoIndexer(FileType fileType, final VirtualFile virtualFile) {
+    final DataIndexer<TodoIndexEntry, Integer, FileContent> indexer = ourTodoIndexers.get(fileType);
+
+    if (indexer != null) {
+      return indexer;
+    }
+
+    final DataIndexer<TodoIndexEntry, Integer, FileContent> extIndexer;
+    if (fileType instanceof SubstitutedFileType) {
+      SubstitutedFileType sft = (SubstitutedFileType)fileType;
+      extIndexer =
+        new CompositeTodoIndexer(getTodoIndexer(sft.getOriginalFileType(), virtualFile), getTodoIndexer(sft.getFileType(), virtualFile));
+    }
+    else {
+      extIndexer = TodoIndexers.INSTANCE.forFileType(fileType);
+    }
+    if (extIndexer != null) {
+      return extIndexer;
+    }
+
+    if (fileType instanceof LanguageFileType) {
+      final Language lang = ((LanguageFileType)fileType).getLanguage();
+      final ParserDefinition parserDef = LanguageParserDefinitions.INSTANCE.forLanguage(lang);
+      final TokenSet commentTokens = parserDef != null ? parserDef.getCommentTokens() : null;
+      if (commentTokens != null) {
+        return new TokenSetTodoIndexer(commentTokens, virtualFile);
+      }
+    }
+
+    if (fileType instanceof AbstractFileType) {
+      return new TokenSetTodoIndexer(ABSTRACT_FILE_COMMENT_TOKENS, virtualFile);
+    }
+
+    return null;
+  }
+
+  public static boolean checkCanUseCachedEditorHighlighter(final CharSequence chars, final EditorHighlighter editorHighlighter) {
+    assert editorHighlighter instanceof LexerEditorHighlighter;
+    final boolean b = ((LexerEditorHighlighter)editorHighlighter).checkContentIsEqualTo(chars);
+    if (!b) {
+      final Logger logger = Logger.getInstance(IdTableBuilding.class.getName());
+      logger.warn("Unexpected mismatch of editor highlighter content with indexing content");
+    }
+    return b;
+  }
+
+
+  private static class CompositeTodoIndexer implements DataIndexer<TodoIndexEntry, Integer, FileContent> {
+
+    private final DataIndexer<TodoIndexEntry, Integer, FileContent>[] indexers;
+
+    public CompositeTodoIndexer(DataIndexer<TodoIndexEntry, Integer, FileContent>... indexers) {
+      this.indexers = indexers;
+    }
+
+    @NotNull
+    @Override
+    public Map<TodoIndexEntry, Integer> map(FileContent inputData) {
+      Map<TodoIndexEntry, Integer> result = CollectionFactory.newTroveMap();
+      for (DataIndexer<TodoIndexEntry, Integer, FileContent> indexer : indexers) {
+        for (Map.Entry<TodoIndexEntry, Integer> entry : indexer.map(inputData).entrySet()) {
+          TodoIndexEntry key = entry.getKey();
+          if (result.containsKey(key)) {
+            result.put(key, result.get(key) + entry.getValue());
+          } else {
+            result.put(key, entry.getValue());
+          }
+        }
+      }
+      return result;
+    }
+  }
+
+  private static class TokenSetTodoIndexer implements DataIndexer<TodoIndexEntry, Integer, FileContent> {
+    @NotNull private final TokenSet myCommentTokens;
+    private final VirtualFile myFile;
+
+    public TokenSetTodoIndexer(@NotNull final TokenSet commentTokens, @NotNull final VirtualFile file) {
+      myCommentTokens = commentTokens;
+      myFile = file;
+    }
+
+    @Override
+    @NotNull
+    public Map<TodoIndexEntry, Integer> map(final FileContent inputData) {
+      if (IndexPatternUtil.getIndexPatternCount() > 0) {
+        final CharSequence chars = inputData.getContentAsText();
+        final OccurrenceConsumer occurrenceConsumer = new OccurrenceConsumer(null, true);
+        EditorHighlighter highlighter;
+
+        final EditorHighlighter editorHighlighter = inputData.getUserData(FileBasedIndexImpl.EDITOR_HIGHLIGHTER);
+        if (editorHighlighter != null && checkCanUseCachedEditorHighlighter(chars, editorHighlighter)) {
+          highlighter = editorHighlighter;
+        }
+        else {
+          highlighter = HighlighterFactory.createHighlighter(null, myFile);
+          highlighter.setText(chars);
+        }
+
+        final int documentLength = chars.length();
+        BaseFilterLexer.TodoScanningData[] todoScanningDatas = null;
+        final HighlighterIterator iterator = highlighter.createIterator(0);
+
+        while (!iterator.atEnd()) {
+          final IElementType token = iterator.getTokenType();
+
+          if (myCommentTokens.contains(token) || CacheUtil.isInComments(token)) {
+            int start = iterator.getStart();
+            if (start >= documentLength) break;
+            int end = iterator.getEnd();
+
+            todoScanningDatas = BaseFilterLexer.advanceTodoItemsCount(
+              chars.subSequence(start, Math.min(end, documentLength)),
+              occurrenceConsumer,
+              todoScanningDatas
+            );
+            if (end > documentLength) break;
+          }
+          iterator.advance();
+        }
+        final Map<TodoIndexEntry, Integer> map = new HashMap<TodoIndexEntry, Integer>();
+        for (IndexPattern pattern : IndexPatternUtil.getIndexPatterns()) {
+          final int count = occurrenceConsumer.getOccurrenceCount(pattern);
+          if (count > 0) {
+            map.put(new TodoIndexEntry(pattern.getPatternString(), pattern.isCaseSensitive()), count);
+          }
+        }
+        return map;
+      }
+      return Collections.emptyMap();
+    }
+  }
+
+  static {
+    IdTableBuilding.registerIdIndexer(FileTypes.PLAIN_TEXT, new IdTableBuilding.PlainTextIndexer());
+    IdTableBuilding.registerTodoIndexer(FileTypes.PLAIN_TEXT, new IdTableBuilding.PlainTextTodoIndexer());
+
+    IdTableBuilding.registerIdIndexer(StdFileTypes.IDEA_MODULE, null);
+    IdTableBuilding.registerIdIndexer(StdFileTypes.IDEA_WORKSPACE, null);
+    IdTableBuilding.registerIdIndexer(StdFileTypes.IDEA_PROJECT, null);
+
+    IdTableBuilding.registerTodoIndexer(StdFileTypes.IDEA_MODULE, null);
+    IdTableBuilding.registerTodoIndexer(StdFileTypes.IDEA_WORKSPACE, null);
+    IdTableBuilding.registerTodoIndexer(StdFileTypes.IDEA_PROJECT, null);
+  }
+}
@@ -26,6 +26,7 @@ import com.intellij.openapi.project.ProjectUtil;
 import com.intellij.openapi.vfs.LocalFileSystem;
 import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.psi.impl.cache.impl.id.IdTableBuilding;
+import com.intellij.psi.impl.cache.impl.id.PlatformIdTableBuilding;
 import com.intellij.psi.search.IndexPatternProvider;
 import com.intellij.psi.tree.TokenSet;
 import com.intellij.util.indexing.*;
@@ -101,7 +102,8 @@ public class TodoIndex extends FileBasedIndexExtension<TodoIndexEntry, Integer>
    @NotNull
    public Map<TodoIndexEntry,Integer> map(final FileContent inputData) {
      final VirtualFile file = inputData.getFile();
-      final DataIndexer<TodoIndexEntry, Integer, FileContent> indexer = IdTableBuilding.getTodoIndexer(inputData.getFileType(), file);
+      final DataIndexer<TodoIndexEntry, Integer, FileContent> indexer = PlatformIdTableBuilding
+        .getTodoIndexer(inputData.getFileType(), file);
      if (indexer != null) {
        return indexer.map(inputData);
      }
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2009 JetBrains s.r.o.
+ * Copyright 2000-2012 JetBrains s.r.o.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
@@ -29,6 +29,8 @@
    <orderEntry type="library" name="swingx" level="project" />
    <orderEntry type="module" module-name="core-impl" exported="" />
    <orderEntry type="library" exported="" name="miglayout-swing" level="project" />
+    <orderEntry type="module" module-name="indexing-api" />
+    <orderEntry type="module" module-name="indexing-impl" />
  </component>
 </module>

@@ -17,7 +17,6 @@ package com.intellij.ide.highlighter.custom.impl;

 import com.intellij.ide.highlighter.custom.SyntaxTable;
 import com.intellij.openapi.fileTypes.impl.AbstractFileType;
-import com.intellij.openapi.options.SettingsEditor;
 import org.jdom.Element;

 public class ReadFileType extends AbstractFileType {