todo: check pattern words are present in text before applying pattern itself

GitOrigin-RevId: ccba21a3af5c78995cc5a8126b2b1e2a88dfba8e

java/java-tests/testSrc/com/intellij/find/FindManagerTest.java

@@ -929,11 +929,11 @@ public class FindManagerTest extends DaemonAnalyzerTestCase {
 
   public void testRegexReplacementStringForIndices() {
     assertEquals("public static   MyType my   = 1;", FindInProjectUtil.buildStringToFindForIndicesFromRegExp("public static (@A)? MyType my\\w+?  = 1;", myProject));
-    assertEquals(" Foo ", FindInProjectUtil.buildStringToFindForIndicesFromRegExp("\\bFoo\\b", myProject));
+    assertEquals("Foo", FindInProjectUtil.buildStringToFindForIndicesFromRegExp("\\bFoo\\b", myProject));
     assertEquals("", FindInProjectUtil.buildStringToFindForIndicesFromRegExp("foo|bar", myProject));
-    assertEquals(" Exit Foo Bar Baz", FindInProjectUtil.buildStringToFindForIndicesFromRegExp("\\nExit\\tFoo\\rBar\\fBaz", myProject));
-    assertEquals(" Foo Bar Baz Exit", FindInProjectUtil.buildStringToFindForIndicesFromRegExp("\\012Foo\\u000ABar\\x0ABaz\\aExit", myProject));
-    assertEquals(" Foo Bar BazCooBoo", FindInProjectUtil.buildStringToFindForIndicesFromRegExp("\\1Foo\\sBar\\DBaz\\QCoo\\E\\QBoo", myProject));
+    assertEquals("Exit Foo Bar Baz", FindInProjectUtil.buildStringToFindForIndicesFromRegExp("\\nExit\\tFoo\\rBar\\fBaz", myProject));
+    assertEquals("Foo Bar Baz Exit", FindInProjectUtil.buildStringToFindForIndicesFromRegExp("\\012Foo\\u000ABar\\x0ABaz\\aExit", myProject));
+    assertEquals("Foo Bar BazCooBoo", FindInProjectUtil.buildStringToFindForIndicesFromRegExp("\\1Foo\\sBar\\DBaz\\QCoo\\E\\QBoo", myProject));
   }
 
   public void testCreateFileMaskCondition() {

platform/indexing-api/src/com/intellij/psi/search/IndexPattern.java

@@ -15,10 +15,13 @@
  */
 package com.intellij.psi.search;
 
+import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.openapi.util.text.StringUtil;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
+import java.util.Collections;
+import java.util.List;
 import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
 
@@ -33,6 +36,7 @@ public class IndexPattern {
   private Pattern myOptimizedIndexingPattern;
   private boolean myCaseSensitive;
   private Pattern myPattern;
+  private @NotNull List<String> myStringsToFindFirst = Collections.emptyList();
 
   /**
    * Creates an instance of an index pattern.
@@ -59,6 +63,11 @@ public class IndexPattern {
     return myOptimizedIndexingPattern;
   }
 
+  @NotNull
+  public List<String> getWordsToFindFirst() {
+    return myStringsToFindFirst;
+  }
+
   public boolean isCaseSensitive() {
     return myCaseSensitive;
   }
@@ -86,10 +95,17 @@ public class IndexPattern {
       String optimizedPattern = myPatternString;
       optimizedPattern = StringUtil.trimStart(optimizedPattern, ".*");
       myOptimizedIndexingPattern = Pattern.compile(optimizedPattern, flags);
+      try {
+        //myStringsToFindFirst = IndexPatternOptimizer.getInstance().extractStringsToFind(myPatternString);
+      }
+      catch (Throwable throwable) {
+        Logger.getInstance(IndexPattern.class).info("OLOLO", throwable);
+      }
     }
     catch(PatternSyntaxException e){
       myPattern = null;
       myOptimizedIndexingPattern = null;
+      myStringsToFindFirst = Collections.emptyList();
     }
   }
 

platform/indexing-api/src/com/intellij/psi/search/IndexPatternOptimizer.java

@@ -0,0 +1,18 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.psi.search;
+
+import com.intellij.openapi.components.ServiceManager;
+import org.jetbrains.annotations.ApiStatus;
+import org.jetbrains.annotations.NotNull;
+
+import java.util.List;
+
+@ApiStatus.Internal
+public interface IndexPatternOptimizer {
+
+ static IndexPatternOptimizer getInstance() {
+  return ServiceManager.getService(IndexPatternOptimizer.class);
+ }
+
+ @NotNull List<String> extractStringsToFind(@NotNull String regexp);
+}

platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/PlainTextIdIndexer.java

@@ -0,0 +1,55 @@
+// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.psi.impl.cache.impl.id;
+
+import com.intellij.openapi.fileTypes.PlainTextFileType;
+import com.intellij.openapi.util.Key;
+import com.intellij.psi.impl.cache.impl.todo.TodoIndexers;
+import com.intellij.psi.search.UsageSearchContext;
+import com.intellij.util.indexing.FileContent;
+import com.intellij.util.indexing.IdDataConsumer;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.Map;
+
+public class PlainTextIdIndexer implements IdIndexer {
+  private static final Key<Map<IdIndexEntry, Integer>> ID_INDEX_DATA_KEY = Key.create("plain.text.id.index");
+
+  @Override
+  @NotNull
+  public Map<IdIndexEntry, Integer> map(@NotNull final FileContent inputData) {
+    return getIdIndexData(inputData);
+  }
+
+  @NotNull
+  public static Map<IdIndexEntry, Integer> getIdIndexData(@NotNull FileContent content) {
+    Map<IdIndexEntry, Integer> idIndexData = content.getUserData(ID_INDEX_DATA_KEY);
+    if (idIndexData != null) {
+      content.putUserData(ID_INDEX_DATA_KEY, null);
+      return idIndexData;
+    }
+
+    IdDataConsumer consumer = new IdDataConsumer();
+    CharSequence text = content.getContentAsText();
+    IdTableBuilding.scanWords(new IdTableBuilding.ScanWordProcessor() {
+      @Override
+      public void run(final CharSequence chars11, char @Nullable [] charsArray, final int start, final int end) {
+        if (charsArray != null) {
+          consumer.addOccurrence(charsArray, start, end, UsageSearchContext.IN_PLAIN_TEXT);
+        }
+        else {
+          consumer.addOccurrence(chars11, start, end, UsageSearchContext.IN_PLAIN_TEXT);
+        }
+      }
+    }, text, 0, text.length());
+
+    Map<IdIndexEntry, Integer> result = consumer.getResult();
+
+    if (TodoIndexers.needsTodoIndex(content.getFile()) &&
+        IdIndex.isIndexable(PlainTextFileType.INSTANCE)) {
+      content.putUserData(ID_INDEX_DATA_KEY, result);
+    }
+
+    return result;
+  }
+}

platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/PlainTextIndexer.java

@@ -1,31 +0,0 @@
-// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
-package com.intellij.psi.impl.cache.impl.id;
-
-import com.intellij.psi.search.UsageSearchContext;
-import com.intellij.util.indexing.FileContent;
-import com.intellij.util.indexing.IdDataConsumer;
-import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
-
-import java.util.Map;
-
-public class PlainTextIndexer implements IdIndexer {
-  @Override
-  @NotNull
-  public Map<IdIndexEntry, Integer> map(@NotNull final FileContent inputData) {
-    final IdDataConsumer consumer = new IdDataConsumer();
-    final CharSequence chars = inputData.getContentAsText();
-    IdTableBuilding.scanWords(new IdTableBuilding.ScanWordProcessor() {
-      @Override
-      public void run(final CharSequence chars11, char @Nullable [] charsArray, final int start, final int end) {
-        if (charsArray != null) {
-          consumer.addOccurrence(charsArray, start, end, (int)UsageSearchContext.IN_PLAIN_TEXT);
-        }
-        else {
-          consumer.addOccurrence(chars11, start, end, (int)UsageSearchContext.IN_PLAIN_TEXT);
-        }
-      }
-    }, chars, 0, chars.length());
-    return consumer.getResult();
-  }
-}

platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/todo/PlainTextTodoIndexer.java

@@ -19,12 +19,15 @@ import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.patterns.StringPattern;
 import com.intellij.psi.impl.cache.impl.IndexPatternUtil;
 import com.intellij.psi.impl.cache.impl.OccurrenceConsumer;
+import com.intellij.psi.impl.cache.impl.id.IdIndexEntry;
+import com.intellij.psi.impl.cache.impl.id.PlainTextIdIndexer;
 import com.intellij.psi.search.IndexPattern;
 import com.intellij.util.indexing.FileContent;
 import org.jetbrains.annotations.NotNull;
 
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -41,6 +44,10 @@ public class PlainTextTodoIndexer extends VersionedTodoIndexer {
     String chars = inputData.getContentAsText().toString(); // matching strings is faster than HeapCharBuffer
     OccurrenceConsumer occurrenceConsumer = new OccurrenceConsumer(null, true);
     for (IndexPattern indexPattern : indexPatterns) {
+      if (!idIndexContainAllWords(indexPattern.getWordsToFindFirst(), indexPattern.isCaseSensitive(), inputData)) {
+        continue;
+      }
+
       Pattern pattern = indexPattern.getOptimizedIndexingPattern();
       try {
         if (pattern != null) {
@@ -66,4 +73,8 @@ public class PlainTextTodoIndexer extends VersionedTodoIndexer {
     return map;
   }
 
+  private static boolean idIndexContainAllWords(@NotNull List<String> words, boolean caseSensitive, @NotNull FileContent content) {
+    Map<IdIndexEntry, Integer> idIndexData = PlainTextIdIndexer.getIdIndexData(content);
+    return words.stream().allMatch(word -> idIndexData.containsKey(new IdIndexEntry(word, caseSensitive)));
+  }
 }

platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/todo/TodoIndexers.java

@@ -27,6 +27,10 @@ public final class TodoIndexers extends FileTypeExtension<DataIndexer<TodoIndexE
   }
 
   public static boolean needsTodoIndex(@NotNull VirtualFile file) {
+    if (FileBasedIndex.IGNORE_PLAIN_TEXT_FILES && file.getFileType() == PlainTextFileType.INSTANCE) {
+      return false;
+    }
+
     for (ExtraPlaceChecker checker : EP_NAME.getExtensionList()) {
       if (checker.accept(null, file)) {
         return true;
@@ -37,10 +41,6 @@ public final class TodoIndexers extends FileTypeExtension<DataIndexer<TodoIndexE
       return false;
     }
 
-    if (FileBasedIndex.IGNORE_PLAIN_TEXT_FILES && file.getFileType() == PlainTextFileType.INSTANCE) {
-      return false;
-    }
-
     return true;
   }
 

platform/lang-impl/src/com/intellij/find/impl/FindInProjectUtil.java

@@ -49,7 +49,6 @@ import com.intellij.usages.FindUsagesProcessPresentation;
 import com.intellij.usages.UsageView;
 import com.intellij.usages.UsageViewPresentation;
 import com.intellij.util.ArrayUtil;
-import com.intellij.util.Function;
 import com.intellij.util.PatternUtil;
 import com.intellij.util.Processor;
 import com.intellij.util.containers.ContainerUtil;
@@ -63,6 +62,7 @@ import javax.swing.*;
 import java.util.*;
 import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
+import java.util.stream.Collectors;
 
 public class FindInProjectUtil {
   private static final int USAGES_PER_READ_ACTION = 100;
@@ -440,29 +440,34 @@ public class FindInProjectUtil {
   }
 
   @NotNull
-  public static String buildStringToFindForIndicesFromRegExp(@NotNull String stringToFind, @NotNull Project project) {
-    if (!Registry.is("idea.regexp.search.uses.indices")) return "";
-
+  public static String extractStringToFind(@NotNull String regexp, @NotNull Project project) {
     return ReadAction.compute(() -> {
       final List<PsiElement> topLevelRegExpChars = getTopLevelRegExpChars("a", project);
-      if (topLevelRegExpChars.size() != 1) return "";
+      if (topLevelRegExpChars.size() != 1) return " ";
 
       // leave only top level regExpChars
-      return StringUtil.join(getTopLevelRegExpChars(stringToFind, project), new Function<PsiElement, String>() {
-        final Class regExpCharPsiClass = topLevelRegExpChars.get(0).getClass();
 
-        @Override
-        public String fun(PsiElement element) {
-          if (regExpCharPsiClass.isInstance(element)) {
-            String text = element.getText();
+      final Class regExpCharPsiClass = topLevelRegExpChars.get(0).getClass();
+      return getTopLevelRegExpChars(regexp, project)
+        .stream()
+        .map(psi -> {
+          if (regExpCharPsiClass.isInstance(psi)) {
+            String text = psi.getText();
             if (!text.startsWith("\\")) return text;
           }
           return " ";
-        }
-      }, "");
+        })
+        .collect(Collectors.joining());
     });
   }
 
+  @NotNull
+  public static String buildStringToFindForIndicesFromRegExp(@NotNull String stringToFind, @NotNull Project project) {
+    if (!Registry.is("idea.regexp.search.uses.indices")) return "";
+
+    return StringUtil.trim(StringUtil.join(extractStringToFind(stringToFind, project), " "));
+  }
+
   public static void initStringToFindFromDataContext(FindModel findModel, @NotNull DataContext dataContext) {
     Editor editor = CommonDataKeys.EDITOR.getData(dataContext);
     FindUtil.initStringToFindWithSelection(findModel, editor);

platform/lang-impl/src/com/intellij/psi/search/IndexPatternOptimizerImpl.java

@@ -0,0 +1,26 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.psi.search;
+
+import com.intellij.find.impl.FindInProjectUtil;
+import com.intellij.openapi.application.ApplicationManager;
+import com.intellij.openapi.project.ProjectManager;
+import com.intellij.openapi.util.text.StringUtil;
+import com.intellij.util.PlatformUtils;
+import org.jetbrains.annotations.ApiStatus;
+import org.jetbrains.annotations.NotNull;
+
+import java.util.Collections;
+import java.util.List;
+
+@ApiStatus.Internal
+public class IndexPatternOptimizerImpl implements IndexPatternOptimizer {
+  @Override
+  public @NotNull List<String> extractStringsToFind(@NotNull String regexp) {
+    // TODO some datagrip tests are not passed with unknown reason (no exception in logs but it's)
+    if (ApplicationManager.getApplication().isUnitTestMode() && PlatformUtils.isDataGrip()) {
+      return Collections.emptyList();
+    }
+    String stringToFind = FindInProjectUtil.extractStringToFind(regexp, ProjectManager.getInstance().getDefaultProject());
+    return StringUtil.getWordsIn(stringToFind);
+  }
+}

platform/platform-resources/src/META-INF/LangExtensions.xml

@@ -239,6 +239,9 @@
     <applicationService serviceInterface="com.intellij.execution.impl.NewRunConfigurationTreePopupFactory"
                         serviceImplementation="com.intellij.execution.impl.DefaultNewRunConfigurationTreePopupFactory"/>
 
+    <applicationService serviceInterface="com.intellij.psi.search.IndexPatternOptimizer"
+                        serviceImplementation="com.intellij.psi.search.IndexPatternOptimizerImpl"/>
+
     <projectService serviceImplementation="com.intellij.psi.codeStyle.ProjectCodeStyleSettingsManager" preload="true"/>
 
     <projectService serviceImplementation="com.intellij.psi.codeStyle.LegacyCodeStyleSettingsManager"/>
@@ -571,7 +574,7 @@
     <syntaxHighlighter factoryClass="com.intellij.openapi.fileTypes.LanguageFileTypeHighlighterProvider"/>
     <structureViewBuilder factoryClass="com.intellij.ide.highlighter.LanguageFileTypeStructureViewBuilderProvider"/>
 
-    <idIndexer filetype="PLAIN_TEXT" implementationClass="com.intellij.psi.impl.cache.impl.id.PlainTextIndexer"/>
+    <idIndexer filetype="PLAIN_TEXT" implementationClass="com.intellij.psi.impl.cache.impl.id.PlainTextIdIndexer"/>
     <todoIndexer filetype="PLAIN_TEXT" implementationClass="com.intellij.psi.impl.cache.impl.todo.PlainTextTodoIndexer"/>
     <lang.elementManipulator forClass="com.intellij.psi.PsiPlainTextFile"
                              implementationClass="com.intellij.psi.impl.source.resolve.reference.impl.manipulators.PlainFileManipulator"/>

plugins/textmate/src/META-INF/plugin.xml

@@ -34,7 +34,7 @@
     <lang.commenter language="textmate" implementationClass="org.jetbrains.plugins.textmate.editor.TextMateCommentProvider"/>
     <customLiveTemplate implementation="org.jetbrains.plugins.textmate.editor.TextMateCustomLiveTemplate"/>
     <todoIndexer filetype="textmate" implementationClass="com.intellij.psi.impl.cache.impl.todo.PlainTextTodoIndexer"/>
-    <idIndexer filetype="textmate" implementationClass="com.intellij.psi.impl.cache.impl.id.PlainTextIndexer"/>
+    <idIndexer filetype="textmate" implementationClass="com.intellij.psi.impl.cache.impl.id.PlainTextIdIndexer"/>
     <indexPatternBuilder implementation="org.jetbrains.plugins.textmate.TextMatePatternBuilder"/>
     <spellchecker.support language="textmate" implementationClass="org.jetbrains.plugins.textmate.spellchecker.TextMateSpellingStrategy"/>
     <completion.contributor language="textmate" implementationClass="org.jetbrains.plugins.textmate.editor.TextMateCompletionContributor"/>