make completion stats prefix-independent, make prefix more important a weighing criterion (IDEA-165932, also needed for IDEA-166442)

java/java-impl/src/com/intellij/codeInsight/completion/JavaCompletionSorting.java

@@ -33,7 +33,6 @@ import com.intellij.psi.javadoc.PsiDocComment;
 import com.intellij.psi.util.PsiTreeUtil;
 import com.intellij.psi.util.PsiUtil;
 import com.intellij.psi.util.TypeConversionUtil;
-import com.intellij.util.Function;
 import com.intellij.util.SmartList;
 import com.intellij.util.containers.ContainerUtil;
 import gnu.trove.THashSet;
@@ -72,22 +71,22 @@ public class JavaCompletionSorting {
       sorter = sorter.weighAfter("priority", new PreferDefaultTypeWeigher(expectedTypes, parameters));
     }
 
-    List<LookupElementWeigher> afterPrefix = ContainerUtil.newArrayList();
-    afterPrefix.add(new PreferByKindWeigher(type, position, expectedTypes));
+    List<LookupElementWeigher> afterStats = ContainerUtil.newArrayList();
+    afterStats.add(new PreferByKindWeigher(type, position, expectedTypes));
     if (!smart) {
-      ContainerUtil.addIfNotNull(afterPrefix, preferStatics(position, expectedTypes));
+      ContainerUtil.addIfNotNull(afterStats, preferStatics(position, expectedTypes));
       if (!afterNew) {
-        afterPrefix.add(new PreferExpected(false, expectedTypes, position));
+        afterStats.add(new PreferExpected(false, expectedTypes, position));
       }
     }
-    ContainerUtil.addIfNotNull(afterPrefix, recursion(parameters, expectedTypes));
-    afterPrefix.add(new PreferSimilarlyEnding(expectedTypes));
+    ContainerUtil.addIfNotNull(afterStats, recursion(parameters, expectedTypes));
+    afterStats.add(new PreferSimilarlyEnding(expectedTypes));
     if (ContainerUtil.or(expectedTypes, info -> !info.getType().equals(PsiType.VOID))) {
-      afterPrefix.add(new PreferNonGeneric());
+      afterStats.add(new PreferNonGeneric());
     }
-    Collections.addAll(afterPrefix, new PreferAccessible(position), new PreferSimple());
+    Collections.addAll(afterStats, new PreferAccessible(position), new PreferSimple());
 
-    sorter = sorter.weighAfter("prefix", afterPrefix.toArray(new LookupElementWeigher[afterPrefix.size()]));
+    sorter = sorter.weighAfter("stats", afterStats.toArray(new LookupElementWeigher[afterStats.size()]));
     sorter = sorter.weighAfter("proximity", afterProximity.toArray(new LookupElementWeigher[afterProximity.size()]));
     return result.withRelevanceSorter(sorter);
   }

java/java-tests/testData/codeInsight/completion/normalSorting/AbandonSameStatsForDifferentQualifiers.java

@@ -1,11 +0,0 @@
-class Foo1 { boolean method1() {} }
-class Foo2 { boolean method2() {} }
-class Foo3 { boolean method3() {} }
-
-public class MyFirstTestClassFoo {
-
-  void foo(Foo1 f1, Foo2 f2, Foo3 f3) {
-    f1.<caret>
-  }
-
-}

java/java-tests/testData/codeInsight/completion/normalSorting/ExpectedTypeIsMoreImportantThanCase.java

@@ -1,9 +0,0 @@
-class Foo {
-  boolean ENABLED;
-  void enable() {}
-
-  {
-    if (!en<caret>)
-  }
-
-}

java/java-tests/testData/codeInsight/completion/normalSorting/StatisticsByPrefix.java

@@ -1,6 +0,0 @@
-class FooBar {
-  int _foo1, _foo2, _boo1, _boo2, _goo1, _goo2;
-  {
-    <caret>
-  }
-}

java/java-tests/testSrc/com/intellij/codeInsight/completion/NormalCompletionOrderingTest.groovy

@@ -27,7 +27,6 @@ import com.intellij.openapi.fileEditor.FileDocumentManager
 import com.intellij.psi.PsiClass
 import com.intellij.psi.PsiField
 import com.intellij.psi.PsiMethod
-import com.intellij.psi.statistics.StatisticsManager
 import com.intellij.ui.JBColor
 
 class NormalCompletionOrderingTest extends CompletionSortingTestCase {
@@ -199,27 +198,6 @@ class NormalCompletionOrderingTest extends CompletionSortingTestCase {
     assertPreferredItems(0, "getComponents", "getComponent")
   }
 
-  void testAbandonSameStatsForDifferentQualifiers() throws Throwable {
-    invokeCompletion(getTestName(false) + ".java")
-    assertPreferredItems 0, "method1", "equals"
-    myFixture.type('eq\n2);\nf2.')
-
-    myFixture.completeBasic()
-    assertPreferredItems 0, "equals", "method2"
-    myFixture.type('me\n);\n')
-
-    for (i in 0..StatisticsManager.OBLIVION_THRESHOLD) {
-      myFixture.type('f2.')
-      myFixture.completeBasic()
-      assertPreferredItems 0, "method2", "equals"
-      myFixture.type('me\n);\n')
-    }
-
-    myFixture.type('f3.')
-    myFixture.completeBasic()
-    assertPreferredItems 0, "method3", "equals"
-  }
-
   void testDispreferFinalize() throws Throwable {
     checkPreferredItems(0, "final", "finalize")
   }
@@ -393,13 +371,6 @@ class NormalCompletionOrderingTest extends CompletionSortingTestCase {
     checkPreferredItems(1, "Foo", "foo1", "foo2")
   }
 
-  void testExpectedTypeIsMoreImportantThanCase() {
-    CodeInsightSettings.getInstance().COMPLETION_CASE_SENSITIVE = CodeInsightSettings.NONE
-    checkPreferredItems 0, "enable", "ENABLED"
-    incUseCount(lookup, 1)
-    assertPreferredItems 0, "ENABLED", "enable"
-  }
-
   void testPreferKeywordsToVoidMethodsInExpectedTypeContext() {
     checkPreferredItems 0, 'noo', 'new', 'null', 'noo2', 'notify', 'notifyAll'
   }
@@ -516,41 +487,6 @@ interface TxANotAnno {}
     assert lookup.currentItem.lookupString == 'JComponent'
   }
 
-  void testStatisticsByPrefix() {
-    Closure repeatCompletion = { String letter ->
-      String var1 = "_${letter}oo1"
-      String var2 = "_${letter}oo2"
-
-      myFixture.type("_$letter")
-      myFixture.completeBasic()
-      assertPreferredItems(0, var1, var2)
-      myFixture.type('2\n;\n')
-
-      for (i in 0..<StatisticsManager.OBLIVION_THRESHOLD - 2) {
-        myFixture.type('_')
-        myFixture.completeBasic()
-        assert myFixture.lookupElementStrings.indexOf(var2) < myFixture.lookupElementStrings.indexOf(var1)
-        myFixture.type(letter)
-        assertPreferredItems(0, var2, var1)
-        myFixture.type('\n;\n')
-      }
-    }
-
-    configureByFile(getTestName(false) + ".java")
-    repeatCompletion 'g'
-    repeatCompletion 'f'
-    repeatCompletion 'b'
-
-    myFixture.completeBasic()
-    assertPreferredItems(0, 'return', '_boo2', '_foo2', '_boo1', '_foo1', '_goo1', '_goo2')
-    myFixture.type('_')
-    assertPreferredItems(0, '_boo2', '_foo2', '_boo1', '_foo1', '_goo1', '_goo2')
-    myFixture.type('g')
-    assertPreferredItems(0, '_goo2', '_goo1')
-    myFixture.type('o')
-    assertPreferredItems(0, '_goo2', '_goo1')
-  }
-
   void testPreferFieldToMethod() {
     checkPreferredItems(0, 'size', 'size')
     assert lookup.items[0].object instanceof PsiField

java/java-tests/testSrc/com/intellij/codeInsight/completion/SmartTypeCompletionOrderingTest.groovy

@@ -53,13 +53,6 @@ class SmartTypeCompletionOrderingTest extends CompletionSortingTestCase {
     }
     refreshSorting(lookup)
     assertPreferredItems(2, "Component", "String", "Container", "FooBean3", "JComponent")
-
-    int component = lookup.items.findIndexOf { it.lookupString == 'Component' }
-    for (int i = 0; i < StatisticsManager.OBLIVION_THRESHOLD; i++) {
-      imitateItemSelection(lookup, component)
-    }
-    refreshSorting(lookup)
-    assertPreferredItems(1, "String", "Component", "FooBean3")
   }
 
   void testNewListAlwaysFirst() {

platform/lang-impl/src/com/intellij/codeInsight/completion/CompletionLookupArranger.java

@@ -456,7 +456,7 @@ public class CompletionLookupArranger extends LookupArranger {
       return new StatisticsUpdate(StatisticsInfo.EMPTY);
     }
 
-    StatisticsUpdate update = new StatisticsUpdate(StatisticsWeigher.composeStatsWithPrefix(base, lookup.itemPattern(item), true));
+    StatisticsUpdate update = new StatisticsUpdate(base);
     ourPendingUpdate = update;
     Disposer.register(update, new Disposable() {
       @Override

platform/lang-impl/src/com/intellij/codeInsight/completion/StatisticsWeigher.java

@@ -17,7 +17,6 @@ package com.intellij.codeInsight.completion;
 
 import com.intellij.codeInsight.lookup.Classifier;
 import com.intellij.codeInsight.lookup.LookupElement;
-import com.intellij.codeInsight.lookup.WeighingContext;
 import com.intellij.openapi.application.ApplicationManager;
 import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.openapi.util.Condition;
@@ -25,7 +24,6 @@ import com.intellij.openapi.util.Key;
 import com.intellij.openapi.util.Pair;
 import com.intellij.psi.statistics.StatisticsInfo;
 import com.intellij.psi.statistics.StatisticsManager;
-import com.intellij.util.Function;
 import com.intellij.util.ProcessingContext;
 import com.intellij.util.SmartList;
 import com.intellij.util.containers.ContainerUtil;
@@ -63,7 +61,7 @@ public class StatisticsWeigher extends CompletionWeigher {
     @Override
     public void addElement(@NotNull LookupElement element, @NotNull ProcessingContext context) {
       StatisticsInfo baseInfo = getBaseStatisticsInfo(element, myLocation);
-      myWeights.put(element, new StatisticsComparable(weigh(element, baseInfo, context.get(CompletionLookupArranger.WEIGHING_CONTEXT)), baseInfo));
+      myWeights.put(element, new StatisticsComparable(weigh(baseInfo), baseInfo));
       if (baseInfo == StatisticsInfo.EMPTY) {
         myNoStats.add(element);
       }
@@ -83,7 +81,7 @@ public class StatisticsWeigher extends CompletionWeigher {
     public Iterable<LookupElement> classify(@NotNull Iterable<LookupElement> source, @NotNull final ProcessingContext context) {
       checkPrefixChanged(context);
 
-      final Collection<List<LookupElement>> byWeight = buildMapByWeight(source, context).descendingMap().values();
+      final Collection<List<LookupElement>> byWeight = buildMapByWeight(source).descendingMap().values();
 
       List<LookupElement> initialList = getInitialNoStatElements(source, context);
 
@@ -117,10 +115,10 @@ public class StatisticsWeigher extends CompletionWeigher {
       return initialList;
     }
 
-    private TreeMap<Integer, List<LookupElement>> buildMapByWeight(Iterable<LookupElement> source, ProcessingContext context) {
+    private TreeMap<Integer, List<LookupElement>> buildMapByWeight(Iterable<LookupElement> source) {
       TreeMap<Integer, List<LookupElement>> map = new TreeMap<>();
       for (LookupElement element : source) {
-        final int weight = getWeight(element, context.get(CompletionLookupArranger.WEIGHING_CONTEXT)).getScalar();
+        final int weight = getWeight(element).getScalar();
         List<LookupElement> list = map.get(weight);
         if (list == null) {
           map.put(weight, list = new SmartList<>());
@@ -130,31 +128,28 @@ public class StatisticsWeigher extends CompletionWeigher {
       return map;
     }
 
-    private StatisticsComparable getWeight(LookupElement t, WeighingContext context) {
+    private StatisticsComparable getWeight(LookupElement t) {
       StatisticsComparable w = myWeights.get(t);
       if (w == null) {
         StatisticsInfo info = getBaseStatisticsInfo(t, myLocation);
-        myWeights.put(t, w = new StatisticsComparable(weigh(t, info, context), info));
+        myWeights.put(t, w = new StatisticsComparable(weigh(info), info));
       }
       return w;
     }
 
-    private static int weigh(@NotNull LookupElement item, final StatisticsInfo baseInfo, WeighingContext context) {
+    private static int weigh(final StatisticsInfo baseInfo) {
       if (baseInfo == StatisticsInfo.EMPTY) {
         return 0;
       }
-      String prefix = context.itemPattern(item);
-      StatisticsInfo composed = composeStatsWithPrefix(baseInfo, prefix, false);
-      int minRecency = composed.getLastUseRecency();
-      int useCount = composed.getUseCount();
-      return minRecency == Integer.MAX_VALUE ? useCount : 100 - minRecency;
+      int minRecency = baseInfo.getLastUseRecency();
+      return minRecency == Integer.MAX_VALUE ? 0 : StatisticsManager.RECENCY_OBLIVION_THRESHOLD - minRecency;
     }
 
     @NotNull
     @Override
     public List<Pair<LookupElement, Object>> getSortingWeights(@NotNull Iterable<LookupElement> items, @NotNull final ProcessingContext context) {
       checkPrefixChanged(context);
-      return ContainerUtil.map(items, lookupElement -> new Pair<LookupElement, Object>(lookupElement, getWeight(lookupElement, context.get(CompletionLookupArranger.WEIGHING_CONTEXT))));
+      return ContainerUtil.map(items, lookupElement -> new Pair<LookupElement, Object>(lookupElement, getWeight(lookupElement)));
     }
 
     @Override
@@ -190,41 +185,4 @@ public class StatisticsWeigher extends CompletionWeigher {
     return info == null ? StatisticsInfo.EMPTY : info;
   }
 
-  /**
-   * For different prefixes we want to prefer different completion items,
-   *   so we decorate their basic stat-infos depending on prefix.
-   * For example, consider that an item "fooBar" was chosen with a prefix "foo"
-   * Then we'll register "fooBar" for each of the sub-prefixes: "", "f", "fo" and "foo"
-   *   and suggest "foobar" whenever we a user types any of those prefixes
-   *
-   * If a user has typed "fooB" for which there's no stat-info registered, we want to check
-   *   all of its sub-prefixes: "", "f", "fo", "foo" and see if any of them is associated with a stat-info
-   * But if the item were "fobia" and the user has typed "fob", we don't want to claim
-   *   that "fooBar" (which matches) is statistically better than "fobia" with prefix "fob" even though both begin with "fo"
-   * So we only check non-partial sub-prefixes, then ones that had been really typed by the user before completing
-   *
-   * @param forWriting controls whether this stat-info will be used for incrementing usage count or for its retrieval (for sorting)
-   */
-  public static StatisticsInfo composeStatsWithPrefix(StatisticsInfo info, final String fullPrefix, boolean forWriting) {
-    ArrayList<StatisticsInfo> infos = new ArrayList<>((fullPrefix.length() + 3) * info.getConjuncts().size());
-    for (StatisticsInfo conjunct : info.getConjuncts()) {
-      if (forWriting) {
-        // some completion contributors may need pure statistical information to speed up searching for frequently chosen items
-        infos.add(conjunct);
-      }
-      for (int i = 0; i <= fullPrefix.length(); i++) {
-        // if we're incrementing usage count, register all sub-prefixes with "partial" mark
-        // if we're sorting and any sub-prefix was used as non-partial to choose this completion item, prefer it
-        infos.add(composeWithPrefix(conjunct, fullPrefix.substring(0, i), forWriting));
-      }
-      // if we're incrementing usage count, the full prefix is registered as non-partial
-      // if we're sorting and the current prefix was used as partial sub-prefix to choose this completion item, prefer it
-      infos.add(composeWithPrefix(conjunct, fullPrefix, !forWriting));
-    }
-    return StatisticsInfo.createComposite(infos);
-  }
-
-  private static StatisticsInfo composeWithPrefix(StatisticsInfo info, String fullPrefix, boolean partial) {
-    return new StatisticsInfo(info.getContext() + "###prefix=" + fullPrefix + "###part#" + partial, info.getValue());
-  }
 }

platform/platform-resources/src/META-INF/LangExtensions.xml

@@ -543,12 +543,12 @@
              order="after inResolveScope"/>
 
     <weigher key="completion" implementationClass="com.intellij.codeInsight.completion.PriorityWeigher" id="priority" order="first"/>
-    <weigher key="completion" implementationClass="com.intellij.codeInsight.completion.StatisticsWeigher" id="stats"
-             order="after priority"/>
     <weigher key="completion" implementationClass="com.intellij.codeInsight.completion.PrefixMatchingWeigher" id="prefix"
-             order="after stats"/>
-    <weigher key="completion" implementationClass="com.intellij.codeInsight.completion.ExplicitProximityWeigher" id="explicitProximity"
+             order="after priority"/>
+    <weigher key="completion" implementationClass="com.intellij.codeInsight.completion.StatisticsWeigher" id="stats"
              order="after prefix"/>
+    <weigher key="completion" implementationClass="com.intellij.codeInsight.completion.ExplicitProximityWeigher" id="explicitProximity"
+             order="after stats"/>
     <weigher key="completion" implementationClass="com.intellij.codeInsight.completion.LookupElementProximityWeigher" id="proximity"
              order="after explicitProximity"/>
     <weigher key="completion" implementationClass="com.intellij.codeInsight.completion.GroupingWeigher" id="grouping"

plugins/groovy/src/META-INF/plugin.xml

@@ -487,7 +487,7 @@
                               implementation="org.jetbrains.plugins.groovy.lang.resolve.GroovyMethodArgumentReferenceContributor"/>
 
     <weigher key="completion" implementationClass="org.jetbrains.plugins.groovy.lang.completion.weighers.GrWithWeigher"
-             id="groovyWithWeigher" order="after prefix"/>
+             id="groovyWithWeigher" order="after prefix, after stats"/>
     <weigher key="completion" implementationClass="org.jetbrains.plugins.groovy.lang.completion.weighers.GrKindWeigher"
              id="groovyKindWeigher" order="after groovyWithWeigher, before explicitProximity"/>