if some no-statistic suggestions are highly-preferred (e.g. local vars), move statistically preferred variants under them (IDEA-98303)

java/java-tests/testData/codeInsight/completion/normalSorting/LocalVarsOverStats.java

@@ -0,0 +1,8 @@
+public class Foo {
+  
+  void foo(PsiElement psiElement) {
+    psiEl<caret>
+  }
+}
+
+class PsiElement {}

java/java-tests/testSrc/com/intellij/codeInsight/completion/NormalCompletionOrderingTest.groovy

@@ -470,7 +470,7 @@ import java.lang.annotation.Target;
     repeatCompletion 'b'
 
     myFixture.completeBasic();
-    assertPreferredItems(0, '_boo2', '_foo2', 'return', '_boo1', '_foo1', '_goo1', '_goo2')
+    assertPreferredItems(0, 'return', '_boo2', '_foo2', '_boo1', '_foo1', '_goo1', '_goo2')
     myFixture.type('_');
     assertPreferredItems(0, '_boo2', '_foo2', '_boo1', '_foo1', '_goo1', '_goo2')
     myFixture.type('g')
@@ -549,4 +549,11 @@ import java.lang.annotation.Target;
     checkPreferredItems(0, 'vx')
   }
 
+  public void testLocalVarsOverStats() {
+    CodeInsightSettings.getInstance().COMPLETION_CASE_SENSITIVE = CodeInsightSettings.NONE;
+    checkPreferredItems 0, 'psiElement', 'PsiElement'
+    incUseCount lookup, 1
+    assertPreferredItems 0, 'psiElement', 'PsiElement'
+  }
+
 }

platform/lang-impl/src/com/intellij/codeInsight/completion/StatisticsWeigher.java

@@ -15,18 +15,24 @@
  */
 package com.intellij.codeInsight.completion;
 
+import com.intellij.codeInsight.lookup.Classifier;
 import com.intellij.codeInsight.lookup.LookupElement;
-import com.intellij.codeInsight.lookup.LookupElementWeigher;
 import com.intellij.openapi.application.ApplicationManager;
 import com.intellij.openapi.diagnostic.Logger;
+import com.intellij.openapi.util.Condition;
 import com.intellij.openapi.util.Key;
 import com.intellij.psi.statistics.StatisticsInfo;
 import com.intellij.psi.statistics.StatisticsManager;
+import com.intellij.util.ProcessingContext;
+import com.intellij.util.SmartList;
+import com.intellij.util.containers.ContainerUtil;
+import com.intellij.util.containers.FlatteningIterator;
+import gnu.trove.THashSet;
+import gnu.trove.TObjectHashingStrategy;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
-import java.util.ArrayList;
-import java.util.List;
+import java.util.*;
 
 /**
  * @author peter
@@ -41,17 +47,101 @@ public class StatisticsWeigher extends CompletionWeigher {
     throw new UnsupportedOperationException();
   }
 
-  public static class LookupStatisticsWeigher extends LookupElementWeigher {
+  public static class LookupStatisticsWeigher extends Classifier<LookupElement> {
     private final CompletionLocation myLocation;
+    private final Classifier<LookupElement> myNext;
+    private final Map<LookupElement, Integer> myWeights = new IdentityHashMap<LookupElement, Integer>();
+    @SuppressWarnings("unchecked") private final Set<LookupElement> myNoStats = new THashSet<LookupElement>(TObjectHashingStrategy.IDENTITY);
+    private int myPrefixChanges;
 
-    public LookupStatisticsWeigher(CompletionLocation location) {
-      super("stats", true, true);
+    public LookupStatisticsWeigher(CompletionLocation location, Classifier<LookupElement> next) {
       myLocation = location;
+      myNext = next;
     }
 
     @Override
-    public Integer weigh(@NotNull LookupElement item) {
-      final StatisticsInfo baseInfo = getBaseStatisticsInfo(item, myLocation);
+    public void addElement(LookupElement element) {
+      StatisticsInfo baseInfo = getBaseStatisticsInfo(element, myLocation);
+      myWeights.put(element, weigh(element, baseInfo));
+      if (baseInfo == StatisticsInfo.EMPTY) {
+        myNoStats.add(element);
+      }
+      myNext.addElement(element);
+    }
+
+    private void checkPrefixChanged(ProcessingContext context) {
+      int actualPrefixChanges = context.get(CompletionLookupArranger.PREFIX_CHANGES).intValue();
+      if (myPrefixChanges != actualPrefixChanges) {
+        myPrefixChanges = actualPrefixChanges;
+        myWeights.clear();
+      }
+    }
+
+    @Override
+    public Iterable<LookupElement> classify(Iterable<LookupElement> source, final ProcessingContext context) {
+      checkPrefixChanged(context);
+
+      final Collection<List<LookupElement>> byWeight = buildMapByWeight(source).descendingMap().values();
+
+      List<LookupElement> initialList = getInitialNoStatElements(source, context);
+
+      //noinspection unchecked
+      final THashSet<LookupElement> initialSet = new THashSet<LookupElement>(initialList, TObjectHashingStrategy.IDENTITY);
+      final Condition<LookupElement> notInInitialList = new Condition<LookupElement>() {
+        @Override
+        public boolean value(LookupElement element) {
+          return !initialSet.contains(element);
+        }
+      };
+
+      return ContainerUtil.concat(initialList, new Iterable<LookupElement>() {
+        @Override
+        public Iterator<LookupElement> iterator() {
+          return new FlatteningIterator<List<LookupElement>, LookupElement>(byWeight.iterator()) {
+            @Override
+            protected Iterator<LookupElement> createValueIterator(List<LookupElement> group) {
+              return myNext.classify(ContainerUtil.findAll(group, notInInitialList), context).iterator();
+            }
+          };
+        }
+      });
+    }
+
+    private List<LookupElement> getInitialNoStatElements(Iterable<LookupElement> source, ProcessingContext context) {
+      List<LookupElement> initialList = new ArrayList<LookupElement>();
+      for (LookupElement next : myNext.classify(source, context)) {
+        if (myNoStats.contains(next)) {
+          initialList.add(next);
+        }
+        else {
+          break;
+        }
+      }
+      return initialList;
+    }
+
+    private TreeMap<Integer, List<LookupElement>> buildMapByWeight(Iterable<LookupElement> source) {
+      TreeMap<Integer, List<LookupElement>> map = new TreeMap<Integer, List<LookupElement>>();
+      for (LookupElement element : source) {
+        final int weight = getWeight(element);
+        List<LookupElement> list = map.get(weight);
+        if (list == null) {
+          map.put(weight, list = new SmartList<LookupElement>());
+        }
+        list.add(element);
+      }
+      return map;
+    }
+
+    private int getWeight(LookupElement t) {
+      Integer w = myWeights.get(t);
+      if (w == null) {
+        myWeights.put(t, w = weigh(t, getBaseStatisticsInfo(t, myLocation)));
+      }
+      return w;
+    }
+
+    private int weigh(@NotNull LookupElement item, final StatisticsInfo baseInfo) {
       if (baseInfo == StatisticsInfo.EMPTY) {
         return 0;
       }
@@ -65,6 +155,18 @@ public class StatisticsWeigher extends CompletionWeigher {
       return minRecency == Integer.MAX_VALUE ? maxUseCount : 100 - minRecency;
     }
 
+    @Override
+    public void describeItems(LinkedHashMap<LookupElement, StringBuilder> map, ProcessingContext context) {
+      checkPrefixChanged(context);
+      for (LookupElement element : map.keySet()) {
+        StringBuilder builder = map.get(element);
+        if (builder.length() > 0) {
+          builder.append(", ");
+        }
+        builder.append("stats=").append(getWeight(element));
+      }
+      myNext.describeItems(map, context);
+    }
   }
 
   public static void clearBaseStatisticsInfo(LookupElement item) {

platform/lang-impl/src/com/intellij/codeInsight/completion/impl/CompletionServiceImpl.java

@@ -246,7 +246,12 @@ public class CompletionServiceImpl extends CompletionService{
         sorter = sorter.withClassifier(CompletionSorterImpl.weighingFactory(new RealPrefixMatchingWeigher(location)));
       }
       else if ("stats".equals(id)) {
-        sorter = sorter.withClassifier(CompletionSorterImpl.weighingFactory(new StatisticsWeigher.LookupStatisticsWeigher(location)));
+        sorter = sorter.withClassifier(new ClassifierFactory<LookupElement>("stats") {
+          @Override
+          public Classifier<LookupElement> createClassifier(Classifier<LookupElement> next) {
+            return new StatisticsWeigher.LookupStatisticsWeigher(location, next);
+          }
+        });
       }
       else {
         sorter = sorter.weigh(new LookupElementWeigher(id, true, false) {