dfa: prevent exponential memory usage in live var analyzer (IDEA-131285)

* skip "not interesting" instructions
* cancel dfa after processing too many different states

java/java-analysis-impl/src/com/intellij/codeInspection/dataFlow/LiveVariablesAnalyzer.java

@@ -19,6 +19,7 @@ import com.intellij.codeInspection.dataFlow.instructions.*;
 import com.intellij.codeInspection.dataFlow.value.DfaValue;
 import com.intellij.codeInspection.dataFlow.value.DfaValueFactory;
 import com.intellij.codeInspection.dataFlow.value.DfaVariableValue;
+import com.intellij.openapi.progress.ProgressManager;
 import com.intellij.openapi.util.Pair;
 import com.intellij.util.PairFunction;
 import com.intellij.util.containers.ContainerUtil;
@@ -26,6 +27,7 @@ import com.intellij.util.containers.FilteringIterator;
 import com.intellij.util.containers.MultiMap;
 import com.intellij.util.containers.Queue;
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
 
 import java.util.*;
 
@@ -35,11 +37,13 @@ import java.util.*;
 public class LiveVariablesAnalyzer {
   private final DfaValueFactory myFactory;
   private final Instruction[] myInstructions;
+  private final MultiMap<Instruction, Instruction> myForwardMap;
   private final MultiMap<Instruction, Instruction> myBackwardMap;
 
   public LiveVariablesAnalyzer(ControlFlow flow, DfaValueFactory factory) {
     myFactory = factory;
     myInstructions = flow.getInstructions();
+    myForwardMap = calcForwardMap();
     myBackwardMap = calcBackwardMap();
   }
 
@@ -64,17 +68,49 @@ public class LiveVariablesAnalyzer {
   private MultiMap<Instruction, Instruction> calcBackwardMap() {
     MultiMap<Instruction, Instruction> result = MultiMap.create();
     for (Instruction instruction : myInstructions) {
-      for (Instruction next : getSuccessors(instruction)) {
+      for (Instruction next : myForwardMap.get(instruction)) {
         result.putValue(next, instruction);
       }
     }
     return result;
   }
 
+  private MultiMap<Instruction, Instruction> calcForwardMap() {
+    MultiMap<Instruction, Instruction> result = MultiMap.create();
+    for (Instruction instruction : myInstructions) {
+      if (isInterestingInstruction(instruction)) {
+        for (Instruction next : getSuccessors(instruction)) {
+          while (true) {
+            if (isInterestingInstruction(next)) {
+              result.putValue(instruction, next);
+              break;
+            }
+            if (next.getIndex() + 1 >= myInstructions.length) {
+              break;
+            }
+            next = myInstructions[next.getIndex() + 1];
+          }
+        }
+      }
+    }
+    return result;
+  }
+
+  private boolean isInterestingInstruction(Instruction instruction) {
+    if (instruction == myInstructions[0]) return true;
+    if (instruction instanceof PushInstruction) return ((PushInstruction)instruction).getValue() instanceof DfaVariableValue;
+    return instruction instanceof FinishElementInstruction ||
+           instruction instanceof FlushVariableInstruction ||
+           instruction instanceof GotoInstruction ||
+           instruction instanceof ConditionalGotoInstruction ||
+           instruction instanceof ReturnInstruction;
+  }
+
+  @Nullable
   private Map<FinishElementInstruction, BitSet> findLiveVars() {
     final Map<FinishElementInstruction, BitSet> result = ContainerUtil.newHashMap();
 
-    runDfa(false, new PairFunction<Instruction, BitSet, BitSet>() {
+    boolean ok = runDfa(false, new PairFunction<Instruction, BitSet, BitSet>() {
       @Override
       public BitSet fun(Instruction instruction, BitSet liveVars) {
         if (instruction instanceof FinishElementInstruction) {
@@ -115,13 +151,16 @@ public class LiveVariablesAnalyzer {
         return liveVars;
       }
     });
-    return result;
+    return ok ? result : null;
   }
 
   void flushDeadVariablesOnStatementFinish() {
     final Map<FinishElementInstruction, BitSet> liveVars = findLiveVars();
+    if (liveVars == null) return;
 
-    runDfa(true, new PairFunction<Instruction, BitSet, BitSet>() {
+    final MultiMap<FinishElementInstruction, DfaVariableValue> toFlush = MultiMap.createSet();
+
+    boolean ok = runDfa(true, new PairFunction<Instruction, BitSet, BitSet>() {
       @Override
       @NotNull
       public BitSet fun(Instruction instruction, @NotNull BitSet prevLiveVars) {
@@ -135,7 +174,7 @@ public class LiveVariablesAnalyzer {
             int setBit = prevLiveVars.nextSetBit(index);
             if (setBit < 0) break;
             if (!currentlyLive.get(setBit)) {
-              ((FinishElementInstruction)instruction).getVarsToFlush().add((DfaVariableValue)myFactory.getValue(setBit));
+              toFlush.putValue((FinishElementInstruction)instruction, (DfaVariableValue)myFactory.getValue(setBit));
             }
             index = setBit + 1;
           }
@@ -145,9 +184,18 @@ public class LiveVariablesAnalyzer {
         return prevLiveVars;
       }
     });
+
+    if (ok) {
+      for (FinishElementInstruction instruction : toFlush.keySet()) {
+        instruction.getVarsToFlush().addAll(toFlush.get(instruction));
+      }
+    }
   }
 
-  private void runDfa(boolean forward, PairFunction<Instruction, BitSet, BitSet> handleState) {
+  /**
+   * @return true if completed, false if "too complex"
+   */
+  private boolean runDfa(boolean forward, PairFunction<Instruction, BitSet, BitSet> handleState) {
     Set<Instruction> entryPoints = ContainerUtil.newHashSet();
     if (forward) {
       entryPoints.add(myInstructions[0]);
@@ -160,25 +208,28 @@ public class LiveVariablesAnalyzer {
       queue.addLast(new InstructionState(i, new BitSet()));
     }
 
-    int steps = 0;
+    int limit = myForwardMap.size() * 20;
     Set<InstructionState> processed = ContainerUtil.newHashSet();
     while (!queue.isEmpty()) {
-      steps++;
+      int steps = processed.size();
+      if (steps > limit) {
+        return false;
+      }
+      if (steps % 1024 == 0) {
+        ProgressManager.checkCanceled();
+      }
       InstructionState state = queue.pullFirst();
       Instruction instruction = state.first;
-      Collection<Instruction> nextInstructions = forward ? getSuccessors(instruction) : myBackwardMap.get(instruction);
-      boolean branching = nextInstructions.size() > 1 || !forward && instruction.getIndex() == 0;
+      Collection<Instruction> nextInstructions = forward ? myForwardMap.get(instruction) : myBackwardMap.get(instruction);
       BitSet nextVars = handleState.fun(instruction, state.second);
       for (Instruction next : nextInstructions) {
         InstructionState nextState = new InstructionState(next, nextVars);
-        if (!branching || processed.add(nextState)) {
+        if (processed.add(nextState)) {
           queue.addLast(nextState);
         }
       }
     }
-    if (steps > 10000) {
-      int a = 1;
-    }
+    return true;
   }
 
   private static class InstructionState extends Pair<Instruction, BitSet> {