Limit tree depth for which incremental reparse is allowed

java/java-tests/testSrc/com/intellij/codeInsight/daemon/LightAdvHighlightingTest.java

@@ -25,12 +25,10 @@ import com.intellij.openapi.fileTypes.StdFileTypes;
 import com.intellij.openapi.roots.LanguageLevelProjectExtension;
 import com.intellij.pom.java.LanguageLevel;
 import com.intellij.psi.*;
-import com.intellij.psi.impl.source.PostprocessReformattingAspect;
 import com.intellij.psi.xml.XmlAttribute;
 import com.intellij.psi.xml.XmlTag;
 import com.intellij.psi.xml.XmlToken;
 import com.intellij.psi.xml.XmlTokenType;
-import com.intellij.util.ui.UIUtil;
 import org.jdom.Element;
 import org.jetbrains.annotations.NonNls;
 import org.jetbrains.annotations.NotNull;
@@ -46,6 +44,7 @@ import static com.intellij.codeInsight.daemon.DaemonAnalyzerTestCase.filter;
  */
 public class LightAdvHighlightingTest extends LightDaemonAnalyzerTestCase {
   @NonNls static final String BASE_PATH = "/codeInsight/daemonCodeAnalyzer/advHighlighting";
+
   private UnusedSymbolLocalInspection myUnusedSymbolLocalInspection;
 
   private void doTest(boolean checkWarnings, boolean checkInfos) throws Exception {
@@ -176,15 +175,18 @@ public class LightAdvHighlightingTest extends LightDaemonAnalyzerTestCase {
   public void testMethodCannotBeApplied() throws Exception { doTest(false, false); }
 
   public void testUnusedParamsOfPublicMethod() throws Exception { doTest(true, false); }
+
   public void testUnusedParamsOfPublicMethodDisabled() throws Exception {
     myUnusedSymbolLocalInspection.REPORT_PARAMETER_FOR_PUBLIC_METHODS = false;
     doTest(true, false);
   }
+
   public void testUnusedNonPrivateMembers() throws Exception {
     UnusedDeclarationInspection deadCodeInspection = new UnusedDeclarationInspection();
     enableInspectionTool(deadCodeInspection);
     doTest(true, false);
   }
+
   public void testUnusedNonPrivateMembers2() throws Exception {
     ExtensionPoint<EntryPoint> point = Extensions.getRootArea().getExtensionPoint(ExtensionPoints.DEAD_CODE_TOOL);
     EntryPoint extension = new EntryPoint() {
@@ -228,7 +230,6 @@ public class LightAdvHighlightingTest extends LightDaemonAnalyzerTestCase {
     point.registerExtension(extension);
     
     try {
-
       UnusedDeclarationInspection deadCodeInspection = new UnusedDeclarationInspection();
       enableInspectionTool(deadCodeInspection);
 
@@ -237,7 +238,6 @@ public class LightAdvHighlightingTest extends LightDaemonAnalyzerTestCase {
     finally {
       point.unregisterExtension(extension);
     }
-
   }
 
   public void testNamesHighlighting() throws Exception {
@@ -264,6 +264,7 @@ public class LightAdvHighlightingTest extends LightDaemonAnalyzerTestCase {
       });
     }
   }
+
   public void testInjectedAnnotator() throws Exception {
     Annotator annotator = new MyAnnotator();
     Language xml = StdFileTypes.XML.getLanguage();
@@ -281,39 +282,33 @@ public class LightAdvHighlightingTest extends LightDaemonAnalyzerTestCase {
     assertFalse(list.toString(), list.contains(annotator));
   }
 
-  // todo does not work without nonrecursive reparse
-  public void _testSOEForTypeOfHugeBinaryExpression() throws IOException {
+  public void testSOEForTypeOfHugeBinaryExpression() throws IOException {
     configureFromFileText("a.java", "class A { String s = \"\"; }");
     assertEmpty(filter(doHighlighting(), HighlightSeverity.ERROR));
-
-    PsiField field = ((PsiJavaFile)getFile()).getClasses()[0].getFields()[0];
-
     PsiDocumentManager.getInstance(getProject()).commitAllDocuments();
-    // have to manipulate PSI, will get SOE in BlockSupportImpl otherwise
-    final PsiExpression literal = JavaPsiFacade.getElementFactory(getProject()).createExpressionFromText("\"xxx\"", field);
 
-    final PsiBinaryExpression binary = (PsiBinaryExpression)JavaPsiFacade.getElementFactory(getProject()).createExpressionFromText("a+b", field);
-    for (int i=0; i<2000;i++) {
-      final PsiExpression expression = field.getInitializer();
-      ApplicationManager.getApplication().runWriteAction(new Runnable() {
-        public void run() {
-          binary.getLOperand().replace(expression);
-          binary.getROperand().replace(literal);
+    final StringBuilder sb = new StringBuilder("\"-\"");
+    for (int i = 0; i < 10000; i++) sb.append("+\"b\"");
+    final String hugeExpr = sb.toString();
+    final int pos = getEditor().getDocument().getText().indexOf("\"\"");
 
-          expression.replace(binary);
-          PostprocessReformattingAspect.getInstance(getProject()).clear(); // OOM otherwise
-        }
-      });
+    ApplicationManager.getApplication().runWriteAction(new Runnable() {
+      public void run() {
+        getEditor().getDocument().replaceString(pos, pos + 2, hugeExpr);
+        PsiDocumentManager.getInstance(getProject()).commitAllDocuments();
+      }
+    });
 
-      UIUtil.dispatchAllInvocationEvents();
-    }
-
-    field.getInitializer().getType(); // SOE
+    final PsiField field = ((PsiJavaFile)getFile()).getClasses()[0].getFields()[0];
+    final PsiExpression expression = field.getInitializer();
+    assert expression != null;
+    final PsiType type = expression.getType();
+    assert type != null;
+    assertEquals("PsiType:String", type.toString());
   }
 
   public void testSOEForCyclicInheritance() throws IOException {
     configureFromFileText("a.java", "class A extends B { String s = \"\"; void f() {}} class B extends A { void f() {} } ");
-
     doHighlighting();
   }
 }

java/java-tests/testSrc/com/intellij/psi/JavaSOEOnReparseTest.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright 2000-2010 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.psi;
+
+import com.intellij.openapi.application.ApplicationManager;
+import com.intellij.testFramework.LightCodeInsightTestCase;
+
+
+public class JavaSOEOnReparseTest extends LightCodeInsightTestCase {
+  private static final String HUGE_EXPR;
+  static {
+    final StringBuilder sb = new StringBuilder("\"-\"");
+    for (int i = 0; i < 10000; i++) sb.append("+\"b\"");
+    HUGE_EXPR = sb.toString();
+  }
+
+  public void testOnHugeBinaryExprInFile() throws Exception {
+    configureFromFileText("a.java", "class A { String s = \"\"; }");
+    doTest();
+  }
+
+  public void testOnHugeBinaryExprInCodeBlock() throws Exception {
+    configureFromFileText("a.java", "class A { void m() { String s = \"\"; } }");
+    doTest();
+  }
+
+  private static void doTest() {
+    final int pos = getEditor().getDocument().getText().indexOf("\"\"");
+
+    // replace small expression with huge binary one
+    ApplicationManager.getApplication().runWriteAction(new Runnable() { public void run() {
+      getEditor().getDocument().replaceString(pos, pos + 2, HUGE_EXPR);
+      PsiDocumentManager.getInstance(getProject()).commitAllDocuments();
+    }});
+
+    // modify huge binary expression (1)
+    ApplicationManager.getApplication().runWriteAction(new Runnable() { public void run() {
+      getEditor().getDocument().insertString(pos, "\".\"+");
+      PsiDocumentManager.getInstance(getProject()).commitAllDocuments();
+    }});
+
+    // modify huge binary expression (2)
+    ApplicationManager.getApplication().runWriteAction(new Runnable() { public void run() {
+      getEditor().getDocument().replaceString(pos, pos + 4, "");
+      PsiDocumentManager.getInstance(getProject()).commitAllDocuments();
+    }});
+
+    // replace huge binary expression with small one
+    ApplicationManager.getApplication().runWriteAction(new Runnable() { public void run() {
+      getEditor().getDocument().replaceString(pos, pos + HUGE_EXPR.length(), "\".\"");
+      PsiDocumentManager.getInstance(getProject()).commitAllDocuments();
+    }});
+  }
+}

platform/lang-impl/src/com/intellij/lang/impl/PsiBuilderImpl.java

@@ -834,16 +834,7 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder {
 
   public ASTNode getTreeBuilt() {
     try {
-      final StartMarker rootMarker = prepareLightTree();
-      if (myOriginalTree != null) {
-        merge(myOriginalTree, rootMarker);
-        throw new BlockSupport.ReparsedSuccessfullyException();
-      }
-      else {
-        final ASTNode rootNode = createRootAST(rootMarker);
-        bind(rootMarker, (CompositeElement)rootNode);
-        return rootNode;
-      }
+      return buildTree();
     }
     finally {
       for (ProductionMarker marker : myProduction) {
@@ -857,6 +848,26 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder {
     }
   }
 
+  private ASTNode buildTree() {
+    final StartMarker rootMarker = prepareLightTree();
+    final boolean isTooDeep = myFile != null && BlockSupport.isTooDeep(myFile.getOriginalFile());
+
+    if (myOriginalTree != null && !isTooDeep) {
+      merge(myOriginalTree, rootMarker);
+      throw new BlockSupport.ReparsedSuccessfullyException();
+    }
+
+    final ASTNode rootNode = createRootAST(rootMarker);
+    bind(rootMarker, (CompositeElement)rootNode);
+
+    if (isTooDeep && !(rootNode instanceof FileElement)) {
+      final ASTNode childNode = rootNode.getFirstChildNode();
+      childNode.putUserData(BlockSupport.TREE_DEPTH_LIMIT_EXCEEDED, Boolean.TRUE);
+    }
+
+    return rootNode;
+  }
+
   public FlyweightCapableTreeStructure<LighterASTNode> getLightTree() {
     final StartMarker rootMarker = prepareLightTree();
     return new MyTreeStructure(rootMarker, myParentLightTree);
@@ -939,7 +950,7 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder {
     final Stack<StartMarker> nodes = new Stack<StartMarker>();
     nodes.push(rootMarker);
 
-    int lastErrorIndex = -1;
+    @SuppressWarnings({"MultipleVariablesInDeclaration"}) int lastErrorIndex = -1, maxDepth = 0, curDepth = 0;
     for (int i = 1; i < myProduction.size(); i++) {
       final ProductionMarker item = myProduction.get(i);
 
@@ -952,9 +963,12 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder {
         curNode.addChild(marker);
         nodes.push(curNode);
         curNode = marker;
+        curDepth++;
+        if (curDepth > maxDepth) maxDepth = curDepth;
       }
       else if (item instanceof DoneMarker) {
         curNode = nodes.pop();
+        curDepth--;
       }
       else if (item instanceof ErrorItem) {
         int curToken = item.myLexemeIndex;
@@ -978,6 +992,9 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder {
     myLexTypes[myCurrentLexeme] = null;
 
     LOG.assertTrue(curNode == rootMarker, UNBALANCED_MESSAGE);
+
+    checkTreeDepth(maxDepth, rootMarker.getTokenType() instanceof IFileElementType);
+
     return rootMarker;
   }
 
@@ -1008,6 +1025,20 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder {
     }
   }
 
+  private void checkTreeDepth(final int maxDepth, final boolean isFileRoot) {
+    if (myFile == null) return;
+    final PsiFile file = myFile.getOriginalFile();
+    final Boolean flag = file.getUserData(BlockSupport.TREE_DEPTH_LIMIT_EXCEEDED);
+    if (maxDepth > BlockSupport.INCREMENTAL_REPARSE_DEPTH_LIMIT) {
+      if (!Boolean.TRUE.equals(flag)) {
+        file.putUserData(BlockSupport.TREE_DEPTH_LIMIT_EXCEEDED, Boolean.TRUE);
+      }
+    }
+    else if (isFileRoot && flag != null) {
+      file.putUserData(BlockSupport.TREE_DEPTH_LIMIT_EXCEEDED, null);
+    }
+  }
+
   private void bind(final StartMarker rootMarker, final CompositeElement rootNode) {
     StartMarker curMarker = rootMarker;
     CompositeElement curNode = rootNode;

platform/lang-impl/src/com/intellij/psi/impl/source/text/BlockSupportImpl.java

@@ -91,8 +91,8 @@ public class BlockSupportImpl extends BlockSupport {
 
     final FileElement treeFileElement = fileImpl.getTreeElement();
 
-    if (treeFileElement.getElementType() instanceof ITemplateDataElementType) {
-      // Not able to perform incremental reparse for template data in JSP
+    if (treeFileElement.getElementType() instanceof ITemplateDataElementType || isTooDeep(file)) {
+      // unable to perform incremental reparse for template data in JSP, or in exceptionally deep trees
       makeFullParse(treeFileElement, newFileText, textLength, fileImpl);
       return;
     }
@@ -157,8 +157,9 @@ public class BlockSupportImpl extends BlockSupport {
       final Boolean data = fileImpl.getUserData(DO_NOT_REPARSE_INCREMENTALLY);
       if (data != null) fileImpl.putUserData(DO_NOT_REPARSE_INCREMENTALLY, null);
 
-      if (data != null && data.booleanValue()) { // TODO: Just to switch off incremental tree patching for certain conditions (like languages) if necessary.
-        replaceFileElementWithEvents(fileImpl, oldFileElement, newFileElement);
+      if (Boolean.TRUE.equals(data) || isTooDeep(fileImpl)) {
+        // TODO: Just to switch off incremental tree patching for certain conditions (like languages) if necessary.
+        replaceElementWithEvents(fileImpl, oldFileElement, newFileElement);
       }
       else {
         assert oldFileElement != null && newFileElement != null;
@@ -168,14 +169,17 @@ public class BlockSupportImpl extends BlockSupport {
     }
   }
 
-  private static void replaceFileElementWithEvents(final PsiFileImpl fileImpl, final FileElement fileElement, final FileElement newFileElement) {
-    fileImpl.getTreeElement().setCharTable(newFileElement.getCharTable());
-    fileElement.replaceAllChildrenToChildrenOf(newFileElement);
+  private static void replaceElementWithEvents(final PsiFileImpl file, final CompositeElement oldRoot, final CompositeElement newRoot) {
+    if (newRoot instanceof FileElement) {
+      file.getTreeElement().setCharTable(((FileElement)newRoot).getCharTable());
+    }
+    oldRoot.replaceAllChildrenToChildrenOf(newRoot);
   }
 
-  static void replaceFileElement(final PsiFileImpl fileImpl, final FileElement fileElement,
-                                         final FileElement newFileElement,
-                                         final PsiManagerEx manager) {
+  static void replaceFileElement(final PsiFileImpl fileImpl,
+                                 final FileElement fileElement,
+                                 final FileElement newFileElement,
+                                 final PsiManagerEx manager) {
     final int oldLength = fileElement.getTextLength();
     sendPsiBeforeEvent(fileImpl);
     if (fileElement.getFirstChildNode() != null) fileElement.rawRemoveAllChildren();
@@ -193,19 +197,31 @@ public class BlockSupportImpl extends BlockSupport {
         ((FileElement)newRoot).setCharTable(file.getTreeElement().getCharTable());
       }
 
-      final PomModel model = PomManager.getModel(file.getProject());
       try {
         newRoot.putUserData(TREE_TO_BE_REPARSED, oldRoot);
 
+        final ASTNode childNode;
         try {
-          newRoot.getFirstChildNode(); // Ensure parsed
+          childNode = newRoot.getFirstChildNode(); // Ensure parsed
         }
         catch (ReparsedSuccessfullyException e) {
           return; // Successfully merged in PsiBuilderImpl
         }
 
+        final boolean childTooDeep = isTooDeep(childNode);
+        if (isTooDeep(file) || childTooDeep) {
+          replaceElementWithEvents(file, (CompositeElement)oldRoot, (CompositeElement)newRoot);
+
+          if (childTooDeep) {
+            childNode.putUserData(TREE_DEPTH_LIMIT_EXCEEDED, null);
+            file.putUserData(TREE_DEPTH_LIMIT_EXCEEDED, Boolean.TRUE);
+          }
+          return;
+        }
+
         TreeUtil.ensureParsedRecursively(oldRoot);
 
+        final PomModel model = PomManager.getModel(file.getProject());
         model.runTransaction(new PomTransactionBase(file, model.getModelAspect(TreeAspect.class)) {
           public PomModelEvent runInner() {
             final ASTDiffBuilder builder = new ASTDiffBuilder(file);
@@ -228,17 +244,6 @@ public class BlockSupportImpl extends BlockSupport {
     }
   }
 
-  private static void sendPsiAfterEvent(final PsiFileImpl scope, int oldLength) {
-    if (!scope.isPhysical()) return;
-    final PsiManagerImpl manager = (PsiManagerImpl)scope.getManager();
-    PsiTreeChangeEventImpl event = new PsiTreeChangeEventImpl(manager);
-    event.setParent(scope);
-    event.setFile(scope);
-    event.setOffset(0);
-    event.setOldLength(oldLength);
-    manager.childrenChanged(event);
-  }
-
   private static void sendPsiBeforeEvent(final PsiFile scope) {
     if (!scope.isPhysical()) return;
     final PsiManagerImpl manager = (PsiManagerImpl)scope.getManager();
@@ -249,4 +254,15 @@ public class BlockSupportImpl extends BlockSupport {
     event.setOldLength(scope.getTextLength());
     manager.beforeChildrenChange(event);
   }
+
+  private static void sendPsiAfterEvent(final PsiFileImpl scope, int oldLength) {
+    if (!scope.isPhysical()) return;
+    final PsiManagerImpl manager = (PsiManagerImpl)scope.getManager();
+    PsiTreeChangeEventImpl event = new PsiTreeChangeEventImpl(manager);
+    event.setParent(scope);
+    event.setFile(scope);
+    event.setOffset(0);
+    event.setOldLength(oldLength);
+    manager.childrenChanged(event);
+  }
 }

platform/lang-impl/src/com/intellij/psi/impl/source/tree/CompositeElement.java

@@ -170,13 +170,19 @@ public class CompositeElement extends TreeElement {
   }
 
   public int getNotCachedLength() {
-    int length = 0;
-    TreeElement child = getFirstChildNode();
-    while(child != null){
-      length += child.getNotCachedLength();
-      child = child.getTreeNext();
-    }
-    return length;
+    final int[] result = new int[]{0};
+
+    acceptTree(new RecursiveTreeElementWalkingVisitor(false) {
+      @Override
+      protected void visitNode(final TreeElement element) {
+        if (element instanceof LeafElement || TreeUtil.isCollapsedChameleon(element)) {
+          result[0] += element.getNotCachedLength();
+        }
+        super.visitNode(element);
+      }
+    });
+
+    return result[0];
   }
 
   @NotNull

platform/lang-impl/src/com/intellij/psi/text/BlockSupport.java

@@ -20,6 +20,8 @@ import com.intellij.lang.ASTNode;
 import com.intellij.openapi.components.ServiceManager;
 import com.intellij.openapi.project.Project;
 import com.intellij.openapi.util.Key;
+import com.intellij.openapi.util.UserDataHolder;
+import com.intellij.openapi.util.registry.Registry;
 import com.intellij.psi.PsiFile;
 import com.intellij.util.IncorrectOperationException;
 import org.jetbrains.annotations.NonNls;
@@ -40,4 +42,14 @@ public abstract class BlockSupport {
       return this;
     }
   }
+
+  // maximal tree depth for which incremental reparse is allowed
+  // if tree is deeper then it will be replaced completely - to avoid SOEs
+  public static final int INCREMENTAL_REPARSE_DEPTH_LIMIT = Registry.intValue("psi.incremental.reparse.depth.limit");
+
+  public static final Key<Boolean> TREE_DEPTH_LIMIT_EXCEEDED = Key.create("TREE_IS_TOO_DEEP");
+
+  public static boolean isTooDeep(final UserDataHolder element) {
+    return element != null && Boolean.TRUE.equals(element.getUserData(TREE_DEPTH_LIMIT_EXCEEDED));
+  }
 }

platform/platform-resources-en/src/misc/registry.properties

@@ -96,6 +96,7 @@ compiler.perform.outputs.refresh.on.start.restartRequired=false
 vcs.show.colored.annotations=true
 vcs.showConsole=true
 
+psi.incremental.reparse.depth.limit=1000
 psi.viewer.selection.color=0,153,153
 psi.deferIconLoading=true