From 11b1a5ea4a4f7395d86b3b0e47a5a3bcee8060ae Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Wed, 17 Nov 2010 14:03:14 +0300 Subject: [PATCH] IDEA-56517 (no more recursion in Java stub builder) --- .../lang/java/parser/JavaParserUtil.java | 13 +++- .../java/stubs/JavaFieldStubElementType.java | 10 +++ .../java/stubs/impl/PsiFieldStubImpl.java | 10 +-- .../com/intellij/psi/JavaStubBuilderTest.java | 52 ++++++++++++-- .../testFramework/LightIdeaTestCase.java | 10 ++- .../intellij/psi/stubs/LightStubBuilder.java | 72 +++++++++++++++---- .../intellij/psi/stubs/StubUpdatingIndex.java | 40 +++++++---- 7 files changed, 157 insertions(+), 50 deletions(-) diff --git a/java/java-impl/src/com/intellij/lang/java/parser/JavaParserUtil.java b/java/java-impl/src/com/intellij/lang/java/parser/JavaParserUtil.java index 2d2cdd7be203..2a09e3af1d7b 100644 --- a/java/java-impl/src/com/intellij/lang/java/parser/JavaParserUtil.java +++ b/java/java-impl/src/com/intellij/lang/java/parser/JavaParserUtil.java @@ -28,6 +28,8 @@ import com.intellij.psi.PsiElement; import com.intellij.psi.impl.source.tree.ElementType; import com.intellij.psi.impl.source.tree.JavaDocElementType; import com.intellij.psi.impl.source.tree.JavaElementType; +import com.intellij.psi.impl.source.tree.TreeUtil; +import com.intellij.psi.stubs.StubUpdatingIndex; import com.intellij.psi.tree.IElementType; import com.intellij.psi.tree.TokenSet; import com.intellij.psi.util.PsiUtil; @@ -149,10 +151,19 @@ public class JavaParserUtil { assert psi != null : chameleon; final Project project = psi.getProject(); + CharSequence text; + if (TreeUtil.isCollapsedChameleon(chameleon)) { + text = chameleon.getChars(); + } + else { + text = psi.getUserData(StubUpdatingIndex.FILE_TEXT_CONTENT_KEY); + if (text == null) text = chameleon.getChars(); + } + final PsiBuilderFactory factory = PsiBuilderFactory.getInstance(); final LanguageLevel level = PsiUtil.getLanguageLevel(psi); final Lexer lexer = JavaParserDefinition.createLexer(level); - final PsiBuilder builder = factory.createBuilder(project, chameleon, lexer, psi.getLanguage(), chameleon.getChars()); + final PsiBuilder builder = factory.createBuilder(project, chameleon, lexer, psi.getLanguage(), text); setLanguageLevel(builder, level); return builder; diff --git a/java/java-impl/src/com/intellij/psi/impl/java/stubs/JavaFieldStubElementType.java b/java/java-impl/src/com/intellij/psi/impl/java/stubs/JavaFieldStubElementType.java index 1326e0edc899..047c5db51e96 100644 --- a/java/java-impl/src/com/intellij/psi/impl/java/stubs/JavaFieldStubElementType.java +++ b/java/java-impl/src/com/intellij/psi/impl/java/stubs/JavaFieldStubElementType.java @@ -49,6 +49,8 @@ import java.io.IOException; * @author max */ public class JavaFieldStubElementType extends JavaStubElementType { + private static final int INITIALIZER_LENGTH_LIMIT = 1000; + public JavaFieldStubElementType(@NotNull @NonNls final String id) { super(id); } @@ -125,6 +127,10 @@ public class JavaFieldStubElementType extends JavaStubElementType INITIALIZER_LENGTH_LIMIT) { + return PsiFieldStub.INITIALIZER_TOO_LONG; + } + return initializer.getText(); } @@ -134,6 +140,10 @@ public class JavaFieldStubElementType extends JavaStubElementType INITIALIZER_LENGTH_LIMIT) { + return PsiFieldStub.INITIALIZER_TOO_LONG; + } + return LightTreeUtil.toFilteredString(tree, initializer, null); } diff --git a/java/java-impl/src/com/intellij/psi/impl/java/stubs/impl/PsiFieldStubImpl.java b/java/java-impl/src/com/intellij/psi/impl/java/stubs/impl/PsiFieldStubImpl.java index f6a490fd75bc..1ce0cc4a44a5 100644 --- a/java/java-impl/src/com/intellij/psi/impl/java/stubs/impl/PsiFieldStubImpl.java +++ b/java/java-impl/src/com/intellij/psi/impl/java/stubs/impl/PsiFieldStubImpl.java @@ -33,8 +33,6 @@ import com.intellij.util.io.StringRef; import org.jetbrains.annotations.NotNull; public class PsiFieldStubImpl extends StubBase implements PsiFieldStub { - private static final int INITIALIZER_LENGTH_LIMIT = 1000; - private final StringRef myName; private final TypeInfo myType; private final StringRef myInitializer; @@ -51,15 +49,9 @@ public class PsiFieldStubImpl extends StubBase implements PsiFieldStub public PsiFieldStubImpl(final StubElement parent, final StringRef name, @NotNull TypeInfo type, final StringRef initializer, final byte flags) { super(parent, isEnumConst(flags) ? JavaStubElementTypes.ENUM_CONSTANT : JavaStubElementTypes.FIELD); - if (initializer != null && initializer.length() > INITIALIZER_LENGTH_LIMIT) { - myInitializer = StringRef.fromString(INITIALIZER_TOO_LONG); - } - else { - myInitializer = initializer; - } - myName = name; myType = type; + myInitializer = initializer; myFlags = flags; } diff --git a/java/java-tests/testSrc/com/intellij/psi/JavaStubBuilderTest.java b/java/java-tests/testSrc/com/intellij/psi/JavaStubBuilderTest.java index e98a0b629cb3..6cd999ee57c7 100644 --- a/java/java-tests/testSrc/com/intellij/psi/JavaStubBuilderTest.java +++ b/java/java-tests/testSrc/com/intellij/psi/JavaStubBuilderTest.java @@ -22,10 +22,13 @@ import com.intellij.psi.impl.source.tree.LazyParseableElement; import com.intellij.psi.stubs.StubElement; import com.intellij.testFramework.LightIdeaTestCase; +import java.security.SecureRandom; + public class JavaStubBuilderTest extends LightIdeaTestCase { private static final StubBuilder OLD_BUILDER = new JavaFileStubBuilder(); private static final StubBuilder NEW_BUILDER = new JavaLightStubBuilder(); + private static final int SOE_TEST_DEPTH = 10000; @Override public void setUp() throws Exception { @@ -33,6 +36,12 @@ public class JavaStubBuilderTest extends LightIdeaTestCase { doTest("@interface A { int i() default 42; }\n class C { void m(int p) throws E { } }", null); // warm up } + public void testEmpty() { + doTest("/**/", + "PsiJavaFileStub []\n" + + " IMPORT_LIST:PsiImportListStub\n"); + } + public void testFileHeader() { doTest("package p;\n" + "import a/*comment to skip*/.b;\n" + @@ -256,6 +265,32 @@ public class JavaStubBuilderTest extends LightIdeaTestCase { " THROWS_LIST:PsiRefListStub[THROWS_LIST:]\n"); } + public void testSOEProof() throws Exception { + final StringBuilder sb = new StringBuilder(); + final SecureRandom random = new SecureRandom(); + sb.append("class SOE_test {\n BigInteger BIG = new BigInteger(\n"); + for (int i = 0; i < SOE_TEST_DEPTH; i++) { + sb.append(" \"").append(Math.abs(random.nextInt())).append("\" +\n"); + } + sb.append(" \"\");\n}"); + + final PsiJavaFile file = (PsiJavaFile)createLightFile("SOE_test.java", sb.toString()); + long t = System.currentTimeMillis(); + final StubElement tree = NEW_BUILDER.buildStubTree(file); + t = System.currentTimeMillis() - t; + assertEquals("PsiJavaFileStub []\n" + + " IMPORT_LIST:PsiImportListStub\n" + + " CLASS:PsiClassStub[name=SOE_test fqn=SOE_test]\n" + + " MODIFIER_LIST:PsiModifierListStub[mask=4096]\n" + + " TYPE_PARAMETER_LIST:PsiTypeParameterListStub\n" + + " EXTENDS_LIST:PsiRefListStub[EXTENDS_LIST:]\n" + + " IMPLEMENTS_LIST:PsiRefListStub[IMPLEMENTS_LIST:]\n" + + " FIELD:PsiFieldStub[BIG:BigInteger=;INITIALIZER_NOT_STORED;]\n" + + " MODIFIER_LIST:PsiModifierListStub[mask=4096]\n", + DebugUtil.stubTreeToString(tree)); + System.out.println("SOE depth=" + SOE_TEST_DEPTH + ", time=" + t + "ms"); + } + private static void doTest(final String source, final String tree) { final PsiJavaFile file = (PsiJavaFile)createLightFile("test.java", source); final LazyParseableElement fileNode = (LazyParseableElement)file.getNode(); @@ -264,25 +299,28 @@ public class JavaStubBuilderTest extends LightIdeaTestCase { long t1 = System.nanoTime(); final StubElement lighterTree = NEW_BUILDER.buildStubTree(file); - t1 = System.nanoTime() - t1; + t1 = (System.nanoTime() - t1)/1000; assertFalse(fileNode.isParsed()); long t2 = System.nanoTime(); final StubElement originalTree = OLD_BUILDER.buildStubTree(file); - t2 = System.nanoTime() - t2; + t2 = (System.nanoTime() - t2)/1000; assertTrue(fileNode.isParsed()); - final String lightStr = DebugUtil.stubTreeToString(originalTree); - final String originalStr = DebugUtil.stubTreeToString(lighterTree); + final String lightStr = DebugUtil.stubTreeToString(lighterTree); + final String originalStr = DebugUtil.stubTreeToString(originalTree); if (tree != null) { - final String msg = "light=" + t1/1000 + "mks, heavy=" + t2/1000 + "mks"; + final String msg = "light=" + t1 + "mks, heavy=" + t2 + "mks, gain=" + (100 * (t2 - t1) / t2) + "%"; // todo: assertTrue("Expected to be at least 2x faster: " + msg, 2*t1 <= t2); System.out.println(msg); - assertEquals("wrong test data", tree, lightStr); + assertEquals("wrong test data", tree, originalStr); if (!"".equals(tree)) { - assertEquals("light tree differs", tree, originalStr); + assertEquals("light tree differs", tree, lightStr); } } + else { + assertEquals(originalStr, lightStr); + } } } \ No newline at end of file diff --git a/java/testFramework/src/com/intellij/testFramework/LightIdeaTestCase.java b/java/testFramework/src/com/intellij/testFramework/LightIdeaTestCase.java index a39cc4eb9cdc..0ba4dd72ad88 100644 --- a/java/testFramework/src/com/intellij/testFramework/LightIdeaTestCase.java +++ b/java/testFramework/src/com/intellij/testFramework/LightIdeaTestCase.java @@ -20,19 +20,18 @@ import com.intellij.openapi.module.StdModuleTypes; import com.intellij.openapi.projectRoots.Sdk; import com.intellij.openapi.projectRoots.impl.JavaSdkImpl; import com.intellij.psi.impl.JavaPsiFacadeEx; -import org.jetbrains.annotations.NonNls; /** - * A testcase that provides IDEA application and project. Note both are reused for each test run in the session so + * A test case that provides IDEA application and project. Note both are reused for each test run in the session so * be careful to return all the modification made to application and project components (such as settings) after * test is finished so other test aren't affected. The project is initialized with single module that have single - * content&source entry. For your convenience the project may be equipped with some mock JDK so your tests may + * content and source entry. For your convenience the project may be equipped with some mock JDK so your tests may * refer to external classes. In order to enable this feature you have to have a folder named "mockJDK" under * idea installation home that is used for test running. Place src.zip under that folder. We'd suggest this is real mock * so it contains classes that is really needed in order to speed up tests startup. */ -@SuppressWarnings({"HardCodedStringLiteral"}) -@NonNls public class LightIdeaTestCase extends LightPlatformTestCase { +public abstract class LightIdeaTestCase extends LightPlatformTestCase { + @SuppressWarnings({"JUnitTestCaseWithNonTrivialConstructors"}) public LightIdeaTestCase() { IdeaTestCase.initPlatformPrefix(); } @@ -41,7 +40,6 @@ import org.jetbrains.annotations.NonNls; return JavaPsiFacadeEx.getInstanceEx(ourProject); } - @Override protected Sdk getProjectJDK() { return JavaSdkImpl.getMockJdk17(); diff --git a/platform/lang-api/src/com/intellij/psi/stubs/LightStubBuilder.java b/platform/lang-api/src/com/intellij/psi/stubs/LightStubBuilder.java index bb965d5f680b..989ddf531e16 100644 --- a/platform/lang-api/src/com/intellij/psi/stubs/LightStubBuilder.java +++ b/platform/lang-api/src/com/intellij/psi/stubs/LightStubBuilder.java @@ -23,11 +23,16 @@ import com.intellij.psi.PsiFile; import com.intellij.psi.StubBuilder; import com.intellij.psi.tree.IElementType; import com.intellij.psi.tree.ILightStubFileElementType; +import gnu.trove.TIntStack; + +import java.util.List; +import java.util.Stack; public class LightStubBuilder implements StubBuilder { private static final Logger LOG = Logger.getInstance("#com.intellij.psi.stubs.LightStubBuilder"); + @Override public StubElement buildStubTree(final PsiFile file) { final FileASTNode node = file.getNode(); assert node != null; @@ -39,7 +44,9 @@ public class LightStubBuilder implements StubBuilder { final ILightStubFileElementType type = (ILightStubFileElementType)node.getElementType(); final LighterAST tree = new LighterAST(node.getCharTable(), type.parseContentsLight(node)); - return buildStubTreeFor(tree, tree.getRoot(), createStubForFile(file, tree)); + final StubElement rootStub = createStubForFile(file, tree); + buildStubTree(tree, tree.getRoot(), rootStub); + return rootStub; } protected StubElement createStubForFile(final PsiFile file, final LighterAST tree) { @@ -47,15 +54,61 @@ public class LightStubBuilder implements StubBuilder { return new PsiFileStubImpl(file); } - protected StubElement buildStubTreeFor(final LighterAST tree, final LighterASTNode element, final StubElement parentStub) { - StubElement stub = parentStub; + protected void buildStubTree(final LighterAST tree, final LighterASTNode root, final StubElement rootStub) { + final Stack parents = new Stack(); + final TIntStack childNumbers = new TIntStack(); + final Stack parentStubs = new Stack(); + LighterASTNode parent = null; + LighterASTNode element = root; + List children = null; + int childNumber = 0; + StubElement parentStub = rootStub; + + nextElement: + while (element != null) { + final StubElement stub = createStub(tree, element, parentStub); + + final List kids = tree.getChildren(element); + if (kids.size() > 0) { + if (parent != null) { + parents.push(parent); + childNumbers.push(childNumber); + parentStubs.push(parentStub); + } + parent = element; + element = (children = kids).get(childNumber = 0); + parentStub = stub; + if (!skipChildProcessingWhenBuildingStubs(parent.getTokenType(), element.getTokenType())) continue nextElement; + } + + while (children != null && ++childNumber < children.size()) { + element = children.get(childNumber); + if (!skipChildProcessingWhenBuildingStubs(parent.getTokenType(), element.getTokenType())) continue nextElement; + } + + element = null; + while (parents.size() > 0) { + children = tree.getChildren((parent = parents.pop())); + childNumber = childNumbers.pop(); + parentStub = parentStubs.pop(); + while (++childNumber < children.size()) { + element = children.get(childNumber); + if (!skipChildProcessingWhenBuildingStubs(parent.getTokenType(), element.getTokenType())) continue nextElement; + } + element = null; + } + } + } + + @SuppressWarnings({"MethodMayBeStatic"}) + protected StubElement createStub(final LighterAST tree, final LighterASTNode element, final StubElement parentStub) { final IElementType elementType = element.getTokenType(); if (elementType instanceof IStubElementType) { if (elementType instanceof ILightStubElementType) { final ILightStubElementType lightElementType = (ILightStubElementType)elementType; if (lightElementType.shouldCreateStub(tree, element, parentStub)) { - stub = lightElementType.createStub(tree, element, parentStub); + return lightElementType.createStub(tree, element, parentStub); } } else { @@ -63,16 +116,11 @@ public class LightStubBuilder implements StubBuilder { } } - for (final LighterASTNode child : tree.getChildren(element)) { - if (!skipChildProcessingWhenBuildingStubs(elementType, child.getTokenType())) { - buildStubTreeFor(tree, child, stub); - } - } - - return stub; + return parentStub; } - public boolean skipChildProcessingWhenBuildingStubs(IElementType nodeType, IElementType childType) { + @Override + public boolean skipChildProcessingWhenBuildingStubs(final IElementType nodeType, final IElementType childType) { return false; } } \ No newline at end of file diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java index 4c3bf7c119a7..5f55f616c8dd 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java @@ -13,10 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -/* - * @author max - */ package com.intellij.psi.stubs; import com.intellij.lang.Language; @@ -47,11 +43,18 @@ import java.io.IOException; import java.util.*; import java.util.concurrent.Callable; +/* + * @author max + */ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtension { private static final Logger LOG = Logger.getInstance("#com.intellij.psi.stubs.StubUpdatingIndex"); + public static final Key FILE_TEXT_CONTENT_KEY = Key.create("file text content cached by stub indexer"); + public static final ID INDEX_ID = ID.create("Stubs"); + private static final int VERSION = 20; + private static final DataExternalizer KEY_EXTERNALIZER = new DataExternalizer() { public void save(final DataOutput out, final SerializedStubTree v) throws IOException { byte[] value = v.getBytes(); @@ -127,13 +130,14 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi }; } - private static Key stubElementKey = Key.create("stub.tree.for.file.content"); + private static final Key stubElementKey = Key.create("stub.tree.for.file.content"); @Nullable public static StubElement buildStubTree(final FileContent inputData) { StubElement data = inputData.getUserData(stubElementKey); if (data != null) return data; + //noinspection SynchronizationOnLocalVariableOrMethodParameter synchronized (inputData) { data = inputData.getUserData(stubElementKey); if (data != null) return data; @@ -152,15 +156,22 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi final IFileElementType type = LanguageParserDefinitions.INSTANCE.forLanguage(l).getFileNodeType(); PsiFile psi = inputData.getPsiFile(); + CharSequence contentAsText = inputData.getContentAsText(); + psi.putUserData(FILE_TEXT_CONTENT_KEY, contentAsText); - if (type instanceof IStubFileElementType) { - data = ((IStubFileElementType)type).getBuilder().buildStubTree(psi); + try { + if (type instanceof IStubFileElementType) { + data = ((IStubFileElementType)type).getBuilder().buildStubTree(psi); + } + else if (filetype instanceof SubstitutedFileType) { + SubstitutedFileType substituted = (SubstitutedFileType) filetype; + LanguageFileType original = (LanguageFileType)substituted.getOriginalFileType(); + final IFileElementType originalType = LanguageParserDefinitions.INSTANCE.forLanguage(original.getLanguage()).getFileNodeType(); + data = ((IStubFileElementType)originalType).getBuilder().buildStubTree(psi); + } } - else if (filetype instanceof SubstitutedFileType) { - SubstitutedFileType substituted = (SubstitutedFileType) filetype; - LanguageFileType original = (LanguageFileType)substituted.getOriginalFileType(); - final IFileElementType originalType = LanguageParserDefinitions.INSTANCE.forLanguage(original.getLanguage()).getFileNodeType(); - data = ((IStubFileElementType)originalType).getBuilder().buildStubTree(psi); + finally { + psi.putUserData(FILE_TEXT_CONTENT_KEY, null); } } @@ -227,7 +238,7 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi } }); } - return new MyIndex(indexId, owner, storage, getIndexer()); + return new MyIndex(indexId, storage, getIndexer()); } private static void updateStubIndices(final Collection indexKeys, @@ -258,7 +269,6 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi private final StubIndexImpl myStubIndex; public MyIndex(final ID indexId, - final FileBasedIndex owner, final IndexStorage storage, final DataIndexer indexer) { super(indexId, indexer, storage); @@ -268,7 +278,7 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi } catch (StorageException e) { LOG.info(e); - owner.requestRebuild(INDEX_ID); + FileBasedIndex.requestRebuild(INDEX_ID); } }