From c4cfcee2514ee3c47317eb19cda3db4c3e738af1 Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 10 Apr 2018 16:54:29 +0200 Subject: [PATCH] simplify stub-AST-PSI relationships replace AstPath with index-based SpineRef, take coarser locks, load multi-root stubs independently What we lose: * an ability to gc AST of stubbed PSI loaded via AST-only * an ability to gc AST of non-stubbed StubBasedPsiElementBase * partial chameleon expansion during findElementAt after AST has been gc-ed but stubbed PSI is still referenced ** probably not a major regression: full parsing has been happening during stub-AST switch for ages What we retain: * we can hold references to PSI received from stub index queries without worrying about retaining their AST (even if loaded, it will be gc-ed) What we gain: * simpler code with less mutable state * outdated AstPath assertion is now nonexistent (WEB-32120, EA-101911) * no more cases when stub has been gc-ed but can't be reloaded due to some cached PSI with unknown stub index (IDEA-189948) * other stub-related exceptions are also likely to be fixed: ** EA-97101 - REWA: StubBasedPsiElementBase.failedToBindStubToAst ** EA-102022 - CCE: PsiFieldImpl.findFirstFieldInDeclaration ** EA-101452 - assert: StubBasedPsiElementBase.getRequiredStubOrPsiChild ** EA-107208 - ISE: StubBasedPsiElementBase.$$$reportNull$$$ ** EA-110161 - PIEAE: SubstrateRef$StubRef.reportError ** EA-113453 - CCE: PsiClassImpl.getContainingClass ** EA-117888 - CCE: PsiClassImpl.getContextStub --- .../psi/impl/source/PsiJavaFileBaseImpl.java | 6 +- .../java/psi/StubAstSwitchTest.groovy | 63 ++-- .../src/com/intellij/psi/stubs/StubBase.java | 3 +- .../extapi/psi/StubBasedPsiElementBase.java | 28 +- .../psi/impl/source/AstPathPsiMap.java | 153 --------- .../intellij/psi/impl/source/FileTrees.java | 178 ++++++++-- .../intellij/psi/impl/source/PsiFileImpl.java | 278 +++------------- .../intellij/psi/impl/source/SpineRef.java | 53 +++ .../psi/impl/source/SubstrateRef.java | 8 +- .../psi/impl/source/tree/AstPath.java | 311 ------------------ .../psi/impl/source/tree/AstSpine.java | 3 +- .../impl/source/tree/CompositeElement.java | 30 +- .../psi/impl/source/tree/FileElement.java | 18 +- .../source/tree/LazyParseableElement.java | 6 +- .../psi/impl/source/tree/TreeUtil.java | 50 +-- .../intellij/psi/stubs/StubTreeLoader.java | 4 - .../psi/stubs/StubTreeLoaderImpl.java | 5 - 17 files changed, 315 insertions(+), 882 deletions(-) delete mode 100644 platform/core-impl/src/com/intellij/psi/impl/source/AstPathPsiMap.java create mode 100644 platform/core-impl/src/com/intellij/psi/impl/source/SpineRef.java delete mode 100644 platform/core-impl/src/com/intellij/psi/impl/source/tree/AstPath.java diff --git a/java/java-psi-impl/src/com/intellij/psi/impl/source/PsiJavaFileBaseImpl.java b/java/java-psi-impl/src/com/intellij/psi/impl/source/PsiJavaFileBaseImpl.java index 0ccc3be41edb..5e97247a95e0 100644 --- a/java/java-psi-impl/src/com/intellij/psi/impl/source/PsiJavaFileBaseImpl.java +++ b/java/java-psi-impl/src/com/intellij/psi/impl/source/PsiJavaFileBaseImpl.java @@ -40,7 +40,6 @@ import com.intellij.psi.tree.IElementType; import com.intellij.psi.util.*; import com.intellij.util.IncorrectOperationException; import com.intellij.util.Processor; -import java.util.HashSet; import com.intellij.util.containers.MostlySingularMultiMap; import com.intellij.util.indexing.IndexingDataKeys; import org.jetbrains.annotations.NotNull; @@ -48,6 +47,7 @@ import org.jetbrains.annotations.Nullable; import java.util.ArrayList; import java.util.Collection; +import java.util.HashSet; import java.util.List; public abstract class PsiJavaFileBaseImpl extends PsiFileImpl implements PsiJavaFile { @@ -137,8 +137,8 @@ public abstract class PsiJavaFileBaseImpl extends PsiFileImpl implements PsiJava if (stub != null) { PsiImportList[] nodes = stub.getChildrenByType(JavaStubElementTypes.IMPORT_LIST, PsiImportList.ARRAY_FACTORY); if (nodes.length == 1) return nodes[0]; - if (nodes.length == 0) return null; - reportStubAstMismatch(stub + "; " + stub.getChildrenStubs(), getStubTree()); + assert nodes.length == 0; + return null; } ASTNode node = calcTreeElement().findChildByType(JavaElementType.IMPORT_LIST); diff --git a/java/java-tests/testSrc/com/intellij/java/psi/StubAstSwitchTest.groovy b/java/java-tests/testSrc/com/intellij/java/psi/StubAstSwitchTest.groovy index 0c7d2fdc5519..af7a14b8c969 100644 --- a/java/java-tests/testSrc/com/intellij/java/psi/StubAstSwitchTest.groovy +++ b/java/java-tests/testSrc/com/intellij/java/psi/StubAstSwitchTest.groovy @@ -34,15 +34,13 @@ import com.intellij.psi.stubs.StubTree import com.intellij.psi.util.PsiTreeUtil import com.intellij.reference.SoftReference import com.intellij.testFramework.LeakHunter -import com.intellij.testFramework.PlatformTestUtil import com.intellij.testFramework.SkipSlowTestLocally import com.intellij.testFramework.fixtures.LightCodeInsightFixtureTestCase import com.intellij.util.ref.GCUtil -import groovy.transform.CompileStatic import java.util.concurrent.Callable import java.util.concurrent.CountDownLatch -import java.util.concurrent.Future +import java.util.concurrent.Future /** * @author peter */ @@ -236,7 +234,7 @@ class B { LeakHunter.checkLeak(file, StubTree) { candidate -> candidate.root.psi == file } } - void "test node is not deeply parsed when loaded in green stub presence"() { + void "test node has same PSI when loaded in green stub presence"() { PsiFileImpl file = (PsiFileImpl)myFixture.addFileToProject("a.java", "class A{}") def stubTree = file.stubTree PsiClass psiClass = ((PsiJavaFile)file).classes[0] @@ -244,7 +242,7 @@ class B { GCUtil.tryGcSoftlyReachableObjects() assert stubTree.is(file.greenStubTree) - assert !file.node.parsed + assert file.node.lastChildNode.psi.is(psiClass) } void "test load stub from non-file PSI after AST is unloaded"() { @@ -258,26 +256,22 @@ class B { assert ((PsiClassImpl) cls).stub } - void "test load PSI via stub when AST is gc-ed but PSI exists that has never known stub"() { + void "test load PSI via stub when AST is gc-ed but PSI exists that was loaded via AST but knows its stub index"() { PsiJavaFileImpl file = (PsiJavaFileImpl)myFixture.addFileToProject("a.java", "class A{}") def cls = file.lastChild assert cls instanceof PsiClass GCUtil.tryGcSoftlyReachableObjects() - assert !file.treeElement - - assert cls == myFixture.findClass('A') - } - - void "test load PSI via stub when AST is gc-ed and PSI remains that never knew stub"() { - PsiJavaFileImpl file = (PsiJavaFileImpl)myFixture.addFileToProject("a.java", "class A{}") - def cls = file.lastChild - assert cls instanceof PsiClass + assert file.treeElement // we still hold a strong reference to AST + assert cls == myFixture.findClass('A') + + // now we know stub index and can GC AST GCUtil.tryGcSoftlyReachableObjects() assert !file.treeElement assert cls == myFixture.findClass('A') + assert !file.treeElement } void "test bind stubs to AST after AST has been loaded and gc-ed"() { @@ -335,30 +329,27 @@ class B { assert stubTree.plainList.find { it.stubType == JavaStubElementTypes.ANONYMOUS_CLASS } } - @CompileStatic - void "test getStub performance with cached PSI"() { - def text = "class Foo { " + "void bar(int a, int b, int c, int d, int e) { int x = null; }\n" * 1000 + "}" - def file = myFixture.addFileToProject "a.java", text - - PsiMethod[] methods = ((PsiJavaFile) file).classes[0].methods - def params = methods.collect { PsiMethod method -> method.parameterList.parameters } - def literal = file.findElementAt(text.indexOf('null')).parent as PsiLiteralExpression // the only cached PSI without stubIndex + void "test stub index is cleared on AST change"() { + def clazz = myFixture.addClass("class Foo { int a; }") + def field = clazz.fields[0] + def file = clazz.containingFile as PsiFileImpl + WriteCommandAction.runWriteCommandAction(project, { + file.viewProvider.document.insertString(0, ' ') + PsiDocumentManager.getInstance(project).commitAllDocuments() + }) + + assert file.calcStubTree() + WriteCommandAction.runWriteCommandAction(project, { + file.viewProvider.document.insertString(file.text.indexOf('int'), 'void foo();') + PsiDocumentManager.getInstance(project).commitAllDocuments() + }) + GCUtil.tryGcSoftlyReachableObjects() - def fileImpl = (PsiFileImpl)file - assert !fileImpl.treeElement - assert !fileImpl.stub + assert file.calcStubTree() - PlatformTestUtil.startPerformanceTest('getStub performance', 100, { - 10_000.times { - if (fileImpl.stub != null) { - throw new IllegalStateException("has stub") - } - } - }).assertTiming() - - assert params - assert literal + assert field.valid + assert field.name == 'a' } } diff --git a/platform/core-api/src/com/intellij/psi/stubs/StubBase.java b/platform/core-api/src/com/intellij/psi/stubs/StubBase.java index 14453b447f15..2f2f7a7fc586 100644 --- a/platform/core-api/src/com/intellij/psi/stubs/StubBase.java +++ b/platform/core-api/src/com/intellij/psi/stubs/StubBase.java @@ -51,7 +51,8 @@ public abstract class StubBase extends ObjectStubBase extends ASTDelegateP public static final Key CREATION_TRACE = Key.create("CREATION_TRACE"); public static final boolean ourTraceStubAstBinding = "true".equals(System.getProperty("trace.stub.ast.binding", "false")); private volatile SubstrateRef mySubstrateRef; - private volatile int myStubIndex = -1; private final IElementType myElementType; public StubBasedPsiElementBase(@NotNull T stub, @NotNull IStubElementType nodeType) { @@ -238,29 +237,6 @@ public class StubBasedPsiElementBase extends ASTDelegateP */ public final void setSubstrateRef(@NotNull SubstrateRef substrateRef) { mySubstrateRef = substrateRef; - myStubIndex = -1; - } - - /** - * Don't invoke this method, it's public for implementation reasons. - */ - public final void setStubIndex(int stubIndex) { - myStubIndex = stubIndex; - } - - /** - * Don't invoke this method, it's public for implementation reasons. - */ - public int getStubIndex() { - return myStubIndex; - } - - /** - * Don't invoke this method, it's public for implementation reasons. - */ - @NotNull - public final SubstrateRef getSubstrateRef() { - return mySubstrateRef; } @NotNull @@ -383,7 +359,7 @@ public class StubBasedPsiElementBase extends ASTDelegateP public T getStub() { ProgressIndicatorProvider.checkCanceled(); // Hope, this is called often //noinspection unchecked - return (T)mySubstrateRef.getStub(myStubIndex); + return (T)mySubstrateRef.getStub(); } /** @@ -395,7 +371,7 @@ public class StubBasedPsiElementBase extends ASTDelegateP public final T getGreenStub() { ProgressIndicatorProvider.checkCanceled(); // Hope, this is called often //noinspection unchecked - return (T)mySubstrateRef.getGreenStub(myStubIndex); + return (T)mySubstrateRef.getGreenStub(); } /** diff --git a/platform/core-impl/src/com/intellij/psi/impl/source/AstPathPsiMap.java b/platform/core-impl/src/com/intellij/psi/impl/source/AstPathPsiMap.java deleted file mode 100644 index 87e300b5a03b..000000000000 --- a/platform/core-impl/src/com/intellij/psi/impl/source/AstPathPsiMap.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright 2000-2016 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.intellij.psi.impl.source; - -import com.intellij.extapi.psi.StubBasedPsiElementBase; -import com.intellij.openapi.project.Project; -import com.intellij.openapi.util.Key; -import com.intellij.openapi.util.LowMemoryWatcher; -import com.intellij.openapi.util.UserDataHolderEx; -import com.intellij.psi.impl.DebugUtil; -import com.intellij.psi.impl.source.tree.AstPath; -import com.intellij.psi.impl.source.tree.CompositeElement; -import com.intellij.reference.SoftReference; -import com.intellij.util.containers.ContainerUtil; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -import java.lang.ref.Reference; -import java.lang.ref.ReferenceQueue; -import java.lang.ref.WeakReference; -import java.util.Objects; -import java.util.concurrent.ConcurrentMap; -import java.util.stream.Stream; - -/** - * A weak cache for all instantiated stub-based PSI to allow {@link CompositeElement#getPsi()} return it when AST is reloaded.

- * - * All methods should be called under an external lock (view provider's PsiLock), except for - * ({@link #getCachedPsi(AstPath)} which can be called without lock. - * - * @author peter - */ -class AstPathPsiMap { - /** - * Not using ConcurrentWeakValueMap because we need to clean each of them up separately, when ASTs in thousands of files are created and gc-ed. - * So we have a per-project single shared reference queue {@link #myQueue} for that. - * Otherwise the files end up retaining lots of maps with all-gc-ed stuff inside, but the maps are still very large. - */ - private final ConcurrentMap myMap = ContainerUtil.newConcurrentMap(); - private volatile Boolean myHasUnbindableCachedPsi = null; - - private static final Key STUB_PSI_REFS = Key.create("STUB_PSI_REFS"); - private final MyReferenceQueue myQueue; - - AstPathPsiMap(@NotNull Project project) { - MyReferenceQueue queue = project.getUserData(STUB_PSI_REFS); - myQueue = queue != null ? queue : ((UserDataHolderEx)project).putUserDataIfAbsent(STUB_PSI_REFS, new MyReferenceQueue(project)); - } - - void invalidatePsi() { - myQueue.cleanupStaleReferences(); - getAllCachedPsi().forEach(psi -> { - DebugUtil.onInvalidated(psi); - psi.setSubstrateRef(SubstrateRef.createInvalidRef(psi)); - }); - myMap.clear(); - myHasUnbindableCachedPsi = false; - } - - void switchToStrongRefs() { - myQueue.cleanupStaleReferences(); - getAllCachedPsi().forEach(psi -> { - CompositeElement node = (CompositeElement)psi.getNode(); - node.setPsi(psi); - psi.setSubstrateRef(SubstrateRef.createAstStrongRef(node)); - }); - myMap.clear(); - myHasUnbindableCachedPsi = false; - } - - @Nullable - StubBasedPsiElementBase getCachedPsi(@NotNull AstPath ref) { - return SoftReference.dereference(myMap.get(ref)); - } - - @NotNull - StubBasedPsiElementBase cachePsi(@NotNull AstPath key, @NotNull StubBasedPsiElementBase psi) { - myQueue.cleanupStaleReferences(); - // ensure PSI will use AST path before making it available to other threads - // otherwise another thread could invoke StubRef.getNode and fail since file's AST isn't set yet - psi.setSubstrateRef(key); - myMap.put(key, new MyReference(psi, key, myQueue)); - clearStubIndexCache(); - return psi; - } - - Stream> getAllCachedPsi() { - myQueue.cleanupStaleReferences(); - if (myMap.isEmpty()) return Stream.empty(); - - return myMap.values().stream().map(Reference::get).filter(Objects::nonNull); - } - - boolean hasUnbindableCachedPsi() { - Boolean answer = myHasUnbindableCachedPsi; - if (answer == null) { - myHasUnbindableCachedPsi = answer = calcHasUnbindableCachedPsi(); - } - return answer; - } - - private boolean calcHasUnbindableCachedPsi() { - myQueue.cleanupStaleReferences(); - if (myMap.isEmpty()) return false; - - return getAllCachedPsi().anyMatch(e -> e.getStubIndex() < 0); - } - - void clearStubIndexCache() { - myHasUnbindableCachedPsi = null; - } - - private static class MyReference extends WeakReference> { - final AstPath pathRef; - - MyReference(StubBasedPsiElementBase psi, AstPath ref, ReferenceQueue> queue) { - super(psi, queue); - pathRef = ref; - } - } - - private static class MyReferenceQueue extends ReferenceQueue> { - MyReferenceQueue(Project project) { - LowMemoryWatcher.register(() -> cleanupStaleReferences(), project); - } - - void cleanupStaleReferences() { - while (true) { - MyReference reference = (MyReference)poll(); - if (reference == null) break; - - AstPath key = reference.pathRef; - AstPathPsiMap refToPsi = key.getContainingFile().getRefToPsi(); - refToPsi.myMap.remove(key, reference); - refToPsi.clearStubIndexCache(); - } - } - - } -} diff --git a/platform/core-impl/src/com/intellij/psi/impl/source/FileTrees.java b/platform/core-impl/src/com/intellij/psi/impl/source/FileTrees.java index 031d99778975..6bf2b1d06ebe 100644 --- a/platform/core-impl/src/com/intellij/psi/impl/source/FileTrees.java +++ b/platform/core-impl/src/com/intellij/psi/impl/source/FileTrees.java @@ -15,32 +15,48 @@ */ package com.intellij.psi.impl.source; +import com.intellij.extapi.psi.StubBasedPsiElementBase; +import com.intellij.lang.ASTNode; +import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.util.Getter; +import com.intellij.psi.PsiElement; +import com.intellij.psi.impl.DebugUtil; +import com.intellij.psi.impl.source.tree.CompositeElement; import com.intellij.psi.impl.source.tree.FileElement; -import com.intellij.psi.stubs.PsiFileStubImpl; -import com.intellij.psi.stubs.StubTree; +import com.intellij.psi.stubs.*; import com.intellij.reference.SoftReference; -import com.intellij.util.containers.ContainerUtil; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import java.lang.ref.Reference; -import java.util.Set; +import java.lang.ref.WeakReference; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Stream; /** * @author peter */ final class FileTrees { + private static final Logger LOG = Logger.getInstance("#com.intellij.psi.impl.source.FileTrees"); + private static final int firstNonFilePsiIndex = 1; + private final PsiFileImpl myFile; private final Reference myStub; private final Getter myTreeElementPointer; // SoftReference/WeakReference to ASTNode or a strong reference to a tree if the file is a DummyHolder - final boolean astLoaded; - final boolean useStrongRefs; + + /** Keeps references to all alive stubbed PSI (using {@link SpineRef}) to ensure PSI identity is preserved after AST/stubs are gc-ed and reloaded */ + @Nullable private final List> myRefToPsi; - private FileTrees(@Nullable Reference stub, @Nullable Getter ast, boolean astLoaded, boolean useStrongRefs) { + private FileTrees(@NotNull PsiFileImpl file, + @Nullable Reference stub, + @Nullable Getter ast, + @Nullable List> refToPsi) { + this.myFile = file; this.myStub = stub; this.myTreeElementPointer = ast; - this.astLoaded = astLoaded; - this.useStrongRefs = useStrongRefs; + this.myRefToPsi = refToPsi; } @Nullable @@ -54,7 +70,41 @@ final class FileTrees { } FileTrees switchToStrongRefs() { - return new FileTrees(myStub, myTreeElementPointer, astLoaded, true); + if (myRefToPsi == null) return this; + + getAllCachedPsi(myRefToPsi).forEach(psi -> { + ASTNode node = psi.getNode(); + LOG.assertTrue(node.getPsi() == psi); + psi.setSubstrateRef(SubstrateRef.createAstStrongRef(node)); + }); + + return new FileTrees(myFile, myStub, myTreeElementPointer, null); + } + + private static Stream getAllCachedPsi(@NotNull List> refToPsi) { + return refToPsi.stream().map(SoftReference::dereference).filter(Objects::nonNull); + } + + boolean useSpineRefs() { + return myRefToPsi != null; + } + + FileTrees switchToSpineRefs(@NotNull StubbedSpine srcSpine) { + List> refToPsi = myRefToPsi; + if (refToPsi == null) refToPsi = new ArrayList<>(Collections.nCopies(srcSpine.getStubCount(), null)); + + for (int i = firstNonFilePsiIndex; i < refToPsi.size(); i++) { + StubBasedPsiElementBase psi = (StubBasedPsiElementBase)srcSpine.getStubPsi(i); + assert psi != null; + psi.setSubstrateRef(new SpineRef(myFile, i)); + StubBasedPsiElementBase existing = SoftReference.dereference(refToPsi.get(i)); + if (existing != null) { + assert existing == psi : "Duplicate PSI found"; + } else { + refToPsi.set(i, new WeakReference<>(psi)); + } + } + return new FileTrees(myFile, myStub, myTreeElementPointer, refToPsi); } FileTrees clearStub(@NotNull String reason) { @@ -62,29 +112,104 @@ final class FileTrees { if (stubHolder != null) { ((PsiFileStubImpl)stubHolder.getRoot()).clearPsi(reason); } - return new FileTrees(null, myTreeElementPointer, astLoaded, useStrongRefs); + + if (myRefToPsi != null) { + DebugUtil.performPsiModification("clearStub", () -> getAllCachedPsi(myRefToPsi).forEach(psi -> { + DebugUtil.onInvalidated(psi); + psi.setSubstrateRef(SubstrateRef.createInvalidRef(psi)); + })); + } + + return new FileTrees(myFile, null, myTreeElementPointer, null); } FileTrees withAst(@NotNull Getter ast) { - return new FileTrees(myStub, ast, true, useStrongRefs); + return new FileTrees(myFile, myStub, ast, myRefToPsi).reconcilePsi(derefStub(), ast.get(), true); } - FileTrees withExclusiveStub(@NotNull StubTree stub, Set allRoots) { - if (derefTreeElement() != null || useStrongRefs) { - throw new RuntimeException(toString() + "; roots=" + allRoots + "; root trees=" + ContainerUtil.map(allRoots, PsiFileImpl::getFileTrees)); - } - return new FileTrees(new SoftReference<>(stub), null, false, false); - } - - FileTrees withGreenStub(@NotNull StubTree stub, @NotNull PsiFileImpl file) { - if (derefTreeElement() == null || !astLoaded) { - throw new RuntimeException("No AST in file " + file + " of " + file.getClass() + "; " + this); - } - return new FileTrees(new SoftReference<>(stub), myTreeElementPointer, true, useStrongRefs); + FileTrees withStub(@NotNull StubTree stub, @Nullable FileElement ast) { + assert derefTreeElement() == ast; + return new FileTrees(myFile, new SoftReference<>(stub), myTreeElementPointer, myRefToPsi) + .reconcilePsi(stub, ast, false); } static FileTrees noStub(@Nullable FileElement ast, @NotNull PsiFileImpl file) { - return new FileTrees(null, ast, ast != null, file instanceof DummyHolder); + return new FileTrees(file, null, ast, null); + } + + /** + * Ensures {@link #myRefToPsi}, stubs and AST all have the same PSI at corresponding indices. + * In case several sources already have PSI (e.g. created during AST parsing), overwrites them with the "correct" one, + * which is taken from {@link #myRefToPsi} if exists, otherwise from either stubs or AST depending on {@code takePsiFromStubs}. + */ + private FileTrees reconcilePsi(@Nullable StubTree stubTree, @Nullable FileElement astRoot, boolean takePsiFromStubs) { + assert stubTree != null || astRoot != null; + + if ((stubTree == null || astRoot == null) && (myRefToPsi == null || !getAllCachedPsi(myRefToPsi).findFirst().isPresent())) { + // there's only one source of PSI, nothing to reconcile + return new FileTrees(myFile, myStub, myTreeElementPointer, null); + } + + List> stubList = stubTree == null ? null : stubTree.getPlainList(); + List nodeList = astRoot == null ? null : astRoot.getStubbedSpine().getSpineNodes(); + + try { + return DebugUtil.performPsiModification("reconcilePsi", () -> { + if (myRefToPsi != null) { + assert myRefToPsi.size() == (stubList != null ? stubList.size() : nodeList.size()) : "Cached PSI count doesn't match actual one"; + bindSubstratesToCachedPsi(stubList, nodeList); + } + + if (stubList != null && nodeList != null) { + assert stubList.size() == nodeList.size() : "Stub count doesn't match stubbed node length"; + + StubbedSpine srcSpine = takePsiFromStubs ? stubTree.getSpine() : astRoot.getStubbedSpine(); + FileTrees result = switchToSpineRefs(srcSpine); + bindStubsWithAst(srcSpine, stubList, nodeList, takePsiFromStubs); + return result; + } + return this; + }); + } + catch (Throwable e) { + LOG.error(e); + myFile.onContentReload(); + myFile.rebuildStub(); + throw StubTreeLoader.getInstance().stubTreeAndIndexDoNotMatch(e.getMessage(), stubTree, myFile); + } + } + + private void bindSubstratesToCachedPsi(List> stubList, List nodeList) { + assert myRefToPsi != null; + for (int i = firstNonFilePsiIndex; i < myRefToPsi.size(); i++) { + StubBasedPsiElementBase cachedPsi = SoftReference.dereference(myRefToPsi.get(i)); + if (cachedPsi != null) { + if (stubList != null) { + //noinspection unchecked + ((StubBase)stubList.get(i)).setPsi(cachedPsi); + } + if (nodeList != null) { + nodeList.get(i).setPsi(cachedPsi); + } + } + } + } + + private static void bindStubsWithAst(StubbedSpine srcSpine, List> stubList, List nodeList, boolean takePsiFromStubs) { + for (int i = firstNonFilePsiIndex; i < stubList.size(); i++) { + StubElement stub = stubList.get(i); + CompositeElement node = nodeList.get(i); + assert stub.getStubType() == node.getElementType() : "Stub type mismatch"; + + PsiElement psi = srcSpine.getStubPsi(i); + assert psi != null; + if (takePsiFromStubs) { + node.setPsi(psi); + } else { + //noinspection unchecked + ((StubBase)stub).setPsi(psi); + } + } } @Override @@ -92,8 +217,7 @@ final class FileTrees { return "FileTrees{" + "stub=" + (myStub == null ? "noRef" : derefStub()) + ", AST=" + (myTreeElementPointer == null ? "noRef" : derefTreeElement()) + - ", astLoaded=" + astLoaded + - ", useStrongRefs=" + useStrongRefs + + ", useSpineRefs=" + useSpineRefs() + '}' ; } } diff --git a/platform/core-impl/src/com/intellij/psi/impl/source/PsiFileImpl.java b/platform/core-impl/src/com/intellij/psi/impl/source/PsiFileImpl.java index 9d2507511d12..eedee4bc13f0 100644 --- a/platform/core-impl/src/com/intellij/psi/impl/source/PsiFileImpl.java +++ b/platform/core-impl/src/com/intellij/psi/impl/source/PsiFileImpl.java @@ -2,7 +2,6 @@ package com.intellij.psi.impl.source; -import com.intellij.extapi.psi.StubBasedPsiElementBase; import com.intellij.ide.util.PsiNavigationSupport; import com.intellij.lang.*; import com.intellij.navigation.ItemPresentation; @@ -14,8 +13,10 @@ import com.intellij.openapi.fileEditor.FileDocumentManager; import com.intellij.openapi.progress.ProgressManager; import com.intellij.openapi.project.Project; import com.intellij.openapi.ui.Queryable; -import com.intellij.openapi.util.Factory; -import com.intellij.openapi.util.*; +import com.intellij.openapi.util.Getter; +import com.intellij.openapi.util.Key; +import com.intellij.openapi.util.Pair; +import com.intellij.openapi.util.TextRange; import com.intellij.openapi.util.text.StringUtil; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.openapi.vfs.VirtualFileWithId; @@ -40,7 +41,6 @@ import com.intellij.util.Function; import com.intellij.util.IncorrectOperationException; import com.intellij.util.PatchedWeakReference; import com.intellij.util.concurrency.AtomicFieldUpdater; -import com.intellij.util.containers.ContainerUtil; import com.intellij.util.text.CharArrayUtil; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -66,8 +66,6 @@ public abstract class PsiFileImpl extends ElementBase implements PsiFileEx, PsiF private volatile FileTrees myTrees = FileTrees.noStub(null, this); private boolean myInvalidated; @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized") - private AstPathPsiMap myRefToPsi; - private final ThreadLocal myFileElementBeingLoaded = new ThreadLocal<>(); protected final PsiManagerEx myManager; public static final Key BUILDING_STUB = new Key<>("Don't use stubs mark!"); private final PsiLock myPsiLock; @@ -80,7 +78,6 @@ public abstract class PsiFileImpl extends ElementBase implements PsiFileEx, PsiF protected PsiFileImpl(@NotNull FileViewProvider provider ) { myManager = (PsiManagerEx)provider.getManager(); myViewProvider = provider; - myRefToPsi = new AstPathPsiMap(getProject()); myPsiLock = ((AbstractFileViewProvider) provider).getFilePsiLock(); } @@ -179,43 +176,16 @@ public abstract class PsiFileImpl extends ElementBase implements PsiFileEx, PsiF } } - FileElement treeElement = createFileElement(viewProvider.getContents()); - treeElement.setPsi(this); - - myFileElementBeingLoaded.set(treeElement); - try { - while (true) { - FileTrees trees = myTrees; - List> bindings = calcStubAstBindings(treeElement, trees); - - FileElement savedTree = ensureTreeElement(viewProvider, treeElement, trees, bindings); - if (savedTree != null) { - return savedTree; - } - } - } - finally { - myFileElementBeingLoaded.remove(); - } - } - - @Nullable - private FileElement ensureTreeElement(@NotNull FileViewProvider viewProvider, - @NotNull FileElement treeElement, - @NotNull FileTrees trees, - @NotNull List> bindings) { synchronized (myPsiLock) { - FileElement existing = derefTreeElement(); - if (existing != null) { - return existing; + FileElement treeElement = derefTreeElement(); + if (treeElement != null) { + return treeElement; } - if (trees != myTrees) { - return null; // try again - } + treeElement = createFileElement(viewProvider.getContents()); + treeElement.setPsi(this); - switchFromStubToAst(bindings, trees); - updateTrees(trees.withAst(createTreeElementPointer(treeElement))); + updateTrees(myTrees.withAst(createTreeElementPointer(treeElement))); if (LOG.isDebugEnabled() && viewProvider.isPhysical()) { LOG.debug("Loaded text for file " + viewProvider.getVirtualFile().getPresentableUrl()); @@ -229,49 +199,15 @@ public abstract class PsiFileImpl extends ElementBase implements PsiFileEx, PsiF @Override public StubbedSpine getStubbedSpine() { StubTree tree = getGreenStubTree(); - return tree != null ? tree.getSpine() : calcTreeElement().getStubbedSpine(); - } - - private void switchFromStubToAst(List> bindings, FileTrees trees) { - if (!bindings.isEmpty() && trees.useStrongRefs) { - List psiStrings = ContainerUtil.map(bindings, pair -> pair.first.getClass().getName()); - LOG.error(this + " of " + getClass() + "; " + psiStrings); - } - - for (int i = 0; i < bindings.size(); i++) { - Pair pair = bindings.get(i); - StubBasedPsiElementBase psi = pair.first; - AstPath path = pair.second; - path.getNode().setPsi(psi); - myRefToPsi.cachePsi(path, psi); - psi.setStubIndex(i + 1); - } - myRefToPsi.clearStubIndexCache(); - } - - private List> calcStubAstBindings(@NotNull FileElement root, FileTrees trees) { - final StubTree stubTree = trees.derefStub(); - if (stubTree == null || trees.astLoaded) { // don't bind green stub to AST: the PSI should already be cached in myRefToPsi - return Collections.emptyList(); - } - - try { - List> result = TreeUtil.calcStubAstBindings(stubTree, root); + if (tree != null) return tree.getSpine(); + + AstSpine astSpine = calcTreeElement().getStubbedSpine(); + if (!myTrees.useSpineRefs()) { synchronized (myPsiLock) { - return ContainerUtil.map(result, pair -> { - StubElement stub = pair.first; - PsiElement psi = stub.getPsi(); - assert psi != null : "Stub " + stub + " (" + stub.getClass() + ") has returned null PSI"; - AstPath path = AstPath.getNodePath((CompositeElement)pair.second); - assert path != null : "Null path"; - return Pair.create((StubBasedPsiElementBase)psi, path); - }); + updateTrees(myTrees.switchToSpineRefs(astSpine)); } } - catch (TreeUtil.StubBindingException e) { - reportStubAstMismatch(e.getMessage(), stubTree); - return Collections.emptyList(); - } + return astSpine; } @Nullable @@ -281,15 +217,6 @@ public abstract class PsiFileImpl extends ElementBase implements PsiFileEx, PsiF return type instanceof IStubFileElementType ? (IStubFileElementType)type : null; } - void reportStubAstMismatch(String message, StubTree stubTree) { - rebuildStub(); - synchronized (myPsiLock) { - updateTrees(myTrees.clearStub(STUB_PSI_MISMATCH)); - } - - throw StubTreeLoader.getInstance().stubTreeAndIndexDoNotMatch(message, stubTree, this); - } - @NotNull protected FileElement createFileElement(CharSequence docText) { final FileElement treeElement; @@ -396,12 +323,13 @@ public abstract class PsiFileImpl extends ElementBase implements PsiFileEx, PsiF PsiFileImpl clone = BlockSupportImpl.getFileCopy(this, providerCopy); copyCopyableDataTo(clone); - clone.myRefToPsi = new AstPathPsiMap(getProject()); if (getTreeElement() != null) { // not set by provider in clone final FileElement treeClone = (FileElement)calcTreeElement().clone(); clone.setTreeElementPointer(treeClone); // should not use setTreeElement here because cloned file still have VirtualFile (SCR17963) treeClone.setPsi(clone); + } else { + clone.setTreeElementPointer(null); } if (viewProvider.isEventSystemEnabled()) { @@ -583,8 +511,6 @@ public abstract class PsiFileImpl extends ElementBase implements PsiFileEx, PsiF DebugUtil.performPsiModification("onContentReload", () -> { synchronized (myPsiLock) { - myRefToPsi.invalidatePsi(); - FileElement treeElement = derefTreeElement(); if (treeElement != null) { treeElement.detachFromFile(); @@ -628,13 +554,12 @@ public abstract class PsiFileImpl extends ElementBase implements PsiFileEx, PsiF public StubTree getStubTree() { assertReadAccessAllowed(); - if (myTrees.astLoaded && !mayReloadStub()) return null; - if (Boolean.TRUE.equals(getUserData(BUILDING_STUB))) return null; + if (getTreeElement() != null) return null; final StubTree derefd = derefStub(); if (derefd != null) return derefd; - if (getElementTypeForStubBuilder() == null) return null; + if (Boolean.TRUE.equals(getUserData(BUILDING_STUB)) || getElementTypeForStubBuilder() == null) return null; final VirtualFile vFile = getVirtualFile(); if (!(vFile instanceof VirtualFileWithId) || !vFile.isValid()) return null; @@ -645,7 +570,7 @@ public abstract class PsiFileImpl extends ElementBase implements PsiFileEx, PsiF final List> roots = StubTreeBuilder.getStubbedRoots(viewProvider); synchronized (myPsiLock) { - if (!mayLoadExclusiveStub()) return null; + if (getTreeElement() != null) return null; final StubTree derefdOnLock = derefStub(); if (derefdOnLock != null) return derefdOnLock; @@ -664,66 +589,31 @@ public abstract class PsiFileImpl extends ElementBase implements PsiFileEx, PsiF return null; } - // first, set all references from stubs to existing PSI (in AST or AstPathPsiMap) - Map bindings = prepareAllStubTrees(roots, stubRoots); - StubTree result = bindings.get(this); - assert result != null : "Current file not in root list: " + roots + ", vp=" + viewProvider; - - // now stubs can be safely published - for (PsiFileImpl eachPsiRoot : bindings.keySet()) { - FileTrees trees = eachPsiRoot.myTrees; - StubTree stub = bindings.get(eachPsiRoot); - FileElement ast = trees.derefTreeElement(); - eachPsiRoot.updateTrees(ast == null ? trees.withExclusiveStub(stub, bindings.keySet()) : trees.withGreenStub(stub, eachPsiRoot)); + StubTree result = null; + for (int i = 0; i < roots.size(); i++) { + PsiFileImpl eachPsiRoot = (PsiFileImpl)roots.get(i).second; + if (eachPsiRoot.derefStub() == null) { + StubTree stubTree = eachPsiRoot.setStubTree(stubRoots[i]); + if (eachPsiRoot == this) { + result = stubTree; + } + } } + + assert result != null : "Current file not in root list: " + roots + ", vp=" + viewProvider; return result; } } - private boolean mayLoadExclusiveStub() { - return getTreeElement() == null && !hasUnbindableCachedPsi(); - } - - private static Map prepareAllStubTrees(List> roots, PsiFileStub[] rootStubs) { - Map bindings = ContainerUtil.newIdentityHashMap(); - for (int i = 0; i < roots.size(); i++) { - PsiFileImpl eachPsiRoot = (PsiFileImpl)roots.get(i).second; - //noinspection unchecked - ((StubBase)rootStubs[i]).setPsi(eachPsiRoot); - StubTree stubTree = new StubTree(rootStubs[i]); - FileElement fileElement = eachPsiRoot.getTreeElement(); - stubTree.setDebugInfo("created in getStubTree(), with AST = " + (fileElement != null)); - if (fileElement != null) { - // Set references from these stubs to AST, because: - // Stub index might call getStubTree on main PSI file, but then use getPlainListFromAllRoots and return stubs from another file. - // Even if that file already has AST, stub.getPsi() should be the same as in AST - TreeUtil.bindStubsToTree(stubTree, fileElement); - eachPsiRoot.myRefToPsi.clearStubIndexCache(); - bindings.put(eachPsiRoot, stubTree); - } else if (eachPsiRoot.derefStub() == null && eachPsiRoot.mayLoadExclusiveStub()) { - eachPsiRoot.bindStubsToCachedPsi(stubTree); - bindings.put(eachPsiRoot, stubTree); - } - } - return bindings; - } - - private boolean mayReloadStub() { - if (getTreeElement() != null || useStrongRefs()) { - return false; - } - StubTreeLoader loader = StubTreeLoader.getInstance(); - if (loader != null && loader.isStubReloadingProhibited()) { - return false; - } - return !hasUnbindableCachedPsi(); - } - - private boolean hasUnbindableCachedPsi() { - synchronized (myPsiLock) { - return ContainerUtil.exists(myViewProvider.getAllFiles(), - file -> file instanceof PsiFileImpl && ((PsiFileImpl)file).myRefToPsi.hasUnbindableCachedPsi()); - } + @NotNull + private StubTree setStubTree(PsiFileStub root) { + //noinspection unchecked + ((StubBase)root).setPsi(this); + StubTree stubTree = new StubTree(root); + FileElement fileElement = getTreeElement(); + stubTree.setDebugInfo("created in getStubTree(), with AST = " + (fileElement != null)); + updateTrees(myTrees.withStub(stubTree, fileElement)); + return stubTree; } @Nullable @@ -738,23 +628,8 @@ public abstract class PsiFileImpl extends ElementBase implements PsiFileEx, PsiF } } - FileTrees getFileTrees() { - return myTrees; - } - - private void bindStubsToCachedPsi(StubTree stubTree) { - myRefToPsi.getAllCachedPsi().forEach(psi -> { - int index = psi.getStubIndex(); - if (index >= 0) { - //noinspection unchecked - ((StubBase)stubTree.getPlainList().get(index)).setPsi(psi); - } - }); - } - protected PsiFileImpl cloneImpl(FileElement treeElementClone) { PsiFileImpl clone = (PsiFileImpl)super.clone(); - clone.myRefToPsi = new AstPathPsiMap(getProject()); clone.setTreeElementPointer(treeElementClone); // should not use setTreeElement here because cloned file still have VirtualFile (SCR17963) treeElementClone.setPsi(clone); return clone; @@ -791,14 +666,8 @@ public abstract class PsiFileImpl extends ElementBase implements PsiFileEx, PsiF @NotNull public final FileElement calcTreeElement() { - // Attempt to find (loaded) tree element without taking lock first. FileElement treeElement = getTreeElement(); - if (treeElement != null) return treeElement; - - treeElement = myFileElementBeingLoaded.get(); - if (treeElement != null) return treeElement; - - return loadTreeElement(); + return treeElement != null ? treeElement : loadTreeElement(); } @Override @@ -1026,7 +895,6 @@ public abstract class PsiFileImpl extends ElementBase implements PsiFileEx, PsiF if (tree != null) { return tree; } - assert myFileElementBeingLoaded.get() == null : "non-empty thread-local"; FileElement fileElement = calcTreeElement(); synchronized (myPsiLock) { tree = derefStub(); @@ -1052,23 +920,14 @@ public abstract class PsiFileImpl extends ElementBase implements PsiFileEx, PsiF tree = new StubTree((PsiFileStub)currentStubTree); tree.setDebugInfo("created in calcStubTree"); - try { - TreeUtil.bindStubsToTree(tree, fileElement); - myRefToPsi.clearStubIndexCache(); - } - catch (TreeUtil.StubBindingException e) { - rebuildStub(); - throw new RuntimeException("Stub and PSI element type mismatch in " + getName(), e); - } - - updateTrees(myTrees.withGreenStub(tree, this)); + updateTrees(myTrees.withStub(tree, fileElement)); } return tree; } } - private void rebuildStub() { + final void rebuildStub() { ApplicationManager.getApplication().invokeLater(() -> { if (!myManager.isDisposed()) { myManager.dropPsiCaches(); @@ -1104,13 +963,10 @@ public abstract class PsiFileImpl extends ElementBase implements PsiFileEx, PsiF public final void beforeAstChange() { checkWritable(); - if (!useStrongRefs()) { - synchronized (myPsiLock) { - for (PsiFile root : myViewProvider.getAllFiles()) { - if (root instanceof PsiFileImpl) { - ((PsiFileImpl)root).switchToStrongRefs(); - } - } + synchronized (myPsiLock) { + FileTrees updated = myTrees.switchToStrongRefs(); + if (updated != myTrees) { + updateTrees(updated); } } } @@ -1123,42 +979,4 @@ public abstract class PsiFileImpl extends ElementBase implements PsiFileEx, PsiF CheckUtil.checkWritable(this); } } - - private void switchToStrongRefs() { - FileElement node = calcTreeElement(); - updateTrees(myTrees.switchToStrongRefs()); - myRefToPsi.switchToStrongRefs(); - AstPath.invalidatePaths(node); - } - - @Nullable - public StubBasedPsiElementBase obtainPsi(@NotNull AstPath path, @NotNull Factory> creator) { - if (useStrongRefs()) { - return null; - } - - StubBasedPsiElementBase psi = myRefToPsi.getCachedPsi(path); - if (psi != null) return psi; - - synchronized (myPsiLock) { - if (useStrongRefs()) { - return null; - } - - psi = myRefToPsi.getCachedPsi(path); - return psi != null ? psi : myRefToPsi.cachePsi(path, creator.create()); - } - } - - final AstPathPsiMap getRefToPsi() { - return myRefToPsi; - } - - public final boolean useStrongRefs() { - return myTrees.useStrongRefs; - } - - public boolean mayCacheAst() { - return myFileElementBeingLoaded.get() == null; - } } diff --git a/platform/core-impl/src/com/intellij/psi/impl/source/SpineRef.java b/platform/core-impl/src/com/intellij/psi/impl/source/SpineRef.java new file mode 100644 index 000000000000..f16d6b4f859f --- /dev/null +++ b/platform/core-impl/src/com/intellij/psi/impl/source/SpineRef.java @@ -0,0 +1,53 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.psi.impl.source; + +import com.intellij.lang.ASTNode; +import com.intellij.psi.PsiFile; +import com.intellij.psi.stubs.Stub; +import com.intellij.psi.stubs.StubTree; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * @author peter + */ +class SpineRef extends SubstrateRef { + private final PsiFileImpl myFile; + private final int myIndex; + + SpineRef(@NotNull PsiFileImpl file, int index) { + myFile = file; + myIndex = index; + } + + @NotNull + @Override + public ASTNode getNode() { + return myFile.calcTreeElement().getStubbedSpine().getSpineNodes().get(myIndex); + } + + @Nullable + @Override + public Stub getStub() { + StubTree tree = myFile.getStubTree(); + return tree == null ? null : tree.getPlainList().get(myIndex); + } + + @Nullable + @Override + public Stub getGreenStub() { + StubTree tree = myFile.getGreenStubTree(); + return tree == null ? null : tree.getPlainList().get(myIndex); + } + + @Override + public boolean isValid() { + return myFile.isValid(); + } + + @NotNull + @Override + public PsiFile getContainingFile() { + return myFile; + } +} diff --git a/platform/core-impl/src/com/intellij/psi/impl/source/SubstrateRef.java b/platform/core-impl/src/com/intellij/psi/impl/source/SubstrateRef.java index 45491ba69508..f862caa7d683 100644 --- a/platform/core-impl/src/com/intellij/psi/impl/source/SubstrateRef.java +++ b/platform/core-impl/src/com/intellij/psi/impl/source/SubstrateRef.java @@ -42,13 +42,13 @@ public abstract class SubstrateRef { public abstract ASTNode getNode(); @Nullable - public Stub getStub(int stubIndex) { + public Stub getStub() { return null; } @Nullable - public Stub getGreenStub(int index) { - return getStub(index); + public Stub getGreenStub() { + return getStub(); } public abstract boolean isValid(); @@ -119,7 +119,7 @@ public abstract class SubstrateRef { @NotNull @Override - public Stub getStub(int stubIndex) { + public Stub getStub() { return myStub; } diff --git a/platform/core-impl/src/com/intellij/psi/impl/source/tree/AstPath.java b/platform/core-impl/src/com/intellij/psi/impl/source/tree/AstPath.java deleted file mode 100644 index b91a3f305117..000000000000 --- a/platform/core-impl/src/com/intellij/psi/impl/source/tree/AstPath.java +++ /dev/null @@ -1,311 +0,0 @@ -/* - * Copyright 2000-2016 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.intellij.psi.impl.source.tree; - -import com.intellij.extapi.psi.StubBasedPsiElementBase; -import com.intellij.openapi.diagnostic.Logger; -import com.intellij.openapi.util.Key; -import com.intellij.openapi.vfs.VirtualFileWithId; -import com.intellij.psi.PsiElement; -import com.intellij.psi.impl.source.PsiFileImpl; -import com.intellij.psi.impl.source.SubstrateRef; -import com.intellij.psi.stubs.IStubElementType; -import com.intellij.psi.stubs.Stub; -import com.intellij.psi.stubs.StubTree; -import com.intellij.reference.SoftReference; -import com.intellij.util.containers.ContainerUtil; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -import java.lang.ref.WeakReference; -import java.util.List; - -/** - * A lightweight object representing a chain of node indices (among all lazy-parseable and stub-based elements) - * allowing to restore a specific node after it's been garbage-collected and recreated. - * - * @author peter - */ -public abstract class AstPath extends SubstrateRef { - @SuppressWarnings("ConstantForZeroLengthArrayAllocation") // need a different instance to distinguish removed children for diagnostics - private static final CompositeElement[] REMOVED_PATH_CHILDREN = new CompositeElement[0]; - private static final Logger LOG = Logger.getInstance("#com.intellij.psi.impl.source.tree.AstPath"); - private static final Key PATH_CHILDREN = Key.create("PATH_CHILDREN"); - private static final Key NODE_PATH = Key.create("NODE_PATH"); - - @NotNull - public abstract PsiFileImpl getContainingFile(); - - @NotNull - public abstract CompositeElement getNode(); - - @Override - public boolean isValid() { - return getContainingFile().isValid(); - } - - protected abstract int getDepth(); - - @Nullable - @Override - public Stub getStub(int stubIndex) { - if (stubIndex < 0) return null; - - StubTree stubTree = getFileStubTree(); - return stubTree == null ? null : stubTree.getPlainList().get(stubIndex); - } - - @Nullable - protected abstract StubTree getFileStubTree(); - - @Nullable - @Override - public Stub getGreenStub(int stubIndex) { - if (stubIndex < 0) return null; - - StubTree stubTree = getContainingFile().getGreenStubTree(); - return stubTree == null ? null : stubTree.getPlainList().get(stubIndex); - } - - @Nullable - public static AstPath getNodePath(@NotNull CompositeElement node) { - if (node instanceof FileElement) { - PsiElement psi = node.getCachedPsi(); - if (!(psi instanceof PsiFileImpl)) return null; - - PsiFileImpl file = (PsiFileImpl)psi; - if (!(file.getVirtualFile() instanceof VirtualFileWithId) || file.getElementTypeForStubBuilder() == null) { - return null; - } - return new RootPath(file); - } - - return node.getUserData(NODE_PATH); - } - - static void cacheNodePaths(@NotNull LazyParseableElement parent, @Nullable TreeElement child, @Nullable AstPath parentPath) { - if (parentPath == null) { - return; - } - - final int depth = parentPath.getDepth() + 1; - - final List children = ContainerUtil.newArrayList(); - while (child != null) { - child.acceptTree(new RecursiveTreeElementWalkingVisitor(false) { - @Override - public void visitComposite(CompositeElement composite) { - boolean lazy = composite instanceof LazyParseableElement; - if (lazy || composite.getElementType() instanceof IStubElementType) { - int index = children.size(); - composite.putUserData(NODE_PATH, depth % 4 == 0 ? new MilestoneChildPath(parentPath, index, depth) : new ChildPath(parentPath, index)); - children.add(composite); - } - - if (!lazy) { - super.visitComposite(composite); - } - } - }); - child = child.getTreeNext(); - } - - parent.putUserData(PATH_CHILDREN, children.isEmpty() ? CompositeElement.EMPTY_ARRAY : children.toArray(CompositeElement.EMPTY_ARRAY)); - - for (CompositeElement each : children) { - if (each instanceof LazyParseableElement && ((LazyParseableElement)each).isParsed()) { - cacheNodePaths((LazyParseableElement)each, each.getFirstChildNode(), getNodePath(each)); - } - } - } - - public static void invalidatePaths(@NotNull LazyParseableElement scope) { - CompositeElement[] children = scope.getUserData(PATH_CHILDREN); - if (children == null) return; - - scope.putUserData(PATH_CHILDREN, REMOVED_PATH_CHILDREN); - for (CompositeElement child : children) { - child.putUserData(NODE_PATH, null); - assertConsistency(child.getCachedPsi()); - if (child instanceof LazyParseableElement) { - invalidatePaths((LazyParseableElement)child); - } - } - } - - private static void assertConsistency(PsiElement cachedPsi) { - if (cachedPsi instanceof StubBasedPsiElementBase && - ((StubBasedPsiElementBase)cachedPsi).getSubstrateRef() instanceof AstPath) { - LOG.error("Expected strong reference at " + cachedPsi + - " of " + cachedPsi.getClass() + - " and " + ((StubBasedPsiElementBase)cachedPsi).getElementType()); - } - } - - private static class ChildPath extends AstPath { - private final AstPath myParent; - private final int myIndex; - - ChildPath(@NotNull AstPath parent, int index) { - myParent = parent; - myIndex = index; - } - - @NotNull - @Override - public PsiFileImpl getContainingFile() { - return myParent.getContainingFile(); - } - - @NotNull - @Override - public CompositeElement getNode() { - CompositeElement parentNode = myParent.getNode(); - //noinspection ResultOfMethodCallIgnored - parentNode.getFirstChildNode(); // expand chameleons, populate PATH_CHILDREN array - CompositeElement[] children = parentNode.getUserData(PATH_CHILDREN); - boolean removed = children == REMOVED_PATH_CHILDREN; - if (children == null || removed) { - throw reportMissingChildren(parentNode, removed); - } - if (myIndex >= children.length) { - throw new AssertionError(myIndex + " >= " + children.length + "; " + parentNode + " of " + parentNode.getClass()); - } - return children[myIndex]; - } - - private AssertionError reportMissingChildren(CompositeElement parentNode, boolean removed) { - String message = "No path children in " + parentNode + " of " + parentNode.getClass() + "; removed=" + removed; - - PsiFileImpl file = getContainingFile(); - message += "\n file: " + file + " of " + file.getClass() + "; physical=" + file.isPhysical() + "; useStrongRefs=" + file.useStrongRefs(); - - FileElement fileElement = file.getTreeElement(); - message += "\n ast=" + fileElement; - if (fileElement != null) { - CompositeElement[] rootChildren = fileElement.getUserData(PATH_CHILDREN); - message += "; root.children=" + (rootChildren == REMOVED_PATH_CHILDREN ? "removed" : rootChildren == null ? "null" : "nonNull"); - } - return new AssertionError(message); - } - - @Override - protected int getDepth() { - return 1 + myParent.getDepth(); - } - - @Override - protected StubTree getFileStubTree() { - return myParent.getFileStubTree(); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof ChildPath)) return false; - - ChildPath path = (ChildPath)o; - return myIndex == path.myIndex && myParent.equals(path.myParent); - } - - @Override - public int hashCode() { - return 31 * myParent.hashCode() + myIndex; - } - } - - private static class MilestoneChildPath extends ChildPath { - private final int myDepth; - private final PsiFileImpl myFile; - private volatile WeakReference myNode; - - MilestoneChildPath(@NotNull AstPath parent, int index, int depth) { - super(parent, index); - myDepth = depth; - myFile = parent.getContainingFile(); - } - - @NotNull - @Override - public CompositeElement getNode() { - CompositeElement node = SoftReference.dereference(myNode); - if (node == null) { - node = super.getNode(); - if (myFile.mayCacheAst()) { - myNode = new WeakReference<>(node); - } - } - return node; - } - - @Override - protected StubTree getFileStubTree() { - return SoftReference.dereference(myNode) == null ? myFile.getStubTree() : null; - } - - @NotNull - @Override - public PsiFileImpl getContainingFile() { - return myFile; - } - - @Override - protected int getDepth() { - return myDepth; - } - } - - private static class RootPath extends AstPath { - private final PsiFileImpl myFile; - - RootPath(@NotNull PsiFileImpl file) { - myFile = file; - } - - @Override - public boolean equals(Object o) { - return this == o || o instanceof RootPath && myFile.equals(((RootPath)o).myFile); - } - - @Override - public int hashCode() { - return myFile.hashCode(); - } - - @NotNull - @Override - public PsiFileImpl getContainingFile() { - return myFile; - } - - @NotNull - @Override - public CompositeElement getNode() { - return myFile.calcTreeElement(); - } - - @Override - protected int getDepth() { - return 0; - } - - @Override - protected StubTree getFileStubTree() { - return myFile.getStubTree(); - } - } - -} diff --git a/platform/core-impl/src/com/intellij/psi/impl/source/tree/AstSpine.java b/platform/core-impl/src/com/intellij/psi/impl/source/tree/AstSpine.java index c93d08e4e92f..f6e1107abbbc 100644 --- a/platform/core-impl/src/com/intellij/psi/impl/source/tree/AstSpine.java +++ b/platform/core-impl/src/com/intellij/psi/impl/source/tree/AstSpine.java @@ -57,7 +57,8 @@ public class AstSpine implements StubbedSpine { return index >= myNodes.size() ? null : myNodes.get(index).getElementType(); } - List getNodes() { + @NotNull + public List getSpineNodes() { return myNodes; } } diff --git a/platform/core-impl/src/com/intellij/psi/impl/source/tree/CompositeElement.java b/platform/core-impl/src/com/intellij/psi/impl/source/tree/CompositeElement.java index 46158a8fa2de..b205c1c3aa34 100644 --- a/platform/core-impl/src/com/intellij/psi/impl/source/tree/CompositeElement.java +++ b/platform/core-impl/src/com/intellij/psi/impl/source/tree/CompositeElement.java @@ -18,7 +18,6 @@ package com.intellij.psi.impl.source.tree; import com.intellij.diagnostic.ThreadDumper; import com.intellij.extapi.psi.ASTDelegatePsiElement; -import com.intellij.extapi.psi.StubBasedPsiElementBase; import com.intellij.lang.*; import com.intellij.openapi.application.ApplicationManager; import com.intellij.openapi.diagnostic.Logger; @@ -29,9 +28,10 @@ import com.intellij.psi.PsiElement; import com.intellij.psi.PsiFile; import com.intellij.psi.impl.DebugUtil; import com.intellij.psi.impl.FreeThreadedFileViewProvider; -import com.intellij.psi.impl.source.*; +import com.intellij.psi.impl.source.DummyHolder; +import com.intellij.psi.impl.source.DummyHolderFactory; +import com.intellij.psi.impl.source.SourceTreeToPsiMap; import com.intellij.psi.impl.source.codeStyle.CodeEditUtil; -import com.intellij.psi.stubs.IStubElementType; import com.intellij.psi.tree.IElementType; import com.intellij.psi.tree.TokenSet; import com.intellij.psi.util.PsiUtilCore; @@ -723,21 +723,10 @@ public class CompositeElement extends TreeElement { PsiElement wrapper = myWrapper; if (wrapper != null) return wrapper; - wrapper = obtainStubBasedPsi(); - if (wrapper == null) wrapper = createPsiNoLock(); + wrapper = createPsiNoLock(); return ourPsiUpdater.compareAndSet(this, null, wrapper) ? wrapper : ObjectUtils.assertNotNull(myWrapper); } - /** - * If AST has been gced and recreated, but someone still holds a reference to a PSI, then {@link #getPsi()} should return the very same PSI object. - * So we try to find that PSI in file's {@link AstPathPsiMap}. - */ - @Nullable - private PsiElement obtainStubBasedPsi() { - AstPath path = getElementType() instanceof IStubElementType ? AstPath.getNodePath(this) : null; - return path == null ? null : path.getContainingFile().obtainPsi(path, () -> (StubBasedPsiElementBase)createPsiNoLock()); - } - @Override public T getPsi(@NotNull Class clazz) { return LeafElement.getPsi(clazz, getPsi(), LOG); @@ -755,6 +744,10 @@ public class CompositeElement extends TreeElement { } public void setPsi(@NotNull PsiElement psi) { + PsiElement prev = myWrapper; + if (prev != null && prev != psi) { + DebugUtil.onInvalidated(prev); + } myWrapper = psi; } @@ -769,13 +762,6 @@ public class CompositeElement extends TreeElement { } public void rawAddChildrenWithoutNotifications(@NotNull TreeElement first) { - if (DebugUtil.DO_EXPENSIVE_CHECKS && !(this instanceof LazyParseableElement)) { - PsiFileImpl file = getCachedFile(this); - if (file != null && !file.useStrongRefs()) { - throw new AssertionError("Attempt to modify PSI in a file with weakly-referenced AST. Possible cause: missing PomTransaction."); - } - } - final TreeElement last = getLastChildNode(); if (last == null){ TreeElement chainLast = rawSetParents(first, this); diff --git a/platform/core-impl/src/com/intellij/psi/impl/source/tree/FileElement.java b/platform/core-impl/src/com/intellij/psi/impl/source/tree/FileElement.java index 60988897c194..311da5c0179f 100644 --- a/platform/core-impl/src/com/intellij/psi/impl/source/tree/FileElement.java +++ b/platform/core-impl/src/com/intellij/psi/impl/source/tree/FileElement.java @@ -18,6 +18,7 @@ package com.intellij.psi.impl.source.tree; import com.intellij.lang.*; import com.intellij.openapi.util.Getter; +import com.intellij.openapi.util.RecursionManager; import com.intellij.psi.PsiElement; import com.intellij.psi.StubBuilder; import com.intellij.psi.impl.PsiManagerEx; @@ -27,12 +28,9 @@ import com.intellij.psi.stubs.IStubElementType; import com.intellij.psi.tree.IElementType; import com.intellij.psi.tree.ILightStubFileElementType; import com.intellij.psi.tree.IStubFileElementType; -import com.intellij.reference.SoftReference; import com.intellij.util.CharTable; import org.jetbrains.annotations.NotNull; -import java.lang.ref.Reference; -import java.lang.ref.WeakReference; import java.util.ArrayList; import java.util.List; @@ -40,7 +38,7 @@ public class FileElement extends LazyParseableElement implements FileASTNode, Ge public static final FileElement[] EMPTY_ARRAY = new FileElement[0]; private volatile CharTable myCharTable = new CharTableImpl(); private volatile boolean myDetached; - private volatile Reference myStubbedSpine; + private volatile AstSpine myStubbedSpine; @Override protected PsiElement createPsiNoLock() { @@ -108,13 +106,17 @@ public class FileElement extends LazyParseableElement implements FileASTNode, Ge @NotNull public final AstSpine getStubbedSpine() { - AstSpine result = SoftReference.dereference(myStubbedSpine); + AstSpine result = myStubbedSpine; if (result == null) { - IStubFileElementType type = ((PsiFileImpl)getPsi()).getElementTypeForStubBuilder(); + PsiFileImpl file = (PsiFileImpl)getPsi(); + IStubFileElementType type = file.getElementTypeForStubBuilder(); if (type == null) return AstSpine.EMPTY_SPINE; - result = new AstSpine(calcStubbedDescendants(type.getBuilder())); - myStubbedSpine = getManager().isBatchFilesProcessingMode() ? new WeakReference<>(result) : new SoftReference<>(result); + result = RecursionManager.doPreventingRecursion(file, false, () -> new AstSpine(calcStubbedDescendants(type.getBuilder()))); + if (result == null) { + throw new StackOverflowError("Endless recursion prevented"); + } + myStubbedSpine = result; } return result; } diff --git a/platform/core-impl/src/com/intellij/psi/impl/source/tree/LazyParseableElement.java b/platform/core-impl/src/com/intellij/psi/impl/source/tree/LazyParseableElement.java index af2fa02c2df1..bf0501167bdc 100644 --- a/platform/core-impl/src/com/intellij/psi/impl/source/tree/LazyParseableElement.java +++ b/platform/core-impl/src/com/intellij/psi/impl/source/tree/LazyParseableElement.java @@ -19,7 +19,6 @@ */ package com.intellij.psi.impl.source.tree; -import com.intellij.openapi.application.ApplicationManager; import com.intellij.openapi.diagnostic.Attachment; import com.intellij.openapi.diagnostic.LogUtil; import com.intellij.openapi.diagnostic.Logger; @@ -200,7 +199,7 @@ public class LazyParseableElement extends CompositeElement { } if (parsedNode != null) { - setChildren(parsedNode, AstPath.getNodePath(this)); + setChildren(parsedNode); } myParsed = true; @@ -220,10 +219,9 @@ public class LazyParseableElement extends CompositeElement { } } - private void setChildren(@NotNull TreeElement parsedNode, @Nullable AstPath thisPath) { + private void setChildren(@NotNull TreeElement parsedNode) { ProgressManager.getInstance().executeNonCancelableSection(() -> { try { - AstPath.cacheNodePaths(this, parsedNode, thisPath); TreeElement last = rawSetParents(parsedNode, this); super.setFirstChildNode(parsedNode); super.setLastChildNode(last); diff --git a/platform/core-impl/src/com/intellij/psi/impl/source/tree/TreeUtil.java b/platform/core-impl/src/com/intellij/psi/impl/source/tree/TreeUtil.java index f25af12cd404..c9acf84d4484 100644 --- a/platform/core-impl/src/com/intellij/psi/impl/source/tree/TreeUtil.java +++ b/platform/core-impl/src/com/intellij/psi/impl/source/tree/TreeUtil.java @@ -16,21 +16,14 @@ package com.intellij.psi.impl.source.tree; -import com.intellij.extapi.psi.StubBasedPsiElementBase; import com.intellij.lang.ASTNode; import com.intellij.lexer.Lexer; -import com.intellij.openapi.application.ApplicationManager; import com.intellij.openapi.progress.ProgressIndicator; import com.intellij.openapi.util.Couple; import com.intellij.openapi.util.Key; -import com.intellij.openapi.util.Pair; import com.intellij.psi.PsiComment; -import com.intellij.psi.PsiFile; import com.intellij.psi.PsiWhiteSpace; import com.intellij.psi.impl.DebugUtil; -import com.intellij.psi.stubs.StubBase; -import com.intellij.psi.stubs.StubElement; -import com.intellij.psi.stubs.StubTree; import com.intellij.psi.templateLanguages.OuterLanguageElement; import com.intellij.psi.tree.IElementType; import com.intellij.psi.tree.IStrongWhitespaceHolderElementType; @@ -38,7 +31,9 @@ import com.intellij.psi.tree.TokenSet; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import java.util.*; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; public class TreeUtil { @@ -441,45 +436,6 @@ public class TreeUtil { boolean isStrongElementOnRisingSlope = true; } - public static class StubBindingException extends RuntimeException { - StubBindingException(String message) { - super(message); - } - } - - public static void bindStubsToTree(@NotNull StubTree stubTree, @NotNull FileElement tree) throws StubBindingException { - List> bindings = calcStubAstBindings(stubTree, tree); - - for (int i = 0; i < bindings.size(); i++) { - Pair pair = bindings.get(i); - StubBasedPsiElementBase psi = (StubBasedPsiElementBase)pair.second.getPsi(); - //noinspection unchecked - pair.first.setPsi(psi); - psi.setStubIndex(i + 1); - } - } - - @NotNull - public static List> calcStubAstBindings(@NotNull StubTree stubTree, @NotNull FileElement tree) throws StubBindingException { - PsiFile file = (PsiFile)tree.getPsi(); - List nodes = tree.getStubbedSpine().getNodes(); - List> stubs = stubTree.getPlainList(); - if (stubs.size() != nodes.size()) { - throw new StubBindingException("Stub list in " + file.getName() + " length differs from PSI"); - } - - List> bindings = new ArrayList<>(); - for (int i = 1; i < stubs.size(); i++) { // start from 1 to skip file root stub - StubBase stub = (StubBase)stubs.get(i); - CompositeElement node = nodes.get(i); - if (stub.getStubType() != node.getElementType()) { - throw new StubBindingException("stub:" + stub + ", AST:" + node.getElementType()); - } - bindings.add(Pair.create(stub, node)); - } - return bindings; - } - @Nullable public static ASTNode skipWhitespaceAndComments(final ASTNode node, boolean forward) { return skipWhitespaceCommentsAndTokens(node, TokenSet.EMPTY, forward); diff --git a/platform/core-impl/src/com/intellij/psi/stubs/StubTreeLoader.java b/platform/core-impl/src/com/intellij/psi/stubs/StubTreeLoader.java index 81c764262246..8196d8352c28 100644 --- a/platform/core-impl/src/com/intellij/psi/stubs/StubTreeLoader.java +++ b/platform/core-impl/src/com/intellij/psi/stubs/StubTreeLoader.java @@ -51,10 +51,6 @@ public abstract class StubTreeLoader { @Nullable public abstract ObjectStubTree readFromVFile(Project project, final VirtualFile vFile); - public boolean isStubReloadingProhibited() { - return false; - } - public abstract void rebuildStubTree(VirtualFile virtualFile); public abstract boolean canHaveStub(VirtualFile file); diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubTreeLoaderImpl.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubTreeLoaderImpl.java index c09422b05da4..8871cba816d1 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubTreeLoaderImpl.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubTreeLoaderImpl.java @@ -189,11 +189,6 @@ public class StubTreeLoaderImpl extends StubTreeLoader { } } - @Override - public boolean isStubReloadingProhibited() { - return ourStubReloadingProhibited; - } - private static int getCurrentTextContentLength(Project project, VirtualFile vFile, Document document, PsiFile psiFile) { if (vFile.getFileType().isBinary()) { return -1;