diff --git a/java/java-psi-impl/src/com/intellij/psi/impl/compiled/ClassFileStubBuilder.java b/java/java-psi-impl/src/com/intellij/psi/impl/compiled/ClassFileStubBuilder.java index 87489e67a52a..6e6546881bc7 100644 --- a/java/java-psi-impl/src/com/intellij/psi/impl/compiled/ClassFileStubBuilder.java +++ b/java/java-psi-impl/src/com/intellij/psi/impl/compiled/ClassFileStubBuilder.java @@ -6,21 +6,24 @@ import com.intellij.openapi.vfs.VirtualFile; import com.intellij.psi.compiled.ClassFileDecompilers; import com.intellij.psi.stubs.BinaryFileStubBuilder; import com.intellij.psi.stubs.PsiFileStub; -import com.intellij.psi.stubs.StubElement; +import com.intellij.psi.stubs.Stub; import com.intellij.util.cls.ClsFormatException; import com.intellij.util.containers.ContainerUtil; import com.intellij.util.indexing.FileContent; +import com.intellij.util.indexing.InvertedIndex; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import java.util.Comparator; import java.util.List; +import java.util.function.Supplier; import static com.intellij.psi.compiled.ClassFileDecompilers.Full; /** * @author max */ -public class ClassFileStubBuilder implements BinaryFileStubBuilder { +public class ClassFileStubBuilder implements BinaryFileStubBuilder.CompositeBinaryFileStubBuilder { private static final Logger LOG = Logger.getInstance(ClassFileStubBuilder.class); public static final int STUB_VERSION = 23; @@ -30,41 +33,47 @@ public class ClassFileStubBuilder implements BinaryFileStubBuilder { return true; } + @Nullable @Override - public StubElement buildStubTree(@NotNull FileContent fileContent) { - VirtualFile file = fileContent.getFile(); - byte[] content = fileContent.getContent(); + public ClassFileDecompilers.Decompiler getSubBuilder(@NotNull FileContent fileContent) { + return setContentAndCompute(fileContent, () -> ClassFileDecompilers.find(fileContent.getFile())); + } - try { + @NotNull + @Override + public String getSubBuilderVersion(@Nullable ClassFileDecompilers.Decompiler decompiler) { + if (decompiler == null) return "default"; + int version = decompiler instanceof Full ? ((Full)decompiler).getStubBuilder().getStubVersion() : 0; + return decompiler.getClass().getName() + ":" + version; + } + + @Nullable + @Override + public Stub buildStubTree(@NotNull FileContent fileContent, @Nullable ClassFileDecompilers.Decompiler decompiler) { + return setContentAndCompute(fileContent, () -> { + VirtualFile file = fileContent.getFile(); try { - file.setPreloadedContentHint(content); - ClassFileDecompilers.Decompiler decompiler = ClassFileDecompilers.find(file); if (decompiler instanceof Full) { - return ((Full)decompiler).getStubBuilder().buildFileStub(fileContent); + return ((Full) decompiler).getStubBuilder().buildFileStub(fileContent); } - } - catch (ClsFormatException e) { + } catch (ClsFormatException e) { if (LOG.isDebugEnabled()) LOG.debug(file.getPath(), e); else LOG.info(file.getPath() + ": " + e.getMessage()); } try { - PsiFileStub stub = ClsFileImpl.buildFileStub(file, content); + PsiFileStub stub = ClsFileImpl.buildFileStub(file, fileContent.getContent()); if (stub == null && fileContent.getFileName().indexOf('$') < 0) { LOG.info("No stub built for the file " + fileContent); } return stub; - } - catch (ClsFormatException e) { + } catch (ClsFormatException e) { if (LOG.isDebugEnabled()) LOG.debug(file.getPath(), e); else LOG.info(file.getPath() + ": " + e.getMessage()); } - } - finally { - file.setPreloadedContentHint(null); - } - return null; + return null; + }); } private static final Comparator CLASS_NAME_COMPARATOR = Comparator.comparing(o -> o.getClass().getName()); @@ -73,14 +82,25 @@ public class ClassFileStubBuilder implements BinaryFileStubBuilder { public int getStubVersion() { int version = STUB_VERSION; - List decompilers = ContainerUtil.newArrayList(ClassFileDecompilers.EP_NAME.getExtensions()); - decompilers.sort(CLASS_NAME_COMPARATOR); - for (ClassFileDecompilers.Decompiler decompiler : decompilers) { - if (decompiler instanceof Full) { - version = version * 31 + ((Full)decompiler).getStubBuilder().getStubVersion() + decompiler.getClass().getName().hashCode(); + if (!InvertedIndex.ARE_COMPOSITE_INDEXERS_ENABLED) { + List decompilers = ContainerUtil.newArrayList(ClassFileDecompilers.EP_NAME.getExtensions()); + decompilers.sort(CLASS_NAME_COMPARATOR); + for (ClassFileDecompilers.Decompiler decompiler : decompilers) { + if (decompiler instanceof Full) { + version = version * 31 + ((Full)decompiler).getStubBuilder().getStubVersion() + decompiler.getClass().getName().hashCode(); + } } } return version; } + + private static T setContentAndCompute(@NotNull FileContent content, @NotNull Supplier computation) { + try { + content.getFile().setPreloadedContentHint(content.getContent()); + return computation.get(); + } finally { + content.getFile().setPreloadedContentHint(null); + } + } } \ No newline at end of file diff --git a/platform/core-api/src/com/intellij/psi/stubs/BinaryFileStubBuilder.java b/platform/core-api/src/com/intellij/psi/stubs/BinaryFileStubBuilder.java index 367f20a7a389..cbc842c783a9 100644 --- a/platform/core-api/src/com/intellij/psi/stubs/BinaryFileStubBuilder.java +++ b/platform/core-api/src/com/intellij/psi/stubs/BinaryFileStubBuilder.java @@ -31,4 +31,21 @@ public interface BinaryFileStubBuilder { Stub buildStubTree(@NotNull FileContent fileContent); int getStubVersion(); + + interface CompositeBinaryFileStubBuilder extends BinaryFileStubBuilder { + @Nullable + SubBuilder getSubBuilder(@NotNull FileContent fileContent); + + @NotNull + String getSubBuilderVersion(@Nullable SubBuilder subBuilder); + + @Nullable + Stub buildStubTree(@NotNull FileContent fileContent, @Nullable SubBuilder builder); + + @Nullable + @Override + default Stub buildStubTree(@NotNull FileContent fileContent) { + return buildStubTree(fileContent, getSubBuilder(fileContent)); + } + } } \ No newline at end of file diff --git a/platform/core-impl/src/com/intellij/psi/stubs/StubBuilderType.java b/platform/core-impl/src/com/intellij/psi/stubs/StubBuilderType.java new file mode 100644 index 000000000000..c5ad25188ad7 --- /dev/null +++ b/platform/core-impl/src/com/intellij/psi/stubs/StubBuilderType.java @@ -0,0 +1,75 @@ +// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.psi.stubs; + +import com.intellij.diagnostic.PluginException; +import com.intellij.openapi.diagnostic.Logger; +import com.intellij.psi.templateLanguages.TemplateLanguage; +import com.intellij.psi.tree.IStubFileElementType; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.Objects; + +class StubBuilderType { + private static final Logger LOG = Logger.getInstance(StubBuilderType.class); + private final IStubFileElementType myElementType; + private final BinaryFileStubBuilder myBinaryFileStubBuilder; + private final Object myBinarySubBuilder; + + StubBuilderType(@NotNull IStubFileElementType elementType) { + myElementType = elementType; + myBinaryFileStubBuilder = null; + myBinarySubBuilder = null; + } + + StubBuilderType(@NotNull BinaryFileStubBuilder binaryFileStubBuilder) { + myElementType = null; + myBinaryFileStubBuilder = binaryFileStubBuilder; + myBinarySubBuilder = null; + } + + StubBuilderType(@NotNull BinaryFileStubBuilder.CompositeBinaryFileStubBuilder binaryFileStubBuilder, + @Nullable Object binarySubBuilder) { + myElementType = null; + myBinaryFileStubBuilder = binaryFileStubBuilder; + myBinarySubBuilder = binarySubBuilder; + } + + BinaryFileStubBuilder getBinaryFileStubBuilder() { + return myBinaryFileStubBuilder; + } + + String getVersion() { + if (myElementType != null) { + if (myElementType.getLanguage() instanceof TemplateLanguage && + myElementType.getStubVersion() < IStubFileElementType.getTemplateStubVersion()) { + PluginException.logPluginError(LOG, myElementType.getLanguage() + " stub version should call super.getStubVersion()", + null, myElementType.getClass()); + } + return myElementType.getClass().getName() + ":" + myElementType.getStubVersion(); + } else { + assert myBinaryFileStubBuilder != null; + String baseVersion = myBinaryFileStubBuilder.getClass().getName() + ":" + myBinaryFileStubBuilder.getStubVersion(); + if (myBinaryFileStubBuilder instanceof BinaryFileStubBuilder.CompositeBinaryFileStubBuilder) { + return baseVersion + ":" + ((BinaryFileStubBuilder.CompositeBinaryFileStubBuilder)myBinaryFileStubBuilder).getSubBuilderVersion(myBinarySubBuilder); + } else { + return baseVersion; + } + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + StubBuilderType type = (StubBuilderType)o; + return Objects.equals(myElementType, type.myElementType) && + Objects.equals(myBinaryFileStubBuilder, type.myBinaryFileStubBuilder) && + Objects.equals(myBinarySubBuilder, type.myBinarySubBuilder); + } + + @Override + public int hashCode() { + return Objects.hash(myElementType, myBinaryFileStubBuilder, myBinarySubBuilder); + } +} diff --git a/platform/core-impl/src/com/intellij/psi/stubs/StubTreeBuilder.java b/platform/core-impl/src/com/intellij/psi/stubs/StubTreeBuilder.java index 72298e17f326..c53c2628a67e 100644 --- a/platform/core-impl/src/com/intellij/psi/stubs/StubTreeBuilder.java +++ b/platform/core-impl/src/com/intellij/psi/stubs/StubTreeBuilder.java @@ -16,8 +16,11 @@ package com.intellij.psi.stubs; import com.intellij.lang.Language; +import com.intellij.lang.LanguageParserDefinitions; +import com.intellij.lang.ParserDefinition; import com.intellij.lang.TreeBackedLighterAST; import com.intellij.openapi.fileTypes.FileType; +import com.intellij.openapi.fileTypes.LanguageFileType; import com.intellij.openapi.util.Key; import com.intellij.openapi.util.Pair; import com.intellij.openapi.util.Trinity; @@ -27,10 +30,12 @@ import com.intellij.psi.PsiFile; import com.intellij.psi.StubBuilder; import com.intellij.psi.impl.source.PsiFileImpl; import com.intellij.psi.tree.IElementType; +import com.intellij.psi.tree.IFileElementType; import com.intellij.psi.tree.IStubFileElementType; import com.intellij.util.SmartList; import com.intellij.util.containers.ContainerUtil; import com.intellij.util.indexing.FileContent; +import com.intellij.util.indexing.IndexedFile; import com.intellij.util.indexing.IndexingDataKeys; import com.intellij.util.indexing.PsiDependentFileContent; import org.jetbrains.annotations.NotNull; @@ -44,8 +49,49 @@ public class StubTreeBuilder { private StubTreeBuilder() { } + static boolean requiresContentToFindBuilder(@NotNull FileType fileType) { + final BinaryFileStubBuilder builder = BinaryFileStubBuilders.INSTANCE.forFileType(fileType); + return builder instanceof BinaryFileStubBuilder.CompositeBinaryFileStubBuilder; + } + + static StubBuilderType getStubBuilderType(@NotNull IndexedFile file, boolean toBuild) { + FileType fileType = file.getFileType(); + final BinaryFileStubBuilder builder = BinaryFileStubBuilders.INSTANCE.forFileType(fileType); + if (builder != null) { + if (builder instanceof BinaryFileStubBuilder.CompositeBinaryFileStubBuilder) { + Object subBuilder = ((BinaryFileStubBuilder.CompositeBinaryFileStubBuilder)builder).getSubBuilder((FileContent) file); + return new StubBuilderType((BinaryFileStubBuilder.CompositeBinaryFileStubBuilder)builder, subBuilder); + } else { + return new StubBuilderType(builder); + } + } + + if (fileType instanceof LanguageFileType) { + final Language l = ((LanguageFileType)fileType).getLanguage(); + final ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(l); + if (parserDefinition == null) { + return null; + } + + final IFileElementType elementType = parserDefinition.getFileNodeType(); + if (!(elementType instanceof IStubFileElementType)) return null; + boolean shouldBuildStubFor = ((IStubFileElementType)elementType).shouldBuildStubFor(file.getFile()); + if (toBuild && !shouldBuildStubFor) return null; + return new StubBuilderType((IStubFileElementType)elementType); + } + + return null; + } + @Nullable - public static Stub buildStubTree(final FileContent inputData) { + public static Stub buildStubTree(@NotNull FileContent inputData) { + StubBuilderType type = getStubBuilderType(inputData, false); + if (type == null) return null; + return buildStubTree(inputData, type); + } + + @Nullable + public static Stub buildStubTree(@NotNull FileContent inputData, @NotNull StubBuilderType stubBuilderType) { Stub data = inputData.getUserData(stubElementKey); if (data != null) return data; @@ -54,9 +100,7 @@ public class StubTreeBuilder { data = inputData.getUserData(stubElementKey); if (data != null) return data; - final FileType fileType = inputData.getFileType(); - - final BinaryFileStubBuilder builder = BinaryFileStubBuilders.INSTANCE.forFileType(fileType); + final BinaryFileStubBuilder builder = stubBuilderType.getBinaryFileStubBuilder(); if (builder != null) { data = builder.buildStubTree(inputData); if (data instanceof PsiFileStubImpl && !((PsiFileStubImpl)data).rootsAreSet()) { diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java index 1c8912f9230d..79564493bbbf 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java @@ -24,9 +24,7 @@ import com.intellij.util.indexing.impl.DebugAssertions; import com.intellij.util.indexing.impl.IndexStorage; import com.intellij.util.indexing.impl.InputDataDiffBuilder; import com.intellij.util.indexing.impl.forward.EmptyForwardIndex; -import com.intellij.util.io.DataExternalizer; -import com.intellij.util.io.DataInputOutputUtil; -import com.intellij.util.io.PersistentHashMapValueStorage; +import com.intellij.util.io.*; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -41,7 +39,7 @@ import java.util.Map; public class StubUpdatingIndex extends SingleEntryFileBasedIndexExtension implements CustomImplementationFileBasedIndexExtension { static final Logger LOG = Logger.getInstance(StubUpdatingIndex.class); - private static final int VERSION = 43 + (PersistentHashMapValueStorage.COMPRESSION_ENABLED ? 1 : 0); + private static final int VERSION = 43 + (PersistentHashMapValueStorage.COMPRESSION_ENABLED ? 1 : 0) + (InvertedIndex.ARE_COMPOSITE_INDEXERS_ENABLED ? 1 : 0); // todo remove once we don't need this for stub-ast mismatch debug info private static final FileAttribute INDEXED_STAMP = new FileAttribute("stubIndexStamp", 3, true); @@ -86,10 +84,33 @@ public class StubUpdatingIndex extends SingleEntryFileBasedIndexExtension getIndexer() { - return new SingleEntryIndexer(false) { + return new SingleEntryCompositeIndexer(false) { + @Override + public boolean requiresContentForSubIndexerEvaluation(@NotNull IndexedFile file) { + return StubTreeBuilder.requiresContentToFindBuilder(file.getFileType()); + } + + @Nullable + @Override + public StubBuilderType calculateSubIndexer(@NotNull IndexedFile file) { + return StubTreeBuilder.getStubBuilderType(file, true); + } + + @NotNull + @Override + public String getSubIndexerVersion(@NotNull StubBuilderType type) { + return type.getVersion(); + } + + @NotNull + @Override + public KeyDescriptor getSubIndexerVersionDescriptor() { + return EnumeratorStringDescriptor.INSTANCE; + } + @Override @Nullable - public SerializedStubTree computeValue(@NotNull final FileContent inputData) { + public SerializedStubTree computeValue(@NotNull final FileContent inputData, @NotNull StubBuilderType type) { return ReadAction.compute(() -> { SerializedStubTree serializedStubTree = null; @@ -109,7 +130,7 @@ public class StubUpdatingIndex extends SingleEntryFileBasedIndexExtension { private StubIndexImpl myStubIndex; - private final StubVersionMap myStubVersionMap = new StubVersionMap(); + @Nullable + private final StubVersionMap myStubVersionMap = InvertedIndex.ARE_COMPOSITE_INDEXERS_ENABLED ? null : new StubVersionMap(); MyIndex(@NotNull FileBasedIndexExtension extension, @NotNull IndexStorage storage) throws StorageException, IOException { super(extension, storage, new EmptyForwardIndex(), new StubUpdatingForwardIndexAccessor(), null, null); ((StubUpdatingForwardIndexAccessor)getForwardIndexAccessor()).setIndex(this); checkNameStorage(); + + if (InvertedIndex.ARE_COMPOSITE_INDEXERS_ENABLED) { + // load stub serializers before usage + for (Language language : Language.getRegisteredLanguages()) { + ParserDefinition definition = LanguageParserDefinitions.INSTANCE.forLanguage(language); + if (definition != null) { + definition.getFileNodeType(); + } + } + } + } @Override @@ -334,7 +367,9 @@ public class StubUpdatingIndex extends SingleEntryFileBasedIndexExtension