diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/PrebuiltStubs.kt b/platform/lang-impl/src/com/intellij/psi/stubs/PrebuiltStubs.kt index 7b54e70b4f6b..e5d59433eccb 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/PrebuiltStubs.kt +++ b/platform/lang-impl/src/com/intellij/psi/stubs/PrebuiltStubs.kt @@ -9,13 +9,17 @@ import com.intellij.openapi.diagnostic.Logger import com.intellij.openapi.fileTypes.FileTypeExtension import com.intellij.openapi.util.Disposer import com.intellij.openapi.util.text.StringUtilRt +import com.intellij.psi.PsiElement import com.intellij.util.indexing.FileContent +import com.intellij.util.indexing.ID import com.intellij.util.io.* import org.jetbrains.annotations.ApiStatus import org.jetbrains.annotations.TestOnly import java.io.DataInput import java.io.DataOutput import java.io.File +import java.io.IOException +import java.util.function.UnaryOperator const val EP_NAME = "com.intellij.filetype.prebuiltStubsProvider" @@ -23,7 +27,7 @@ object PrebuiltStubsProviders : FileTypeExtension(EP_NAME @ApiStatus.Experimental interface PrebuiltStubsProvider { - fun findStub(fileContent: FileContent): Stub? + fun findStub(fileContent: FileContent): SerializedStubTree? } class FileContentHashing { @@ -57,23 +61,55 @@ open class HashCodeExternalizers : DataExternalizer { } } -class StubTreeExternalizer : DataExternalizer { +class FullStubExternalizer : DataExternalizer { + private val stubForwardIndexExternalizer = FileLocalStubForwardIndexExternalizer() + override fun save(out: DataOutput, value: SerializedStubTree) { value.write(out) + stubForwardIndexExternalizer.save(out, value.indexedStubs) } - override fun read(`in`: DataInput): SerializedStubTree = SerializedStubTree(`in`) + override fun read(`in`: DataInput): SerializedStubTree { + val tree = SerializedStubTree(`in`) + tree.indexedStubs = stubForwardIndexExternalizer.read(`in`) + return tree + } +} + +private class FileLocalStubForwardIndexExternalizer : StubForwardIndexExternalizer() { + override fun createStubIndexKeySerializationState(out: DataOutput, + set: MutableSet>): FileLocalStringEnumerator { + val enumerator = FileLocalStringEnumerator(true) + set.map { it.name }.forEach { enumerator.enumerate(it)} + enumerator.write(out) + return enumerator + } + + override fun writeStubIndexKey(out: DataOutput, key: StubIndexKey<*, *>, state: FileLocalStringEnumerator?) { + DataInputOutputUtil.writeINT(out, state!!.enumerate(key.name)) + } + + override fun createStubIndexKeySerializationState(input: DataInput, stubIndexKeyCount: Int): FileLocalStringEnumerator { + val enumerator = FileLocalStringEnumerator(false) + FileLocalStringEnumerator.readEnumeratedStrings(enumerator, input, UnaryOperator.identity()) + return enumerator + } + + override fun readStubIndexKey(input: DataInput, stubKeySerializationState: FileLocalStringEnumerator?): ID<*, *> { + return ID.findByName(stubKeySerializationState!!.valueOf(DataInputOutputUtil.readINT(input))!!)!! + } } abstract class PrebuiltStubsProviderBase : PrebuiltIndexProviderBase(), PrebuiltStubsProvider { private var mySerializationManager: SerializationManagerImpl? = null + private val myIdeSerializationManager = SerializationManager.getInstance() as SerializationManagerImpl protected abstract val stubVersion: Int override val indexName: String get() = SDK_STUBS_STORAGE_NAME - override val indexExternalizer: StubTreeExternalizer get() = StubTreeExternalizer() + override val indexExternalizer: FullStubExternalizer get() = FullStubExternalizer() companion object { const val PREBUILT_INDICES_PATH_PROPERTY: String = "prebuilt_indices_path" @@ -94,22 +130,18 @@ abstract class PrebuiltStubsProviderBase : PrebuiltIndexProviderBase) { - stub.psi = fileContent.psiFile - } - return stub + return null } } diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/SerializedStubTree.java b/platform/lang-impl/src/com/intellij/psi/stubs/SerializedStubTree.java index 5ad3e4d1a5c0..0e37c43a865d 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/SerializedStubTree.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/SerializedStubTree.java @@ -83,7 +83,9 @@ public class SerializedStubTree { @NotNull SerializationManagerImpl newSerializationManager) throws IOException { BufferExposingByteArrayOutputStream outStub = new BufferExposingByteArrayOutputStream(); currentSerializationManager.reSerialize(new ByteArrayInputStream(myBytes, 0, myLength), outStub, newSerializationManager); - return new SerializedStubTree(outStub.getInternalBuffer(), outStub.size(), null); + SerializedStubTree reSerialized = new SerializedStubTree(outStub.getInternalBuffer(), outStub.size(), null); + reSerialized.setIndexedStubs(getIndexedStubs()); + return reSerialized; } // willIndexStub is one time optimization hint, once can safely pass false @@ -104,23 +106,9 @@ public class SerializedStubTree { return serializationManager.deserialize(new UnsyncByteArrayInputStream(myBytes)); } - void indexTree() throws SerializerNotFoundException { + public void indexTree() throws SerializerNotFoundException { ObjectStubBase root = (ObjectStubBase)getStub(true); - ObjectStubTree objectStubTree = root instanceof PsiFileStub ? new StubTree((PsiFileStub)root, false) : - new ObjectStubTree(root, false); - Map> map = objectStubTree.indexStubTree(); - - // xxx:fix refs inplace - for (StubIndexKey key : map.keySet()) { - Map value = map.get(key); - for (Object k : value.keySet()) { - int[] ints = value.get(k); - StubIdList stubList = ints.length == 1 ? new StubIdList(ints[0]) : new StubIdList(ints, ints.length); - ((Map)(Map)value).put(k, stubList); - } - } - - myIndexedStubs = new IndexedStubs(calculateHash(myBytes, myLength), (Map)map); + myIndexedStubs = new IndexedStubs(calculateHash(myBytes, myLength), indexTree(root)); } @NotNull @@ -128,6 +116,10 @@ public class SerializedStubTree { return myIndexedStubs; } + void setIndexedStubs(@NotNull IndexedStubs indexedStubs) { + myIndexedStubs = indexedStubs; + } + public boolean equals(final Object that) { if (this == that) { return true; @@ -179,6 +171,24 @@ public class SerializedStubTree { return deserialized + "\n bytes: " + toHexString(myBytes, myLength); } + @NotNull + static Map> indexTree(@NotNull Stub root) { + ObjectStubTree objectStubTree = root instanceof PsiFileStub ? new StubTree((PsiFileStub)root, false) : + new ObjectStubTree((ObjectStubBase)root, false); + Map> map = objectStubTree.indexStubTree(); + + // xxx:fix refs inplace + for (StubIndexKey key : map.keySet()) { + Map value = map.get(key); + for (Object k : value.keySet()) { + int[] ints = value.get(k); + StubIdList stubList = ints.length == 1 ? new StubIdList(ints[0]) : new StubIdList(ints, ints.length); + ((Map)(Map)value).put(k, stubList); + } + } + return (Map>)(Map)map; + } + @NotNull private static byte[] calculateHash(@NotNull byte[] content, int length) { MessageDigest digest = HASHER.getValue(); diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java index 8c121100eecf..98c27f78895a 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java @@ -110,27 +110,51 @@ public class StubUpdatingIndex extends SingleEntryFileBasedIndexExtension { - Stub rootStub = null; + SerializedStubTree serializedStubTree = null; - if (Registry.is("use.prebuilt.indices")) { - final PrebuiltStubsProvider prebuiltStubsProvider = - PrebuiltStubsProviders.INSTANCE.forFileType(inputData.getFileType()); - if (prebuiltStubsProvider != null) { - rootStub = prebuiltStubsProvider.findStub(inputData); - if (PrebuiltIndexProviderBase.DEBUG_PREBUILT_INDICES) { - Stub stub = StubTreeBuilder.buildStubTree(inputData); - if (rootStub != null && stub != null) { - check(rootStub, stub); + try { + if (Registry.is("use.prebuilt.indices")) { + final PrebuiltStubsProvider prebuiltStubsProvider = + PrebuiltStubsProviders.INSTANCE.forFileType(inputData.getFileType()); + if (prebuiltStubsProvider != null) { + serializedStubTree = prebuiltStubsProvider.findStub(inputData); + if (PrebuiltIndexProviderBase.DEBUG_PREBUILT_INDICES) { + Stub stub = StubTreeBuilder.buildStubTree(inputData); + if (serializedStubTree != null && stub != null) { + check(serializedStubTree.getStub(false), stub); + checkStubIndexes(serializedStubTree, stub); + } + } + } + } + + if (serializedStubTree == null) { + Stub rootStub; + rootStub = StubTreeBuilder.buildStubTree(inputData); + if (rootStub != null) { + final BufferExposingByteArrayOutputStream bytes = new BufferExposingByteArrayOutputStream(); + SerializationManagerEx.getInstanceEx().serialize(rootStub, bytes); + serializedStubTree = new SerializedStubTree(bytes.getInternalBuffer(), bytes.size(), rootStub); + serializedStubTree.indexTree(); + if (DebugAssertions.DEBUG) { + + Stub deserialized = SerializationManagerEx.getInstanceEx().deserialize(bytes.toInputStream()); + check(deserialized, rootStub); } } } } - - if (rootStub == null) { - rootStub = StubTreeBuilder.buildStubTree(inputData); + catch (ProcessCanceledException pce) { + throw pce; + } + catch (SerializerNotFoundException e) { + throw new RuntimeException(e); + } + catch (Throwable t) { + LOG.error("Error indexing:" + inputData.getFile(), t); } - if (rootStub == null) return null; + if (serializedStubTree == null) return null; VirtualFile file = inputData.getFile(); boolean isBinary = file.getFileType().isBinary(); @@ -138,39 +162,20 @@ public class StubUpdatingIndex extends SingleEntryFileBasedIndexExtension> calculatedStubIndexes = SerializedStubTree.indexTree(calculatedStub); + assert calculatedStubIndexes.equals(prebuiltSerializedTree.getIndexedStubs().getStubIndicesValueMap()); + } + private static void check(@NotNull Stub stub, @NotNull Stub stub2) { assert stub.getStubType() == stub2.getStubType(); List stubs = stub.getChildrenStubs(); diff --git a/tools/index-tools/src/org/jetbrains/index/stubs/StubsGenerator.kt b/tools/index-tools/src/org/jetbrains/index/stubs/StubsGenerator.kt index ddb70d31952f..6e9cc9213174 100644 --- a/tools/index-tools/src/org/jetbrains/index/stubs/StubsGenerator.kt +++ b/tools/index-tools/src/org/jetbrains/index/stubs/StubsGenerator.kt @@ -54,12 +54,14 @@ open class StubsGenerator(private val stubsVersion: String, private val stubsSto val bytes = BufferExposingByteArrayOutputStream() serializationManager.serialize(stub, bytes) - return SerializedStubTree(bytes.internalBuffer, bytes.size(), stub) + val tree = SerializedStubTree(bytes.internalBuffer, bytes.size(), stub) + tree.indexTree() + return tree } override fun createStorage(stubsStorageFilePath: String): PersistentHashMap { return PersistentHashMap(File("$stubsStorageFilePath.input"), - HashCodeDescriptor.instance, StubTreeExternalizer()) + HashCodeDescriptor.instance, FullStubExternalizer()) } open fun buildStubForFile(fileContent: FileContentImpl, @@ -79,7 +81,7 @@ fun mergeStubs(paths: List, stubsFilePath: String, stubsFileName: String // we don't need a project here, but I didn't find a better way to wait until indices and components are initialized try { - val stubExternalizer = StubTreeExternalizer() + val stubExternalizer = FullStubExternalizer() val storageFile = File(stubsFilePath, "$stubsFileName.input") if (storageFile.exists()) {