MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

prebuilt stubs: use partial stub re-serialization; use prebuilt stub index maps

Browse Source 
GitOrigin-RevId: 57bf67005201101281e21698cdd32407a845c5c7
This commit is contained in:
Dmitry Batkovich
2019-06-16 07:57:36 +03:00
committed by intellij-monorepo-bot
parent cf31735622
commit 5d862192f0
4 changed files with 121 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/lang-impl/src/com/intellij/psi/stubs/PrebuiltStubs.kt
+46 -14
View File
@@ -9,13 +9,17 @@ import com.intellij.openapi.diagnostic.Logger
import com.intellij.openapi.fileTypes.FileTypeExtension
import com.intellij.openapi.util.Disposer
import com.intellij.openapi.util.text.StringUtilRt
import com.intellij.psi.PsiElement
import com.intellij.util.indexing.FileContent
import com.intellij.util.indexing.ID
import com.intellij.util.io.*
import org.jetbrains.annotations.ApiStatus
import org.jetbrains.annotations.TestOnly
import java.io.DataInput
import java.io.DataOutput
import java.io.File
import java.io.IOException
import java.util.function.UnaryOperator
const val EP_NAME = "com.intellij.filetype.prebuiltStubsProvider"
@@ -23,7 +27,7 @@ object PrebuiltStubsProviders : FileTypeExtension<PrebuiltStubsProvider>(EP_NAME
@ApiStatus.Experimental
interface PrebuiltStubsProvider {
fun findStub(fileContent: FileContent): Stub?
fun findStub(fileContent: FileContent): SerializedStubTree?
}
class FileContentHashing {
@@ -57,23 +61,55 @@ open class HashCodeExternalizers : DataExternalizer<HashCode> {
}
}
class StubTreeExternalizer : DataExternalizer<SerializedStubTree> {
class FullStubExternalizer : DataExternalizer<SerializedStubTree> {
private val stubForwardIndexExternalizer = FileLocalStubForwardIndexExternalizer()
override fun save(out: DataOutput, value: SerializedStubTree) {
value.write(out)
stubForwardIndexExternalizer.save(out, value.indexedStubs)
}
override fun read(`in`: DataInput): SerializedStubTree = SerializedStubTree(`in`)
override fun read(`in`: DataInput): SerializedStubTree {
val tree = SerializedStubTree(`in`)
tree.indexedStubs = stubForwardIndexExternalizer.read(`in`)
return tree
}
}
private class FileLocalStubForwardIndexExternalizer : StubForwardIndexExternalizer<FileLocalStringEnumerator>() {
override fun createStubIndexKeySerializationState(out: DataOutput,
set: MutableSet<StubIndexKey<Any, PsiElement>>): FileLocalStringEnumerator {
val enumerator = FileLocalStringEnumerator(true)
set.map { it.name }.forEach { enumerator.enumerate(it)}
enumerator.write(out)
return enumerator
}
override fun writeStubIndexKey(out: DataOutput, key: StubIndexKey<*, *>, state: FileLocalStringEnumerator?) {
DataInputOutputUtil.writeINT(out, state!!.enumerate(key.name))
}
override fun createStubIndexKeySerializationState(input: DataInput, stubIndexKeyCount: Int): FileLocalStringEnumerator {
val enumerator = FileLocalStringEnumerator(false)
FileLocalStringEnumerator.readEnumeratedStrings(enumerator, input, UnaryOperator.identity())
return enumerator
}
override fun readStubIndexKey(input: DataInput, stubKeySerializationState: FileLocalStringEnumerator?): ID<*, *> {
return ID.findByName<Any, Any>(stubKeySerializationState!!.valueOf(DataInputOutputUtil.readINT(input))!!)!!
}
}
abstract class PrebuiltStubsProviderBase : PrebuiltIndexProviderBase<SerializedStubTree>(), PrebuiltStubsProvider {
private var mySerializationManager: SerializationManagerImpl? = null
private val myIdeSerializationManager = SerializationManager.getInstance() as SerializationManagerImpl
protected abstract val stubVersion: Int
override val indexName: String get() = SDK_STUBS_STORAGE_NAME
override val indexExternalizer: StubTreeExternalizer get() = StubTreeExternalizer()
override val indexExternalizer: FullStubExternalizer get() = FullStubExternalizer()
companion object {
const val PREBUILT_INDICES_PATH_PROPERTY: String = "prebuilt_indices_path"
@@ -94,22 +130,18 @@ abstract class PrebuiltStubsProviderBase : PrebuiltIndexProviderBase<SerializedS
}
}
override fun findStub(fileContent: FileContent): Stub? {
var stub: Stub? = null
override fun findStub(fileContent: FileContent): SerializedStubTree? {
try {
val stubTree = get(fileContent)
if (stubTree != null) {
stub = stubTree.getStub(false, mySerializationManager!!)
return stubTree.reSerialize(mySerializationManager!!, myIdeSerializationManager)
}
}
catch (e: SerializerNotFoundException) {
LOG.error("Can't deserialize stub tree", e)
catch (e: IOException) {
LOG.error("Can't re-serialize stub tree", e)
}
if (stub is PsiFileStubImpl<*>) {
stub.psi = fileContent.psiFile
}
return stub
return null
}
}
platform/lang-impl/src/com/intellij/psi/stubs/SerializedStubTree.java
+27 -17
View File
@@ -83,7 +83,9 @@ public class SerializedStubTree {
@NotNull SerializationManagerImpl newSerializationManager) throws IOException {
BufferExposingByteArrayOutputStream outStub = new BufferExposingByteArrayOutputStream();
currentSerializationManager.reSerialize(new ByteArrayInputStream(myBytes, 0, myLength), outStub, newSerializationManager);
return new SerializedStubTree(outStub.getInternalBuffer(), outStub.size(), null);
SerializedStubTree reSerialized = new SerializedStubTree(outStub.getInternalBuffer(), outStub.size(), null);
reSerialized.setIndexedStubs(getIndexedStubs());
return reSerialized;
}
// willIndexStub is one time optimization hint, once can safely pass false
@@ -104,23 +106,9 @@ public class SerializedStubTree {
return serializationManager.deserialize(new UnsyncByteArrayInputStream(myBytes));
}
void indexTree() throws SerializerNotFoundException {
public void indexTree() throws SerializerNotFoundException {
ObjectStubBase root = (ObjectStubBase)getStub(true);
ObjectStubTree objectStubTree = root instanceof PsiFileStub ? new StubTree((PsiFileStub)root, false) :
new ObjectStubTree(root, false);
Map<StubIndexKey, Map<Object, int[]>> map = objectStubTree.indexStubTree();
// xxx:fix refs inplace
for (StubIndexKey key : map.keySet()) {
Map<Object, int[]> value = map.get(key);
for (Object k : value.keySet()) {
int[] ints = value.get(k);
StubIdList stubList = ints.length == 1 ? new StubIdList(ints[0]) : new StubIdList(ints, ints.length);
((Map<Object, StubIdList>)(Map)value).put(k, stubList);
}
}
myIndexedStubs = new IndexedStubs(calculateHash(myBytes, myLength), (Map)map);
myIndexedStubs = new IndexedStubs(calculateHash(myBytes, myLength), indexTree(root));
}
@NotNull
@@ -128,6 +116,10 @@ public class SerializedStubTree {
return myIndexedStubs;
}
void setIndexedStubs(@NotNull IndexedStubs indexedStubs) {
myIndexedStubs = indexedStubs;
}
public boolean equals(final Object that) {
if (this == that) {
return true;
@@ -179,6 +171,24 @@ public class SerializedStubTree {
return deserialized + "\n bytes: " + toHexString(myBytes, myLength);
}
@NotNull
static Map<StubIndexKey, Map<Object, StubIdList>> indexTree(@NotNull Stub root) {
ObjectStubTree objectStubTree = root instanceof PsiFileStub ? new StubTree((PsiFileStub)root, false) :
new ObjectStubTree((ObjectStubBase)root, false);
Map<StubIndexKey, Map<Object, int[]>> map = objectStubTree.indexStubTree();
// xxx:fix refs inplace
for (StubIndexKey key : map.keySet()) {
Map<Object, int[]> value = map.get(key);
for (Object k : value.keySet()) {
int[] ints = value.get(k);
StubIdList stubList = ints.length == 1 ? new StubIdList(ints[0]) : new StubIdList(ints, ints.length);
((Map<Object, StubIdList>)(Map)value).put(k, stubList);
}
}
return (Map<StubIndexKey, Map<Object, StubIdList>>)(Map)map;
}
@NotNull
private static byte[] calculateHash(@NotNull byte[] content, int length) {
MessageDigest digest = HASHER.getValue();
platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java
+43 -38
View File
@@ -110,27 +110,51 @@ public class StubUpdatingIndex extends SingleEntryFileBasedIndexExtension<Serial
@Nullable
public SerializedStubTree computeValue(@NotNull final FileContent inputData) {
return ReadAction.compute(() -> {
Stub rootStub = null;
SerializedStubTree serializedStubTree = null;
if (Registry.is("use.prebuilt.indices")) {
final PrebuiltStubsProvider prebuiltStubsProvider =
PrebuiltStubsProviders.INSTANCE.forFileType(inputData.getFileType());
if (prebuiltStubsProvider != null) {
rootStub = prebuiltStubsProvider.findStub(inputData);
if (PrebuiltIndexProviderBase.DEBUG_PREBUILT_INDICES) {
Stub stub = StubTreeBuilder.buildStubTree(inputData);
if (rootStub != null && stub != null) {
check(rootStub, stub);
try {
if (Registry.is("use.prebuilt.indices")) {
final PrebuiltStubsProvider prebuiltStubsProvider =
PrebuiltStubsProviders.INSTANCE.forFileType(inputData.getFileType());
if (prebuiltStubsProvider != null) {
serializedStubTree = prebuiltStubsProvider.findStub(inputData);
if (PrebuiltIndexProviderBase.DEBUG_PREBUILT_INDICES) {
Stub stub = StubTreeBuilder.buildStubTree(inputData);
if (serializedStubTree != null && stub != null) {
check(serializedStubTree.getStub(false), stub);
checkStubIndexes(serializedStubTree, stub);
}
}
}
}
if (serializedStubTree == null) {
Stub rootStub;
rootStub = StubTreeBuilder.buildStubTree(inputData);
if (rootStub != null) {
final BufferExposingByteArrayOutputStream bytes = new BufferExposingByteArrayOutputStream();
SerializationManagerEx.getInstanceEx().serialize(rootStub, bytes);
serializedStubTree = new SerializedStubTree(bytes.getInternalBuffer(), bytes.size(), rootStub);
serializedStubTree.indexTree();
if (DebugAssertions.DEBUG) {
Stub deserialized = SerializationManagerEx.getInstanceEx().deserialize(bytes.toInputStream());
check(deserialized, rootStub);
}
}
}
}
if (rootStub == null) {
rootStub = StubTreeBuilder.buildStubTree(inputData);
catch (ProcessCanceledException pce) {
throw pce;
}
catch (SerializerNotFoundException e) {
throw new RuntimeException(e);
}
catch (Throwable t) {
LOG.error("Error indexing:" + inputData.getFile(), t);
}
if (rootStub == null) return null;
if (serializedStubTree == null) return null;
VirtualFile file = inputData.getFile();
boolean isBinary = file.getFileType().isBinary();
@@ -138,39 +162,20 @@ public class StubUpdatingIndex extends SingleEntryFileBasedIndexExtension<Serial
long byteLength = file.getLength();
rememberIndexingStamp(file, isBinary, byteLength, contentLength);
final BufferExposingByteArrayOutputStream bytes = new BufferExposingByteArrayOutputStream();
SerializationManagerEx.getInstanceEx().serialize(rootStub, bytes);
if (DebugAssertions.DEBUG) {
try {
Stub deserialized =
SerializationManagerEx.getInstanceEx().deserialize(bytes.toInputStream());
check(deserialized, rootStub);
}
catch (ProcessCanceledException pce) {
throw pce;
}
catch (Throwable t) {
LOG.error("Error indexing:" + file, t);
}
}
SerializedStubTree serializedStubTree =
new SerializedStubTree(bytes.getInternalBuffer(), bytes.size(), rootStub);
if (LOG.isDebugEnabled()) {
LOG.debug("Indexing " + file + "; " + IndexingStampInfo.dumpSize(byteLength, contentLength));
}
try {
serializedStubTree.indexTree();
}
catch (SerializerNotFoundException ex) {
throw new RuntimeException(ex);
}
return serializedStubTree;
});
}
};
}
private static void checkStubIndexes(@NotNull SerializedStubTree prebuiltSerializedTree, @NotNull Stub calculatedStub) {
Map<StubIndexKey, Map<Object, StubIdList>> calculatedStubIndexes = SerializedStubTree.indexTree(calculatedStub);
assert calculatedStubIndexes.equals(prebuiltSerializedTree.getIndexedStubs().getStubIndicesValueMap());
}
private static void check(@NotNull Stub stub, @NotNull Stub stub2) {
assert stub.getStubType() == stub2.getStubType();
List<? extends Stub> stubs = stub.getChildrenStubs();
tools/index-tools/src/org/jetbrains/index/stubs/StubsGenerator.kt
+5 -3
View File
@@ -54,12 +54,14 @@ open class StubsGenerator(private val stubsVersion: String, private val stubsSto
val bytes = BufferExposingByteArrayOutputStream()
serializationManager.serialize(stub, bytes)
return SerializedStubTree(bytes.internalBuffer, bytes.size(), stub)
val tree = SerializedStubTree(bytes.internalBuffer, bytes.size(), stub)
tree.indexTree()
return tree
}
override fun createStorage(stubsStorageFilePath: String): PersistentHashMap<HashCode, SerializedStubTree> {
return PersistentHashMap(File("$stubsStorageFilePath.input"),
HashCodeDescriptor.instance, StubTreeExternalizer())
HashCodeDescriptor.instance, FullStubExternalizer())
}
open fun buildStubForFile(fileContent: FileContentImpl,
@@ -79,7 +81,7 @@ fun mergeStubs(paths: List<String>, stubsFilePath: String, stubsFileName: String
// we don't need a project here, but I didn't find a better way to wait until indices and components are initialized
try {
val stubExternalizer = StubTreeExternalizer()
val stubExternalizer = FullStubExternalizer()
val storageFile = File(stubsFilePath, "$stubsFileName.input")
if (storageFile.exists()) {