MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-01-06 03:21:12 +07:00
Code Issues Packages Projects Releases Wiki Activity

StubUpdatingIndex: explicitly specify SerializationManager to be used and initialized.

Browse Source 
GitOrigin-RevId: d6855dccfb0dcd1ee158d7b4f510ebb954589eb8
This commit is contained in:
Sergey Patrikeev
2020-01-31 10:15:52 +03:00
committed by intellij-monorepo-bot
parent 4921b332cf
commit ec471a6ab2
6 changed files with 75 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
platform/indexing-impl/src/com/intellij/psi/stubs/PrebuiltStubs.kt
View File

@@ -60,13 +60,13 @@ open class HashCodeExternalizers : DataExternalizer<HashCode> {
}
}
class FullStubExternalizer : SerializedStubTreeDataExternalizer(true, SerializationManagerEx.getInstanceEx(), StubForwardIndexExternalizer.FileLocalStubForwardIndexExternalizer.INSTANCE)
private val IDE_SERIALIZATION_MANAGER = SerializationManagerEx.getInstanceEx()
class FullStubExternalizer : SerializedStubTreeDataExternalizer(true, IDE_SERIALIZATION_MANAGER, StubForwardIndexExternalizer.createFileLocalExternalizer(IDE_SERIALIZATION_MANAGER))
abstract class PrebuiltStubsProviderBase : PrebuiltIndexProvider<SerializedStubTree>(), PrebuiltStubsProvider {
private var mySerializationManager: SerializationManagerImpl? = null
private val myIdeSerializationManager = SerializationManager.getInstance() as SerializationManagerImpl
private val myIndexedStubsSerializer = StubForwardIndexExternalizer.FileLocalStubForwardIndexExternalizer.INSTANCE
protected abstract val stubVersion: Int
@@ -114,7 +114,10 @@ abstract class PrebuiltStubsProviderBase : PrebuiltIndexProvider<SerializedStubT
try {
val stubTree = get(fileContent)
if (stubTree != null) {
return stubTree.reSerialize(mySerializationManager!!, myIdeSerializationManager, myIndexedStubsSerializer, SerializedStubTree.IDE_USED_EXTERNALIZER)
val newSerializationManager = IDE_SERIALIZATION_MANAGER
val newForwardIndexSerializer = StubForwardIndexExternalizer.getIdeUsedExternalizer(newSerializationManager)
val indexedStubsSerializer = StubForwardIndexExternalizer.createFileLocalExternalizer(mySerializationManager!!)
return stubTree.reSerialize(mySerializationManager!!, newSerializationManager, indexedStubsSerializer, newForwardIndexSerializer)
}
}
catch (e: IOException) {

7
platform/indexing-impl/src/com/intellij/psi/stubs/SerializedStubTree.java
View File

@@ -20,9 +20,6 @@ import java.security.MessageDigest;
import java.util.Map;
public class SerializedStubTree {
static final StubForwardIndexExternalizer<?> IDE_USED_EXTERNALIZER = System.getProperty("idea.uses.shareable.serialized.stubs") == null
? new StubForwardIndexExternalizer.IdeStubForwardIndexesExternalizer()
: new StubForwardIndexExternalizer.FileLocalStubForwardIndexExternalizer();
private static final MessageDigest HASHER = DigestUtil.sha256();
@@ -86,8 +83,8 @@ public class SerializedStubTree {
}
@NotNull
public SerializedStubTree reSerialize(@NotNull SerializationManagerImpl currentSerializationManager,
@NotNull SerializationManagerImpl newSerializationManager,
public SerializedStubTree reSerialize(@NotNull SerializationManagerEx currentSerializationManager,
@NotNull SerializationManagerEx newSerializationManager,
@NotNull StubForwardIndexExternalizer currentForwardIndexSerializer,
@NotNull StubForwardIndexExternalizer newForwardIndexSerializer) throws IOException {
BufferExposingByteArrayOutputStream outStub = new BufferExposingByteArrayOutputStream();

48
platform/indexing-impl/src/com/intellij/psi/stubs/StubForwardIndexExternalizer.java
View File

@@ -18,7 +18,34 @@ import java.util.Set;
import java.util.function.UnaryOperator;
public abstract class StubForwardIndexExternalizer<StubKeySerializationState> implements DataExternalizer<Map<StubIndexKey, Map<Object, StubIdList>>> {
private volatile boolean myEnsuredStubElementTypesLoaded;
private final SerializationManagerEx myManagerToInitialize;
protected StubForwardIndexExternalizer(SerializationManagerEx managerToInitialize) {
myManagerToInitialize = managerToInitialize;
}
@NotNull
public static StubForwardIndexExternalizer<?> getIdeUsedExternalizer(@NotNull SerializationManagerEx managerToInitialize) {
if (System.getProperty("idea.uses.shareable.serialized.stubs") == null) {
return new StubForwardIndexExternalizer.IdeStubForwardIndexesExternalizer(managerToInitialize);
}
return new FileLocalStubForwardIndexExternalizer(managerToInitialize);
}
@NotNull
public static StubForwardIndexExternalizer<?> createFileLocalExternalizer(@NotNull SerializationManagerEx serializationManager) {
return new FileLocalStubForwardIndexExternalizer(serializationManager);
}
private static void initializeSerializationManager(@NotNull SerializationManagerEx serializationManager) {
//TODO: consider initializing the serialization manager in another place.
ProgressManager.getInstance().executeNonCancelableSection(() -> {
serializationManager.initSerializers();
StubIndexEx.initExtensions();
});
((StubIndexEx)StubIndex.getInstance()).ensureLoaded();
}
protected abstract StubKeySerializationState createStubIndexKeySerializationState(@NotNull DataOutput out, @NotNull Set<StubIndexKey> set) throws IOException;
@@ -50,14 +77,7 @@ public abstract class StubForwardIndexExternalizer<StubKeySerializationState> im
}
<K> Map<StubIndexKey, Map<Object, StubIdList>> doRead(@NotNull DataInput in, @Nullable StubIndexKey<K, ?> requestedIndex, @Nullable K requestedKey) throws IOException {
if (!myEnsuredStubElementTypesLoaded) {
ProgressManager.getInstance().executeNonCancelableSection(() -> {
SerializationManager.getInstance().initSerializers();
StubIndexEx.initExtensions();
});
((StubIndexEx)StubIndex.getInstance()).ensureLoaded();
myEnsuredStubElementTypesLoaded = true;
}
initializeSerializationManager(myManagerToInitialize);
int stubIndicesValueMapSize = DataInputOutputUtil.readINT(in);
if (stubIndicesValueMapSize > 0) {
THashMap<StubIndexKey, Map<Object, StubIdList>> stubIndicesValueMap = requestedIndex != null ? null : new THashMap<>(stubIndicesValueMapSize);
@@ -84,7 +104,11 @@ public abstract class StubForwardIndexExternalizer<StubKeySerializationState> im
return Collections.emptyMap();
}
static final class IdeStubForwardIndexesExternalizer extends StubForwardIndexExternalizer<Void> {
private static final class IdeStubForwardIndexesExternalizer extends StubForwardIndexExternalizer<Void> {
private IdeStubForwardIndexesExternalizer(@NotNull SerializationManagerEx managerToInitialize) {
super(managerToInitialize);
}
@Override
protected void writeStubIndexKey(@NotNull DataOutput out, @NotNull StubIndexKey key, Void aVoid) throws IOException {
DataInputOutputUtil.writeINT(out, key.getUniqueId());
@@ -107,7 +131,9 @@ public abstract class StubForwardIndexExternalizer<StubKeySerializationState> im
}
public static final class FileLocalStubForwardIndexExternalizer extends StubForwardIndexExternalizer<FileLocalStringEnumerator> {
public static final FileLocalStubForwardIndexExternalizer INSTANCE = new FileLocalStubForwardIndexExternalizer();
private FileLocalStubForwardIndexExternalizer(SerializationManagerEx managerToInitialize) {
super(managerToInitialize);
}
@Override
protected FileLocalStringEnumerator createStubIndexKeySerializationState(@NotNull DataOutput out, @NotNull Set<StubIndexKey> set) throws IOException {

4
platform/lang-impl/src/com/intellij/psi/stubs/StubSharedIndexExtension.java
View File

@@ -8,9 +8,7 @@ import com.intellij.util.indexing.provided.SharedIndexExtension;
import com.intellij.util.io.DataExternalizer;
import com.intellij.util.io.EnumeratorIntegerDescriptor;
import com.intellij.util.io.KeyDescriptor;
import com.intellij.util.io.VoidDataExternalizer;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.nio.file.Path;
@@ -32,6 +30,6 @@ public class StubSharedIndexExtension implements SharedIndexExtension<Integer, S
public DataExternalizer<SerializedStubTree> createValueExternalizer(@NotNull Path indexPath) {
SerializationManagerImpl manager = new SerializationManagerImpl(indexPath.getParent().resolve("rep.names"), true);
Disposer.register(ApplicationManager.getApplication(), manager);
return new SerializedStubTreeDataExternalizer(true, manager, StubForwardIndexExternalizer.FileLocalStubForwardIndexExternalizer.INSTANCE);
return new SerializedStubTreeDataExternalizer(true, manager, StubForwardIndexExternalizer.createFileLocalExternalizer(manager));
}
}

38
platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java
View File

@@ -54,12 +54,18 @@ public class StubUpdatingIndex extends SingleEntryFileBasedIndexExtension<Serial
@NotNull
private final StubForwardIndexExternalizer<?> myStubIndexesExternalizer;
@NotNull
private final SerializationManagerEx mySerializationManager;
public StubUpdatingIndex() {
myStubIndexesExternalizer = SerializedStubTree.IDE_USED_EXTERNALIZER;
this(StubForwardIndexExternalizer.getIdeUsedExternalizer(SerializationManagerEx.getInstanceEx()),
SerializationManagerEx.getInstanceEx());
}
public StubUpdatingIndex(@NotNull StubForwardIndexExternalizer<?> stubIndexesExternalizer) {
public StubUpdatingIndex(@NotNull StubForwardIndexExternalizer<?> stubIndexesExternalizer,
@NotNull SerializationManagerEx serializationManager) {
myStubIndexesExternalizer = stubIndexesExternalizer;
mySerializationManager = serializationManager;
}
public static boolean canHaveStub(@NotNull VirtualFile file) {
@@ -138,9 +144,13 @@ public class StubUpdatingIndex extends SingleEntryFileBasedIndexExtension<Serial
if (serializedStubTree == null) {
Stub rootStub = StubTreeBuilder.buildStubTree(inputData, type);
if (rootStub != null) {
serializedStubTree = SerializedStubTree.serializeStub(rootStub, SerializationManagerEx.getInstanceEx(), SerializedStubTree.IDE_USED_EXTERNALIZER);
serializedStubTree = SerializedStubTree.serializeStub(
rootStub,
mySerializationManager,
StubForwardIndexExternalizer.getIdeUsedExternalizer(mySerializationManager)
);
if (DebugAssertions.DEBUG) {
Stub deserialized = serializedStubTree.getStub(SerializationManagerEx.getInstanceEx());
Stub deserialized = serializedStubTree.getStub(mySerializationManager);
check(deserialized, rootStub);
}
}
@@ -244,7 +254,7 @@ public class StubUpdatingIndex extends SingleEntryFileBasedIndexExtension<Serial
@NotNull
@Override
public DataExternalizer<SerializedStubTree> getValueExternalizer() {
return new SerializedStubTreeDataExternalizer(true, SerializationManagerEx.getInstanceEx(), myStubIndexesExternalizer);
return new SerializedStubTreeDataExternalizer(true, mySerializationManager, myStubIndexesExternalizer);
}
@NotNull
@@ -278,9 +288,17 @@ public class StubUpdatingIndex extends SingleEntryFileBasedIndexExtension<Serial
}
});
}
checkNameStorage();
return new MyIndex(extension, storage);
}
private void checkNameStorage() throws StorageException {
if (mySerializationManager.isNameStorageCorrupted()) {
mySerializationManager.repairNameStorage();
throw new StorageException("NameStorage for stubs serialization has been corrupted");
}
}
private static class MyIndex extends VfsAwareMapReduceIndex<Integer, SerializedStubTree> {
private StubIndexImpl myStubIndex;
@Nullable
@@ -294,7 +312,6 @@ public class StubUpdatingIndex extends SingleEntryFileBasedIndexExtension<Serial
throws StorageException, IOException {
super(extension, storage, new EmptyForwardIndex(), new StubUpdatingForwardIndexAccessor(), null, null);
((StubUpdatingForwardIndexAccessor)getForwardIndexAccessor()).setIndex(this);
checkNameStorage();
if (InvertedIndex.ARE_COMPOSITE_INDEXERS_ENABLED) {
// load stub serializers before usage
@@ -330,15 +347,6 @@ public class StubUpdatingIndex extends SingleEntryFileBasedIndexExtension<Serial
return index;
}
private static void checkNameStorage() throws StorageException {
final SerializationManagerEx serializationManager = SerializationManagerEx.getInstanceEx();
if (serializationManager.isNameStorageCorrupted()) {
serializationManager.repairNameStorage();
throw new StorageException("NameStorage for stubs serialization has been corrupted");
}
}
@Override
protected void removeTransientDataForInMemoryKeys(int inputId, @NotNull Map<? extends Integer, ? extends SerializedStubTree> map) {
super.removeTransientDataForInMemoryKeys(inputId, map);

13
tools/index-tools/src/org/jetbrains/index/stubs/StubsGenerator.kt
View File

@@ -23,11 +23,6 @@ import java.nio.file.Paths
import java.util.*
import kotlin.system.exitProcess
/**
* Generates stubs and stores them in one persistent hash map
*/
private val STUB_EXTERNALIZER = StubForwardIndexExternalizer.FileLocalStubForwardIndexExternalizer()
open class StubsGenerator(private val stubsVersion: String, private val stubsStorageFilePath: String) :
IndexGenerator<SerializedStubTree>(stubsStorageFilePath) {
@@ -55,7 +50,7 @@ open class StubsGenerator(private val stubsVersion: String, private val stubsSto
return null
}
return SerializedStubTree.serializeStub(stub, serializationManager, STUB_EXTERNALIZER)
return SerializedStubTree.serializeStub(stub, serializationManager, StubForwardIndexExternalizer.createFileLocalExternalizer(serializationManager))
}
override fun createStorage(stubsStorageFilePath: String): PersistentHashMap<HashCode, SerializedStubTree> {
@@ -112,14 +107,16 @@ fun mergeStubs(paths: List<String>, stubsFilePath: String, stubsFileName: String
val value = fromStorage.get(key)
// re-serialize stub tree to correctly enumerate strings in the new string enumerator
val newStubTree = value.reSerialize(serializationManager, newSerializationManager, STUB_EXTERNALIZER, STUB_EXTERNALIZER)
val oldForwardIndexSerializer = StubForwardIndexExternalizer.createFileLocalExternalizer(serializationManager)
val newForwardIndexSerializer = StubForwardIndexExternalizer.createFileLocalExternalizer(newSerializationManager)
val newStubTree = value.reSerialize(serializationManager, newSerializationManager, oldForwardIndexSerializer, newForwardIndexSerializer)
if (storage.containsMapping(key)) {
if (newStubTree != storage.get(key)) { // TODO: why are they slightly different???
storage.get(key).getStub(newSerializationManager)
val stub = value.getStub(serializationManager)
val newStubTree2 = SerializedStubTree.serializeStub(stub, newSerializationManager, STUB_EXTERNALIZER)
val newStubTree2 = SerializedStubTree.serializeStub(stub, newSerializationManager, newForwardIndexSerializer)
TestCase.assertTrue(newStubTree == newStubTree2) // wtf!!! why are they equal now???
}