MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

[vcs-log] mark index corrupted when log storage gets corrupted

Browse Source
This commit is contained in:
Julia Beliaeva
2016-09-20 00:15:17 +03:00
parent 37af75f498
commit 95443d7e5a
11 changed files with 133 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/vcs-log/impl/src/com/intellij/vcs/log/data/VcsLogData.java
+3 -2
View File
@@ -33,6 +33,7 @@ import com.intellij.util.containers.ContainerUtil;
import com.intellij.vcs.log.*;
import com.intellij.vcs.log.data.index.VcsLogIndex;
import com.intellij.vcs.log.data.index.VcsLogPersistentIndex;
import com.intellij.vcs.log.impl.FatalErrorConsumer;
import com.intellij.vcs.log.util.StopWatch;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -72,12 +73,12 @@ public class VcsLogData implements Disposable, VcsLogDataProvider {
@NotNull private final VcsLogRefresherImpl myRefresher;
@NotNull private final List<DataPackChangeListener> myDataPackChangeListeners = ContainerUtil.createLockFreeCopyOnWriteList();
@NotNull private final Consumer<Exception> myFatalErrorsConsumer;
@NotNull private final FatalErrorConsumer myFatalErrorsConsumer;
@NotNull private final VcsLogIndex myIndex;
public VcsLogData(@NotNull Project project,
@NotNull Map<VirtualFile, VcsLogProvider> logProviders,
@NotNull Consumer<Exception> fatalErrorsConsumer) {
@NotNull FatalErrorConsumer fatalErrorsConsumer) {
myProject = project;
myLogProviders = logProviders;
myDataLoaderQueue = new BackgroundTaskQueue(project, "Loading history...");
platform/vcs-log/impl/src/com/intellij/vcs/log/data/VcsLogStorageImpl.java
+9 -8
View File
@@ -29,6 +29,7 @@ import com.intellij.util.io.IOUtil;
import com.intellij.util.io.KeyDescriptor;
import com.intellij.util.io.PersistentEnumeratorBase;
import com.intellij.vcs.log.*;
import com.intellij.vcs.log.impl.FatalErrorConsumer;
import com.intellij.vcs.log.impl.HashImpl;
import com.intellij.vcs.log.impl.VcsRefImpl;
import com.intellij.vcs.log.util.PersistentUtil;
@@ -60,12 +61,12 @@ public class VcsLogStorageImpl implements Disposable, VcsLogStorage {
@NotNull private final PersistentEnumeratorBase<CommitId> myCommitIdEnumerator;
@NotNull private final PersistentEnumeratorBase<VcsRef> myRefsEnumerator;
@NotNull private final Consumer<Exception> myExceptionReporter;
@NotNull private final FatalErrorConsumer myExceptionReporter;
private volatile boolean myDisposed = false;
public VcsLogStorageImpl(@NotNull Project project,
@NotNull Map<VirtualFile, VcsLogProvider> logProviders,
@NotNull Consumer<Exception> exceptionReporter,
@NotNull FatalErrorConsumer exceptionReporter,
@NotNull Disposable parent) throws IOException {
myExceptionReporter = exceptionReporter;
@@ -100,7 +101,7 @@ public class VcsLogStorageImpl implements Disposable, VcsLogStorage {
return getOrPut(hash, root);
}
catch (IOException e) {
myExceptionReporter.consume(e);
myExceptionReporter.consume(this, e);
}
return NO_INDEX;
}
@@ -112,12 +113,12 @@ public class VcsLogStorageImpl implements Disposable, VcsLogStorage {
try {
CommitId commitId = doGetCommitId(commitIndex);
if (commitId == null) {
myExceptionReporter.consume(new RuntimeException("Unknown commit index: " + commitIndex));
myExceptionReporter.consume(this, new RuntimeException("Unknown commit index: " + commitIndex));
}
return commitId;
}
catch (IOException e) {
myExceptionReporter.consume(e);
myExceptionReporter.consume(this, e);
}
return null;
}
@@ -141,7 +142,7 @@ public class VcsLogStorageImpl implements Disposable, VcsLogStorage {
return hashRef.get();
}
catch (IOException e) {
myExceptionReporter.consume(e);
myExceptionReporter.consume(this, e);
return null;
}
}
@@ -153,7 +154,7 @@ public class VcsLogStorageImpl implements Disposable, VcsLogStorage {
return myRefsEnumerator.enumerate(ref);
}
catch (IOException e) {
myExceptionReporter.consume(e);
myExceptionReporter.consume(this, e);
}
return NO_INDEX;
}
@@ -166,7 +167,7 @@ public class VcsLogStorageImpl implements Disposable, VcsLogStorage {
return myRefsEnumerator.valueOf(refIndex);
}
catch (IOException e) {
myExceptionReporter.consume(e);
myExceptionReporter.consume(this, e);
return null;
}
}
platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogFullDetailsIndex.java
+31 -8
View File
@@ -21,10 +21,7 @@ import com.intellij.openapi.util.Pair;
import com.intellij.util.Consumer;
import com.intellij.util.PathUtilRt;
import com.intellij.util.indexing.*;
import com.intellij.util.io.DataExternalizer;
import com.intellij.util.io.EnumeratorIntegerDescriptor;
import com.intellij.util.io.KeyDescriptor;
import com.intellij.util.io.PersistentHashMap;
import com.intellij.util.io.*;
import com.intellij.vcs.log.VcsFullCommitDetails;
import com.intellij.vcs.log.util.PersistentUtil;
import gnu.trove.TIntHashSet;
@@ -37,6 +34,8 @@ import java.util.Set;
import java.util.function.ObjIntConsumer;
public class VcsLogFullDetailsIndex<T> implements Disposable {
@NotNull protected static final String INDEX = "index-";
@NotNull protected static final String INDEX_INPUTS = "index-inputs-";
@NotNull protected final MyMapReduceIndex myMapReduceIndex;
@NotNull private final ID<Integer, T> myID;
@NotNull private final String myLogId;
@@ -45,7 +44,7 @@ public class VcsLogFullDetailsIndex<T> implements Disposable {
public VcsLogFullDetailsIndex(@NotNull String logId,
@NotNull String name,
int version,
final int version,
@NotNull DataIndexer<Integer, T, VcsFullCommitDetails> indexer,
@NotNull DataExternalizer<T> externalizer,
@NotNull Disposable disposableParent)
@@ -55,7 +54,13 @@ public class VcsLogFullDetailsIndex<T> implements Disposable {
myLogId = logId;
myIndexer = indexer;
myMapReduceIndex = new MyMapReduceIndex(myIndexer, externalizer, version);
MyMapReduceIndex result = IOUtil.openCleanOrResetBroken(() -> new MyMapReduceIndex(myIndexer, externalizer, version),
() -> {
IOUtil.deleteAllFilesStartingWith(getStorageFile(version));
IOUtil.deleteAllFilesStartingWith(getInputsStorageFile(version));
});
if (result == null) throw new IOException("Can not create " + myName + " index for " + myLogId);
myMapReduceIndex = result;
Disposer.register(disposableParent, this);
}
@@ -156,6 +161,20 @@ public class VcsLogFullDetailsIndex<T> implements Disposable {
protected void onNotIndexableCommit(int commit) throws StorageException {
}
public void markCorrupted() {
myMapReduceIndex.markCorrupted();
}
@NotNull
private File getStorageFile(int version) {
return getStorageFile(INDEX + myName, myLogId, version);
}
@NotNull
private File getInputsStorageFile(int version) {
return PersistentUtil.getStorageFile(INDEX_INPUTS + myName, myLogId, version);
}
@NotNull
public static File getStorageFile(@NotNull String kind, @NotNull String id, int version) {
File subdir = new File(PersistentUtil.LOG_CACHE, kind);
@@ -169,7 +188,7 @@ public class VcsLogFullDetailsIndex<T> implements Disposable {
@NotNull DataExternalizer<T> externalizer,
int version) throws IOException {
super(new MyIndexExtension(indexer, externalizer, version),
new MapIndexStorage<>(getStorageFile("index-" + myName, VcsLogFullDetailsIndex.this.myLogId, version),
new MapIndexStorage<>(getStorageFile(version),
EnumeratorIntegerDescriptor.INSTANCE,
externalizer, 5000));
}
@@ -181,7 +200,7 @@ public class VcsLogFullDetailsIndex<T> implements Disposable {
@Override
protected PersistentHashMap<Integer, Collection<Integer>> createInputsIndex() throws IOException {
IndexExtension<Integer, T, VcsFullCommitDetails> extension = getExtension();
return new PersistentHashMap<>(PersistentUtil.getStorageFile("index-inputs-" + myName, myLogId, extension.getVersion()),
return new PersistentHashMap<>(getInputsStorageFile(extension.getVersion()),
EnumeratorIntegerDescriptor.INSTANCE,
new InputIndexDataExternalizer<>(extension.getKeyDescriptor(), myID));
}
@@ -193,6 +212,10 @@ public class VcsLogFullDetailsIndex<T> implements Disposable {
}
super.updateWithMap(inputId, updateData);
}
public void markCorrupted() {
myInputsIndex.markCorrupted();
}
}
private class MyIndexExtension extends IndexExtension<Integer, T, VcsFullCommitDetails> {
platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogIndex.java
+2
View File
@@ -35,4 +35,6 @@ public interface VcsLogIndex {
@NotNull
Set<Integer> filter(@NotNull List<VcsLogDetailsFilter> detailsFilters);
void markCorrupted();
}
platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogMessagesTrigramIndex.java
+6
View File
@@ -93,6 +93,12 @@ public class VcsLogMessagesTrigramIndex extends VcsLogFullDetailsIndex<Void> {
}
}
@Override
public void markCorrupted() {
super.markCorrupted();
myNoTrigramsCommits.markCorrupted();
}
public static class TrigramMessageIndexer implements DataIndexer<Integer, Void, VcsFullCommitDetails> {
@NotNull
@Override
platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogPathsIndex.java
+24 -3
View File
@@ -35,6 +35,7 @@ import org.jetbrains.annotations.NotNull;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.Map;
@@ -52,9 +53,7 @@ public class VcsLogPathsIndex extends VcsLogFullDetailsIndex<Integer> {
public VcsLogPathsIndex(@NotNull String logId,
@NotNull Set<VirtualFile> roots,
@NotNull Disposable disposableParent) throws IOException {
super(logId, NAME, VcsLogPersistentIndex.getVersion(), new PathsIndexer(
PersistentUtil.createPersistentEnumerator(EnumeratorStringDescriptor.INSTANCE, "index-paths-ids", logId,
VcsLogPersistentIndex.getVersion()), roots),
super(logId, NAME, VcsLogPersistentIndex.getVersion(), new PathsIndexer(createPathsEnumerator(logId), roots),
new NullableIntKeyDescriptor(), disposableParent);
myEmptyCommits = PersistentUtil.createPersistentHashMap(EnumeratorIntegerDescriptor.INSTANCE, "index-no-" + NAME, logId,
@@ -62,6 +61,22 @@ public class VcsLogPathsIndex extends VcsLogFullDetailsIndex<Integer> {
myPathsIndexer = (PathsIndexer)myIndexer;
}
@NotNull
private static PersistentEnumeratorBase<String> createPathsEnumerator(@NotNull String logId) throws IOException {
int version = VcsLogPersistentIndex.getVersion();
final File storageFile = PersistentUtil.getStorageFile("index-paths-ids", logId, version);
PersistentBTreeEnumerator<String> enumerator = IOUtil.openCleanOrResetBroken(
() -> new PersistentBTreeEnumerator<>(storageFile, EnumeratorStringDescriptor.INSTANCE, Page.PAGE_SIZE, null, version),
() -> {
IOUtil.deleteAllFilesStartingWith(getStorageFile(INDEX + NAME, logId, version));
IOUtil.deleteAllFilesStartingWith(getStorageFile(INDEX_INPUTS + NAME, logId, version));
IOUtil.deleteAllFilesStartingWith(storageFile);
});
if (enumerator == null) throw new IOException("Can not create enumerator " + NAME + " for " + logId);
return enumerator;
}
@Override
protected void onNotIndexableCommit(int commit) throws StorageException {
try {
@@ -134,6 +149,12 @@ public class VcsLogPathsIndex extends VcsLogFullDetailsIndex<Integer> {
}
}
@Override
public void markCorrupted() {
super.markCorrupted();
myEmptyCommits.markCorrupted();
}
private static class PathsIndexer implements DataIndexer<Integer, Integer, VcsFullCommitDetails> {
@NotNull private final PersistentEnumeratorBase<String> myPathsEnumerator;
@NotNull private final Set<String> myRoots;
platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogPersistentIndex.java
+21 -15
View File
@@ -32,7 +32,6 @@ import com.intellij.openapi.util.text.StringUtil;
import com.intellij.openapi.vcs.FilePath;
import com.intellij.openapi.vcs.VcsException;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.util.Consumer;
import com.intellij.util.Processor;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.indexing.StorageException;
@@ -45,6 +44,7 @@ import com.intellij.vcs.log.data.InMemoryMap;
import com.intellij.vcs.log.data.TroveUtil;
import com.intellij.vcs.log.data.VcsLogStorageImpl;
import com.intellij.vcs.log.data.VcsUserRegistryImpl;
import com.intellij.vcs.log.impl.FatalErrorConsumer;
import com.intellij.vcs.log.ui.filter.VcsLogUserFilterImpl;
import com.intellij.vcs.log.util.PersistentUtil;
import gnu.trove.TIntHashSet;
@@ -55,7 +55,6 @@ import org.jetbrains.annotations.Nullable;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicIntegerArray;
import java.util.stream.IntStream;
public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
@@ -64,7 +63,7 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
private static final int VERSION = 0;
@NotNull private final Project myProject;
@NotNull private final Consumer<Exception> myFatalErrorsConsumer;
@NotNull private final FatalErrorConsumer myFatalErrorsConsumer;
@NotNull private final Map<VirtualFile, VcsLogProvider> myProviders;
@NotNull private final VcsLogStorage myHashMap;
@NotNull private final VcsUserRegistryImpl myUserRegistry;
@@ -82,7 +81,7 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
public VcsLogPersistentIndex(@NotNull Project project,
@NotNull VcsLogStorage hashMap,
@NotNull Map<VirtualFile, VcsLogProvider> providers,
@NotNull Consumer<Exception> fatalErrorsConsumer,
@NotNull FatalErrorConsumer fatalErrorsConsumer,
@NotNull Disposable disposableParent) {
myHashMap = hashMap;
myProject = project;
@@ -122,7 +121,7 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
return computable.compute();
}
catch (IOException e) {
myFatalErrorsConsumer.consume(e);
myFatalErrorsConsumer.consume(this, e);
}
return null;
}
@@ -135,7 +134,7 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
return PersistentUtil.createPersistentHashMap(descriptor, kind, logId, version);
}
catch (IOException e) {
myFatalErrorsConsumer.consume(e);
myFatalErrorsConsumer.consume(this, e);
return new InMemoryMap<>();
}
}
@@ -174,7 +173,7 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
}
}
catch (IOException e) {
myFatalErrorsConsumer.consume(e);
myFatalErrorsConsumer.consume(this, e);
}
}
@@ -186,10 +185,17 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
if (myPathsIndex != null) myPathsIndex.flush();
}
catch (StorageException e) {
myFatalErrorsConsumer.consume(e);
myFatalErrorsConsumer.consume(this, e);
}
}
public void markCorrupted() {
if (myMessagesIndex instanceof PersistentHashMap) ((PersistentHashMap)myMessagesIndex).markCorrupted();
if (myTrigramIndex != null) myTrigramIndex.markCorrupted();
if (myUserIndex != null) myUserIndex.markCorrupted();
if (myPathsIndex != null) myPathsIndex.markCorrupted();
}
@Override
public boolean isIndexed(int commit) {
try {
@@ -199,7 +205,7 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
(myTrigramIndex == null || myTrigramIndex.isIndexed(commit));
}
catch (IOException e) {
myFatalErrorsConsumer.consume(e);
myFatalErrorsConsumer.consume(this, e);
}
return false;
}
@@ -234,7 +240,7 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
}
}
catch (IOException e) {
myFatalErrorsConsumer.consume(e);
myFatalErrorsConsumer.consume(this, e);
return false;
}
return true;
@@ -247,7 +253,7 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
}
}
catch (IOException e) {
myFatalErrorsConsumer.consume(e);
myFatalErrorsConsumer.consume(this, e);
}
return result;
@@ -260,7 +266,7 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
return myUserIndex.getCommitsForUsers(users);
}
catch (IOException | StorageException e) {
myFatalErrorsConsumer.consume(e);
myFatalErrorsConsumer.consume(this, e);
}
}
return new TIntHashSet();
@@ -273,7 +279,7 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
return myPathsIndex.getCommitsForPaths(paths);
}
catch (IOException | StorageException e) {
myFatalErrorsConsumer.consume(e);
myFatalErrorsConsumer.consume(this, e);
}
}
return new TIntHashSet();
@@ -298,7 +304,7 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
}
}
catch (IOException e) {
myFatalErrorsConsumer.consume(e);
myFatalErrorsConsumer.consume(this, e);
return false;
}
@@ -309,7 +315,7 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
}
}
catch (StorageException e) {
myFatalErrorsConsumer.consume(e);
myFatalErrorsConsumer.consume(this, e);
}
}
platform/vcs-log/impl/src/com/intellij/vcs/log/impl/FatalErrorConsumer.java
+23
View File
@@ -0,0 +1,23 @@
/*
* Copyright 2000-2016 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.vcs.log.impl;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
public interface FatalErrorConsumer {
void consume(@Nullable Object source, @NotNull Exception exception);
}
platform/vcs-log/impl/src/com/intellij/vcs/log/impl/VcsLogManager.java
+8 -4
View File
@@ -33,6 +33,7 @@ import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.containers.MultiMap;
import com.intellij.vcs.log.VcsLogProvider;
import com.intellij.vcs.log.VcsLogRefresher;
import com.intellij.vcs.log.VcsLogStorage;
import com.intellij.vcs.log.data.VcsLogData;
import com.intellij.vcs.log.data.VcsLogFiltererImpl;
import com.intellij.vcs.log.data.VcsLogTabsProperties;
@@ -201,15 +202,15 @@ public class VcsLogManager implements Disposable {
disposeLog();
}
private class MyFatalErrorsConsumer implements Consumer<Exception> {
private class MyFatalErrorsConsumer implements FatalErrorConsumer {
private boolean myIsBroken = false;
@Override
public void consume(@NotNull final Exception e) {
public void consume(@Nullable Object source, @NotNull final Exception e) {
ApplicationManager.getApplication().invokeLater(() -> {
if (!myIsBroken) {
myIsBroken = true;
processErrorFirstTime(e);
processErrorFirstTime(source, e);
}
else {
LOG.debug(e);
@@ -217,7 +218,7 @@ public class VcsLogManager implements Disposable {
});
}
protected void processErrorFirstTime(@NotNull Exception e) {
protected void processErrorFirstTime(@Nullable Object source, @NotNull Exception e) {
if (myRecreateMainLogHandler != null) {
String message = "Fatal error, VCS Log recreated: " + e.getMessage();
if (isLogVisible()) {
@@ -232,6 +233,9 @@ public class VcsLogManager implements Disposable {
else {
LOG.error(e);
}
if (source instanceof VcsLogStorage) {
myLogData.getIndex().markCorrupted();
}
}
}
}
platform/vcs-log/impl/test/com/intellij/vcs/log/data/VcsLogRefresherTest.java
+3 -5
View File
@@ -32,12 +32,10 @@ import com.intellij.vcs.log.TimedVcsCommit;
import com.intellij.vcs.log.VcsCommitMetadata;
import com.intellij.vcs.log.VcsLogProvider;
import com.intellij.vcs.log.graph.GraphCommit;
import com.intellij.vcs.log.impl.HashImpl;
import com.intellij.vcs.log.impl.TestVcsLogProvider;
import com.intellij.vcs.log.impl.TimedVcsCommitImpl;
import com.intellij.vcs.log.impl.VcsRefImpl;
import com.intellij.vcs.log.impl.*;
import com.intellij.vcs.test.VcsPlatformTest;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.util.*;
import java.util.concurrent.*;
@@ -199,7 +197,7 @@ public class VcsLogRefresherTest extends VcsPlatformTest {
}
private VcsLogRefresherImpl createLoader(Consumer<DataPack> dataPackConsumer) {
myLogData = new VcsLogData(myProject, myLogProviders, LOG::error);
myLogData = new VcsLogData(myProject, myLogProviders, (source, exception) -> LOG.error(exception));
Disposer.register(myProject, myLogData);
return new VcsLogRefresherImpl(myProject, myLogData.getHashMap(), myLogProviders, myLogData.getUserRegistry(), myLogData.getIndex(),
myLogData.getTopCommitsCache(), dataPackConsumer, FAILING_EXCEPTION_HANDLER, RECENT_COMMITS_COUNT,
platform/vcs-log/impl/test/com/intellij/vcs/log/data/VisiblePackBuilderTest.kt
+3
View File
@@ -269,6 +269,9 @@ class VisiblePackBuilderTest {
override fun filter(detailsFilters: MutableList<VcsLogDetailsFilter>): MutableSet<Int> {
throw UnsupportedOperationException()
}
override fun markCorrupted() {
}
}
}