MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

[vcs-log] inputs index is not needed in log

Browse Source 
* InputsIndex is null for log indexes.
* Common persistent set is used for keeping indexed status.
* Remove myEmptyCommits from paths index and myNoTrigramsCommits from trigrams index. Commits with no paths or no trigrams wont be re-indexed now since they will be stored in the persistent set.
* Remove PrintIndexInfo action, not very useful and used for debug.
* Rewrite create/dispose code for indexes. Each constructor now throws an exception if something's wrong and all indexes are cleaned-up and recreated together if it is.
This commit is contained in:
Julia Beliaeva
2016-11-10 20:12:25 +03:00
parent 0ff6d44aa7
commit 32e141ef1b
7 changed files with 143 additions and 412 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogFullDetailsIndex.java
+7 -56
View File
@@ -20,7 +20,10 @@ import com.intellij.openapi.util.Disposer;
import com.intellij.util.Consumer; import com.intellij.util.Consumer;
import com.intellij.util.PathUtilRt; import com.intellij.util.PathUtilRt;
import com.intellij.util.indexing.*; import com.intellij.util.indexing.*;
import com.intellij.util.io.*; import com.intellij.util.io.DataExternalizer;
import com.intellij.util.io.EnumeratorIntegerDescriptor;
import com.intellij.util.io.KeyDescriptor;
import com.intellij.util.io.PersistentHashMap;
import com.intellij.vcs.log.VcsFullCommitDetails; import com.intellij.vcs.log.VcsFullCommitDetails;
import com.intellij.vcs.log.impl.FatalErrorHandler; import com.intellij.vcs.log.impl.FatalErrorHandler;
import com.intellij.vcs.log.util.PersistentUtil; import com.intellij.vcs.log.util.PersistentUtil;
@@ -36,7 +39,6 @@ import java.util.function.ObjIntConsumer;
public class VcsLogFullDetailsIndex<T> implements Disposable { public class VcsLogFullDetailsIndex<T> implements Disposable {
@NotNull protected static final String INDEX = "index-"; @NotNull protected static final String INDEX = "index-";
@NotNull protected static final String INDEX_INPUTS = "index-inputs-";
@NotNull protected final MyMapReduceIndex myMapReduceIndex; @NotNull protected final MyMapReduceIndex myMapReduceIndex;
@NotNull private final ID<Integer, T> myID; @NotNull private final ID<Integer, T> myID;
@NotNull private final String myLogId; @NotNull private final String myLogId;
@@ -58,13 +60,7 @@ public class VcsLogFullDetailsIndex<T> implements Disposable {
myIndexer = indexer; myIndexer = indexer;
myFatalErrorHandler = fatalErrorHandler; myFatalErrorHandler = fatalErrorHandler;
MyMapReduceIndex result = IOUtil.openCleanOrResetBroken(() -> new MyMapReduceIndex(myIndexer, externalizer, version), myMapReduceIndex = new MyMapReduceIndex(myIndexer, externalizer, version);
() -> {
IOUtil.deleteAllFilesStartingWith(getStorageFile(version));
IOUtil.deleteAllFilesStartingWith(getInputsStorageFile(version));
});
if (result == null) throw new IOException("Can not create " + myName + " index for " + myLogId);
myMapReduceIndex = result;
Disposer.register(disposableParent, this); Disposer.register(disposableParent, this);
} }
@@ -120,32 +116,16 @@ public class VcsLogFullDetailsIndex<T> implements Disposable {
myMapReduceIndex.flush(); myMapReduceIndex.flush();
} }
public boolean isIndexed(int commit) throws IOException {
return myMapReduceIndex.isIndexed(commit);
}
@Override @Override
public void dispose() { public void dispose() {
myMapReduceIndex.dispose(); myMapReduceIndex.dispose();
} }
protected void onNotIndexableCommit(int commit) throws StorageException {
}
public void markCorrupted() {
myMapReduceIndex.markCorrupted();
}
@NotNull @NotNull
private File getStorageFile(int version) { public File getStorageFile(int version) {
return getStorageFile(INDEX + myName, myLogId, version); return getStorageFile(INDEX + myName, myLogId, version);
} }
@NotNull
private File getInputsStorageFile(int version) {
return PersistentUtil.getStorageFile(INDEX_INPUTS + myName, myLogId, version);
}
@NotNull @NotNull
public static File getStorageFile(@NotNull String kind, @NotNull String id, int version) { public static File getStorageFile(@NotNull String kind, @NotNull String id, int version) {
File subdir = new File(PersistentUtil.LOG_CACHE, kind); File subdir = new File(PersistentUtil.LOG_CACHE, kind);
@@ -153,11 +133,6 @@ public class VcsLogFullDetailsIndex<T> implements Disposable {
return new File(subdir, safeLogId + "." + version); return new File(subdir, safeLogId + "." + version);
} }
@Nullable
protected Collection<Integer> getKeysForCommit(int commit) throws IOException {
return myMapReduceIndex.getInputsIndex().get(commit);
}
private class MyMapReduceIndex extends MapReduceIndex<Integer, T, VcsFullCommitDetails> { private class MyMapReduceIndex extends MapReduceIndex<Integer, T, VcsFullCommitDetails> {
public MyMapReduceIndex(@NotNull DataIndexer<Integer, T, VcsFullCommitDetails> indexer, public MyMapReduceIndex(@NotNull DataIndexer<Integer, T, VcsFullCommitDetails> indexer,
@@ -169,33 +144,9 @@ public class VcsLogFullDetailsIndex<T> implements Disposable {
externalizer, 5000)); externalizer, 5000));
} }
@NotNull
public PersistentHashMap<Integer, Collection<Integer>> getInputsIndex() {
return myInputsIndex;
}
public boolean isIndexed(int commitId) throws IOException {
return myInputsIndex.containsMapping(commitId);
}
@Override @Override
protected PersistentHashMap<Integer, Collection<Integer>> createInputsIndex() throws IOException { protected PersistentHashMap<Integer, Collection<Integer>> createInputsIndex() throws IOException {
IndexExtension<Integer, T, VcsFullCommitDetails> extension = getExtension(); return null;
return new PersistentHashMap<>(getInputsStorageFile(extension.getVersion()),
EnumeratorIntegerDescriptor.INSTANCE,
new InputIndexDataExternalizer<>(extension.getKeyDescriptor(), myID));
}
@Override
protected void updateWithMap(int inputId, @NotNull UpdateData<Integer, T> updateData) throws StorageException {
if (((SimpleUpdateData)updateData).getNewData().isEmpty()) {
onNotIndexableCommit(inputId);
}
super.updateWithMap(inputId, updateData);
}
public void markCorrupted() {
myInputsIndex.markCorrupted();
} }
@Override @Override
platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogMessagesTrigramIndex.java
+10 -70
View File
@@ -16,40 +16,38 @@
package com.intellij.vcs.log.data.index; package com.intellij.vcs.log.data.index;
import com.intellij.openapi.Disposable; import com.intellij.openapi.Disposable;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.text.TrigramBuilder; import com.intellij.openapi.util.text.TrigramBuilder;
import com.intellij.util.indexing.DataIndexer; import com.intellij.util.indexing.DataIndexer;
import com.intellij.util.indexing.ScalarIndexExtension; import com.intellij.util.indexing.ScalarIndexExtension;
import com.intellij.util.indexing.StorageException; import com.intellij.util.indexing.StorageException;
import com.intellij.util.indexing.ValueContainer; import com.intellij.util.indexing.ValueContainer;
import com.intellij.util.io.EnumeratorIntegerDescriptor;
import com.intellij.vcs.log.VcsFullCommitDetails; import com.intellij.vcs.log.VcsFullCommitDetails;
import com.intellij.vcs.log.impl.FatalErrorHandler; import com.intellij.vcs.log.impl.FatalErrorHandler;
import com.intellij.vcs.log.util.PersistentSet;
import com.intellij.vcs.log.util.PersistentUtil;
import gnu.trove.THashMap; import gnu.trove.THashMap;
import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.Nullable;
import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.Collection; import java.util.Collection;
import java.util.Collections;
import java.util.Map; import java.util.Map;
public class VcsLogMessagesTrigramIndex extends VcsLogFullDetailsIndex<Void> { import static com.intellij.vcs.log.data.index.VcsLogPersistentIndex.getVersion;
private static final Logger LOG = Logger.getInstance(VcsLogMessagesTrigramIndex.class);
private static final String TRIGRAMS = "trigrams";
@NotNull private final PersistentSet<Integer> myNoTrigramsCommits; public class VcsLogMessagesTrigramIndex extends VcsLogFullDetailsIndex<Void> {
private static final String TRIGRAMS = "trigrams";
public VcsLogMessagesTrigramIndex(@NotNull String logId, public VcsLogMessagesTrigramIndex(@NotNull String logId,
@NotNull FatalErrorHandler fatalErrorHandler, @NotNull FatalErrorHandler fatalErrorHandler,
@NotNull Disposable disposableParent) throws IOException { @NotNull Disposable disposableParent) throws IOException {
super(logId, TRIGRAMS, VcsLogPersistentIndex.getVersion(), new TrigramMessageIndexer(), ScalarIndexExtension.VOID_DATA_EXTERNALIZER, super(logId, TRIGRAMS, getVersion(), new TrigramMessageIndexer(), ScalarIndexExtension.VOID_DATA_EXTERNALIZER,
fatalErrorHandler, disposableParent); fatalErrorHandler, disposableParent);
}
myNoTrigramsCommits = @NotNull
PersistentUtil.createPersistentSet(EnumeratorIntegerDescriptor.INSTANCE, "index-no-" + TRIGRAMS, logId, public static Collection<File> getStorageFiles(@NotNull String logId) {
VcsLogPersistentIndex.getVersion()); return Collections.singletonList(getStorageFile(TRIGRAMS, logId, getVersion()));
} }
@Nullable @Nullable
@@ -62,64 +60,6 @@ public class VcsLogMessagesTrigramIndex extends VcsLogFullDetailsIndex<Void> {
return getCommitsWithAllKeys(trigramProcessor.map.keySet()); return getCommitsWithAllKeys(trigramProcessor.map.keySet());
} }
@Override
protected void onNotIndexableCommit(int commit) throws StorageException {
try {
myNoTrigramsCommits.put(commit);
}
catch (IOException e) {
throw new StorageException(e);
}
}
@Override
public boolean isIndexed(int commit) throws IOException {
return super.isIndexed(commit) || myNoTrigramsCommits.contains(commit);
}
@Override
public void flush() throws StorageException {
super.flush();
myNoTrigramsCommits.flush();
}
@Override
public void dispose() {
super.dispose();
try {
myNoTrigramsCommits.close();
}
catch (IOException e) {
LOG.warn(e);
}
}
@Override
public void markCorrupted() {
super.markCorrupted();
myNoTrigramsCommits.markCorrupted();
}
@NotNull
public String getTrigramInfo(int commit) throws IOException {
if (myNoTrigramsCommits.contains(commit)) {
return "No trigrams";
}
Collection<Integer> keys = getKeysForCommit(commit);
assert keys != null;
StringBuilder builder = new StringBuilder();
for (Integer key : keys) {
builder.append((char)(key >> 16));
builder.append((char)((key >> 8) % 256));
builder.append((char)(key % 256));
builder.append(" ");
}
return builder.toString();
}
public static class TrigramMessageIndexer implements DataIndexer<Integer, Void, VcsFullCommitDetails> { public static class TrigramMessageIndexer implements DataIndexer<Integer, Void, VcsFullCommitDetails> {
@NotNull @NotNull
@Override @Override
platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogPathsIndex.java
+14 -63
View File
@@ -33,7 +33,6 @@ import com.intellij.util.text.CaseInsensitiveStringHashingStrategy;
import com.intellij.vcs.log.VcsFullCommitDetails; import com.intellij.vcs.log.VcsFullCommitDetails;
import com.intellij.vcs.log.impl.FatalErrorHandler; import com.intellij.vcs.log.impl.FatalErrorHandler;
import com.intellij.vcs.log.impl.VcsChangesLazilyParsedDetails; import com.intellij.vcs.log.impl.VcsChangesLazilyParsedDetails;
import com.intellij.vcs.log.util.PersistentSet;
import com.intellij.vcs.log.util.PersistentUtil; import com.intellij.vcs.log.util.PersistentUtil;
import gnu.trove.THashMap; import gnu.trove.THashMap;
import gnu.trove.TIntHashSet; import gnu.trove.TIntHashSet;
@@ -43,72 +42,50 @@ import java.io.DataInput;
import java.io.DataOutput; import java.io.DataOutput;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import static com.intellij.util.containers.ContainerUtil.newTroveSet; import static com.intellij.util.containers.ContainerUtil.newTroveSet;
import static com.intellij.vcs.log.data.index.VcsLogPersistentIndex.getVersion;
public class VcsLogPathsIndex extends VcsLogFullDetailsIndex<Integer> { public class VcsLogPathsIndex extends VcsLogFullDetailsIndex<Integer> {
private static final Logger LOG = Logger.getInstance(VcsLogPathsIndex.class); private static final Logger LOG = Logger.getInstance(VcsLogPathsIndex.class);
private static final String NAME = "paths"; private static final String PATHS = "paths";
private static final String INDEX_PATHS_IDS = "index-paths-ids";
@NotNull private final PersistentSet<Integer> myEmptyCommits;
@NotNull private final PathsIndexer myPathsIndexer; @NotNull private final PathsIndexer myPathsIndexer;
public VcsLogPathsIndex(@NotNull String logId, public VcsLogPathsIndex(@NotNull String logId,
@NotNull Set<VirtualFile> roots, @NotNull Set<VirtualFile> roots,
@NotNull FatalErrorHandler fatalErrorHandler, @NotNull FatalErrorHandler fatalErrorHandler,
@NotNull Disposable disposableParent) throws IOException { @NotNull Disposable disposableParent) throws IOException {
super(logId, NAME, VcsLogPersistentIndex.getVersion(), new PathsIndexer(createPathsEnumerator(logId), roots), super(logId, PATHS, getVersion(), new PathsIndexer(createPathsEnumerator(logId), roots),
new NullableIntKeyDescriptor(), fatalErrorHandler, disposableParent); new NullableIntKeyDescriptor(), fatalErrorHandler, disposableParent);
myEmptyCommits = PersistentUtil.createPersistentSet(EnumeratorIntegerDescriptor.INSTANCE, "index-no-" + NAME, logId,
VcsLogPersistentIndex.getVersion());
myPathsIndexer = (PathsIndexer)myIndexer; myPathsIndexer = (PathsIndexer)myIndexer;
myPathsIndexer.setFatalErrorConsumer(e -> { myPathsIndexer.setFatalErrorConsumer(e -> fatalErrorHandler.consume(this, e));
fatalErrorHandler.consume(this, e);
markCorrupted();
});
} }
@NotNull @NotNull
private static PersistentEnumeratorBase<String> createPathsEnumerator(@NotNull String logId) throws IOException { private static PersistentEnumeratorBase<String> createPathsEnumerator(@NotNull String logId) throws IOException {
int version = VcsLogPersistentIndex.getVersion(); File storageFile = PersistentUtil.getStorageFile(INDEX_PATHS_IDS, logId, getVersion());
final File storageFile = PersistentUtil.getStorageFile("index-paths-ids", logId, version);
PersistentBTreeEnumerator<String> enumerator = IOUtil.openCleanOrResetBroken( return new PersistentBTreeEnumerator<>(storageFile, SystemInfo.isFileSystemCaseSensitive ? EnumeratorStringDescriptor.INSTANCE
() -> new PersistentBTreeEnumerator<>(storageFile, SystemInfo.isFileSystemCaseSensitive ? EnumeratorStringDescriptor.INSTANCE : new ToLowerCaseStringDescriptor(),
: new ToLowerCaseStringDescriptor(), Page.PAGE_SIZE, null, getVersion());
Page.PAGE_SIZE, null, version),
() -> {
IOUtil.deleteAllFilesStartingWith(getStorageFile(INDEX + NAME, logId, version));
IOUtil.deleteAllFilesStartingWith(getStorageFile(INDEX_INPUTS + NAME, logId, version));
IOUtil.deleteAllFilesStartingWith(storageFile);
});
if (enumerator == null) throw new IOException("Can not create enumerator " + NAME + " for " + logId);
return enumerator;
} }
@Override @NotNull
protected void onNotIndexableCommit(int commit) throws StorageException { public static Collection<File> getStorageFiles(@NotNull String logId) {
try { return Arrays.asList(PersistentUtil.getStorageFile(INDEX_PATHS_IDS, logId, getVersion()),
myEmptyCommits.put(commit); getStorageFile(PATHS, logId, getVersion()));
}
catch (IOException e) {
throw new StorageException(e);
}
}
@Override
public boolean isIndexed(int commit) throws IOException {
return super.isIndexed(commit) || myEmptyCommits.contains(commit);
} }
@Override @Override
public void flush() throws StorageException { public void flush() throws StorageException {
super.flush(); super.flush();
myEmptyCommits.flush();
myPathsIndexer.getPathsEnumerator().force(); myPathsIndexer.getPathsEnumerator().force();
} }
@@ -148,12 +125,6 @@ public class VcsLogPathsIndex extends VcsLogFullDetailsIndex<Integer> {
@Override @Override
public void dispose() { public void dispose() {
super.dispose(); super.dispose();
try {
myEmptyCommits.close();
}
catch (IOException e) {
LOG.warn(e);
}
try { try {
myPathsIndexer.getPathsEnumerator().close(); myPathsIndexer.getPathsEnumerator().close();
} }
@@ -162,26 +133,6 @@ public class VcsLogPathsIndex extends VcsLogFullDetailsIndex<Integer> {
} }
} }
@Override
public void markCorrupted() {
super.markCorrupted();
myEmptyCommits.markCorrupted();
}
@NotNull
public String getPathInfo(int commit) throws IOException {
if (myEmptyCommits.contains(commit)) {
return "No paths";
}
Collection<Integer> keys = getKeysForCommit(commit);
assert keys != null;
StringBuilder builder = new StringBuilder();
for (int key : keys) {
builder.append(myPathsIndexer.getPathsEnumerator().valueOf(key)).append("\n");
}
return builder.toString();
}
private static class PathsIndexer implements DataIndexer<Integer, Integer, VcsFullCommitDetails> { private static class PathsIndexer implements DataIndexer<Integer, Integer, VcsFullCommitDetails> {
@NotNull private final PersistentEnumeratorBase<String> myPathsEnumerator; @NotNull private final PersistentEnumeratorBase<String> myPathsEnumerator;
@NotNull private final Set<String> myRoots; @NotNull private final Set<String> myRoots;
platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogPersistentIndex.java
+95 -102
View File
@@ -23,25 +23,24 @@ import com.intellij.openapi.progress.*;
import com.intellij.openapi.project.Project; import com.intellij.openapi.project.Project;
import com.intellij.openapi.util.Condition; import com.intellij.openapi.util.Condition;
import com.intellij.openapi.util.Disposer; import com.intellij.openapi.util.Disposer;
import com.intellij.openapi.util.ThrowableComputable;
import com.intellij.openapi.util.text.StringUtil; import com.intellij.openapi.util.text.StringUtil;
import com.intellij.openapi.vcs.FilePath; import com.intellij.openapi.vcs.FilePath;
import com.intellij.openapi.vcs.VcsException; import com.intellij.openapi.vcs.VcsException;
import com.intellij.openapi.vfs.VirtualFile; import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.util.EmptyConsumer; import com.intellij.util.EmptyConsumer;
import com.intellij.util.Processor; import com.intellij.util.Processor;
import com.intellij.util.ThrowableRunnable;
import com.intellij.util.containers.ContainerUtil; import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.containers.EmptyIntHashSet;
import com.intellij.util.indexing.StorageException; import com.intellij.util.indexing.StorageException;
import com.intellij.util.indexing.ValueContainer; import com.intellij.util.indexing.ValueContainer;
import com.intellij.util.io.EnumeratorStringDescriptor; import com.intellij.util.io.*;
import com.intellij.util.io.KeyDescriptor;
import com.intellij.util.io.PersistentHashMap;
import com.intellij.util.io.PersistentMap;
import com.intellij.vcs.log.*; import com.intellij.vcs.log.*;
import com.intellij.vcs.log.data.*; import com.intellij.vcs.log.data.*;
import com.intellij.vcs.log.impl.FatalErrorHandler; import com.intellij.vcs.log.impl.FatalErrorHandler;
import com.intellij.vcs.log.impl.VcsLogUtil;
import com.intellij.vcs.log.impl.VcsLogUserFilterImpl; import com.intellij.vcs.log.impl.VcsLogUserFilterImpl;
import com.intellij.vcs.log.impl.VcsLogUtil;
import com.intellij.vcs.log.util.PersistentSet;
import com.intellij.vcs.log.util.PersistentUtil; import com.intellij.vcs.log.util.PersistentUtil;
import com.intellij.vcs.log.util.StopWatch; import com.intellij.vcs.log.util.StopWatch;
import com.intellij.vcs.log.util.TroveUtil; import com.intellij.vcs.log.util.TroveUtil;
@@ -68,10 +67,7 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
@NotNull private final VcsUserRegistryImpl myUserRegistry; @NotNull private final VcsUserRegistryImpl myUserRegistry;
@NotNull private final Set<VirtualFile> myRoots; @NotNull private final Set<VirtualFile> myRoots;
@NotNull private final PersistentMap<Integer, String> myMessagesIndex; @Nullable private final MyIndexStorage myIndexStorage;
@Nullable private final VcsLogMessagesTrigramIndex myTrigramIndex;
@Nullable private final VcsLogUserIndex myUserIndex;
@Nullable private final VcsLogPathsIndex myPathsIndex;
@NotNull private final SingleTaskController<IndexingRequest, Void> mySingleTaskController = new MySingleTaskController(); @NotNull private final SingleTaskController<IndexingRequest, Void> mySingleTaskController = new MySingleTaskController();
@NotNull private final Map<VirtualFile, AtomicInteger> myNumberOfTasks = ContainerUtil.newHashMap(); @NotNull private final Map<VirtualFile, AtomicInteger> myNumberOfTasks = ContainerUtil.newHashMap();
@@ -99,12 +95,7 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
myUserRegistry = (VcsUserRegistryImpl)ServiceManager.getService(myProject, VcsUserRegistry.class); myUserRegistry = (VcsUserRegistryImpl)ServiceManager.getService(myProject, VcsUserRegistry.class);
String logId = PersistentUtil.calcLogId(myProject, providers); myIndexStorage = createIndexStorage(fatalErrorsConsumer, PersistentUtil.calcLogId(myProject, providers));
myMessagesIndex = createMap(EnumeratorStringDescriptor.INSTANCE, "messages", logId, 0);
myTrigramIndex = createIndex(() -> new VcsLogMessagesTrigramIndex(logId, fatalErrorsConsumer, this));
myUserIndex = createIndex(() -> new VcsLogUserIndex(logId, myUserRegistry, fatalErrorsConsumer, this));
myPathsIndex = createIndex(() -> new VcsLogPathsIndex(logId, myRoots, fatalErrorsConsumer, this));
for (VirtualFile root : myRoots) { for (VirtualFile root : myRoots) {
myNumberOfTasks.put(root, new AtomicInteger()); myNumberOfTasks.put(root, new AtomicInteger());
@@ -113,14 +104,10 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
Disposer.register(disposableParent, this); Disposer.register(disposableParent, this);
} }
public static int getVersion() { protected MyIndexStorage createIndexStorage(@NotNull FatalErrorHandler fatalErrorHandler, @NotNull String logId) {
return VcsLogStorageImpl.VERSION + VERSION;
}
@Nullable
private <I extends VcsLogFullDetailsIndex> I createIndex(@NotNull ThrowableComputable<I, IOException> computable) {
try { try {
return computable.compute(); return IOUtil.openCleanOrResetBroken(() -> new MyIndexStorage(logId, myUserRegistry, myRoots, fatalErrorHandler, this),
() -> MyIndexStorage.cleanup(logId));
} }
catch (IOException e) { catch (IOException e) {
myFatalErrorsConsumer.consume(this, e); myFatalErrorsConsumer.consume(this, e);
@@ -128,17 +115,8 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
return null; return null;
} }
@NotNull public static int getVersion() {
private <V> PersistentMap<Integer, V> createMap(@NotNull KeyDescriptor<V> descriptor, return VcsLogStorageImpl.VERSION + VERSION;
@NotNull String kind,
@NotNull String logId, int version) {
try {
return PersistentUtil.createPersistentHashMap(descriptor, kind, logId, version);
}
catch (IOException e) {
myFatalErrorsConsumer.consume(this, e);
return new InMemoryMap<>();
}
} }
@Override @Override
@@ -155,14 +133,17 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
} }
private void storeDetails(@NotNull List<? extends VcsFullCommitDetails> details, boolean flush) { private void storeDetails(@NotNull List<? extends VcsFullCommitDetails> details, boolean flush) {
if (myIndexStorage == null) return;
try { try {
for (VcsFullCommitDetails detail : details) { for (VcsFullCommitDetails detail : details) {
int index = myHashMap.getCommitIndex(detail.getId(), detail.getRoot()); int index = myHashMap.getCommitIndex(detail.getId(), detail.getRoot());
myMessagesIndex.put(index, detail.getFullMessage()); myIndexStorage.messages.put(index, detail.getFullMessage());
if (myTrigramIndex != null) myTrigramIndex.update(index, detail); myIndexStorage.trigrams.update(index, detail);
if (myUserIndex != null) myUserIndex.update(index, detail); myIndexStorage.users.update(index, detail);
if (myPathsIndex != null) myPathsIndex.update(index, detail); myIndexStorage.paths.update(index, detail);
myIndexStorage.commits.put(index);
} }
if (flush) { if (flush) {
flush(); flush();
@@ -175,10 +156,13 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
private void flush() { private void flush() {
try { try {
myMessagesIndex.force(); if (myIndexStorage != null) {
if (myTrigramIndex != null) myTrigramIndex.flush(); myIndexStorage.messages.force();
if (myUserIndex != null) myUserIndex.flush(); myIndexStorage.trigrams.flush();
if (myPathsIndex != null) myPathsIndex.flush(); myIndexStorage.users.flush();
myIndexStorage.paths.flush();
myIndexStorage.commits.flush();
}
} }
catch (StorageException e) { catch (StorageException e) {
myFatalErrorsConsumer.consume(this, e); myFatalErrorsConsumer.consume(this, e);
@@ -186,19 +170,13 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
} }
public void markCorrupted() { public void markCorrupted() {
if (myMessagesIndex instanceof PersistentHashMap) ((PersistentHashMap)myMessagesIndex).markCorrupted(); if (myIndexStorage != null) myIndexStorage.commits.markCorrupted();
if (myTrigramIndex != null) myTrigramIndex.markCorrupted();
if (myUserIndex != null) myUserIndex.markCorrupted();
if (myPathsIndex != null) myPathsIndex.markCorrupted();
} }
@Override @Override
public boolean isIndexed(int commit) { public boolean isIndexed(int commit) {
try { try {
return myMessagesIndex.get(commit) != null && return myIndexStorage == null || myIndexStorage.commits.contains(commit);
(myUserIndex == null || myUserIndex.isIndexed(commit)) &&
(myPathsIndex == null || myPathsIndex.isIndexed(commit)) &&
(myTrigramIndex == null || myTrigramIndex.isIndexed(commit));
} }
catch (IOException e) { catch (IOException e) {
myFatalErrorsConsumer.consume(this, e); myFatalErrorsConsumer.consume(this, e);
@@ -225,6 +203,7 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
@NotNull @NotNull
private <T> TIntHashSet filter(@NotNull PersistentMap<Integer, T> map, @NotNull Condition<T> condition) { private <T> TIntHashSet filter(@NotNull PersistentMap<Integer, T> map, @NotNull Condition<T> condition) {
TIntHashSet result = new TIntHashSet(); TIntHashSet result = new TIntHashSet();
if (myIndexStorage == null) return result;
try { try {
Processor<Integer> processor = integer -> { Processor<Integer> processor = integer -> {
try { try {
@@ -241,11 +220,11 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
} }
return true; return true;
}; };
if (myMessagesIndex instanceof PersistentHashMap) { if (myIndexStorage.messages instanceof PersistentHashMap) {
((PersistentHashMap<Integer, T>)myMessagesIndex).processKeysWithExistingMapping(processor); ((PersistentHashMap<Integer, T>)myIndexStorage.messages).processKeysWithExistingMapping(processor);
} }
else { else {
myMessagesIndex.processKeys(processor); myIndexStorage.messages.processKeys(processor);
} }
} }
catch (IOException e) { catch (IOException e) {
@@ -257,9 +236,9 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
@NotNull @NotNull
private TIntHashSet filterUsers(@NotNull Set<VcsUser> users) { private TIntHashSet filterUsers(@NotNull Set<VcsUser> users) {
if (myUserIndex != null) { if (myIndexStorage != null) {
try { try {
return myUserIndex.getCommitsForUsers(users); return myIndexStorage.users.getCommitsForUsers(users);
} }
catch (IOException | StorageException e) { catch (IOException | StorageException e) {
myFatalErrorsConsumer.consume(this, e); myFatalErrorsConsumer.consume(this, e);
@@ -270,9 +249,9 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
@NotNull @NotNull
private TIntHashSet filterPaths(@NotNull Collection<FilePath> paths) { private TIntHashSet filterPaths(@NotNull Collection<FilePath> paths) {
if (myPathsIndex != null) { if (myIndexStorage != null) {
try { try {
return myPathsIndex.getCommitsForPaths(paths); return myIndexStorage.paths.getCommitsForPaths(paths);
} }
catch (IOException | StorageException e) { catch (IOException | StorageException e) {
myFatalErrorsConsumer.consume(this, e); myFatalErrorsConsumer.consume(this, e);
@@ -283,20 +262,20 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
@NotNull @NotNull
public TIntHashSet filterMessages(@NotNull String text) { public TIntHashSet filterMessages(@NotNull String text) {
if (myTrigramIndex != null) { if (myIndexStorage != null) {
try { try {
if (VcsLogUtil.isRegexp(text)) { if (VcsLogUtil.isRegexp(text)) {
Pattern pattern = Pattern.compile(text); Pattern pattern = Pattern.compile(text);
return filter(myMessagesIndex, message -> pattern.matcher(message).find()); return filter(myIndexStorage.messages, message -> pattern.matcher(message).find());
} }
else { else {
ValueContainer.IntIterator commitsForSearch = myTrigramIndex.getCommitsForSubstring(text); ValueContainer.IntIterator commitsForSearch = myIndexStorage.trigrams.getCommitsForSubstring(text);
if (commitsForSearch != null) { if (commitsForSearch != null) {
TIntHashSet result = new TIntHashSet(); TIntHashSet result = new TIntHashSet();
while (commitsForSearch.hasNext()) { while (commitsForSearch.hasNext()) {
int commit = commitsForSearch.next(); int commit = commitsForSearch.next();
try { try {
String value = myMessagesIndex.get(commit); String value = myIndexStorage.messages.get(commit);
if (value != null) { if (value != null) {
if (StringUtil.containsIgnoreCase(value, text)) { if (StringUtil.containsIgnoreCase(value, text)) {
result.add(commit); result.add(commit);
@@ -315,18 +294,20 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
catch (StorageException e) { catch (StorageException e) {
myFatalErrorsConsumer.consume(this, e); myFatalErrorsConsumer.consume(this, e);
} }
return filter(myIndexStorage.messages, message -> StringUtil.containsIgnoreCase(message, text));
} }
return filter(myMessagesIndex, message -> StringUtil.containsIgnoreCase(message, text)); return EmptyIntHashSet.INSTANCE;
} }
@Override @Override
public boolean canFilter(@NotNull List<VcsLogDetailsFilter> filters) { public boolean canFilter(@NotNull List<VcsLogDetailsFilter> filters) {
if (filters.isEmpty()) return false; if (filters.isEmpty() || myIndexStorage == null) return false;
for (VcsLogDetailsFilter filter : filters) { for (VcsLogDetailsFilter filter : filters) {
if (filter instanceof VcsLogTextFilter && myTrigramIndex != null || if (filter instanceof VcsLogTextFilter ||
filter instanceof VcsLogUserFilterImpl && myUserIndex != null || filter instanceof VcsLogUserFilterImpl ||
filter instanceof VcsLogStructureFilter && myPathsIndex != null) { filter instanceof VcsLogStructureFilter) {
continue; continue;
} }
return false; return false;
@@ -366,49 +347,61 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
@Override @Override
public void dispose() { public void dispose() {
try {
myMessagesIndex.close();
}
catch (IOException e) {
LOG.warn(e);
}
} }
public void printDebugInfoForCommit(@NotNull CommitId commitId) { private static class MyIndexStorage {
int commit = myHashMap.getCommitIndex(commitId.getHash(), commitId.getRoot()); private static final String COMMITS = "commits";
if (!isIndexed(commit)) { private static final String MESSAGES = "messages";
LOG.info("Commit " + commitId.getHash().asString() + " is not indexed."); private static final int MESSAGES_VERSION = 0;
} @NotNull private final PersistentSet<Integer> commits;
else { @NotNull private final PersistentMap<Integer, String> messages;
StringBuilder builder = new StringBuilder(); @NotNull private final VcsLogMessagesTrigramIndex trigrams;
try { @NotNull private final VcsLogUserIndex users;
builder.append("Commit ").append(commitId.getHash().asString()).append(" index info:\n"); @NotNull private final VcsLogPathsIndex paths;
builder.append("Message:\n").append(myMessagesIndex.get(commit)).append("\n"); public MyIndexStorage(@NotNull String logId,
if (myTrigramIndex != null) { @NotNull VcsUserRegistryImpl userRegistry,
builder.append("Trigrams:\n").append(myTrigramIndex.getTrigramInfo(commit)).append("\n"); @NotNull Set<VirtualFile> roots,
} @NotNull FatalErrorHandler fatalErrorHandler,
else { @NotNull Disposable parentDisposable)
builder.append("Trigrams index is null"); throws IOException {
} Disposable disposable = Disposer.newDisposable();
if (myUserIndex != null) { Disposer.register(parentDisposable, disposable);
builder.append("User:\n").append(myUserIndex.getUserInfo(commit)).append("\n");
} try {
else { commits =
builder.append("User index is null"); PersistentUtil.createPersistentSetOrFailIfBroken(EnumeratorIntegerDescriptor.INSTANCE, COMMITS, logId, getVersion());
} Disposer.register(disposable, () -> catchAndWarn(commits::close));
if (myPathsIndex != null) { messages = new PersistentHashMap<>(PersistentUtil.getStorageFile(MESSAGES, logId, MESSAGES_VERSION), new IntInlineKeyDescriptor(),
builder.append("Paths:\n").append(myPathsIndex.getPathInfo(commit)).append("\n"); EnumeratorStringDescriptor.INSTANCE, Page.PAGE_SIZE);
} Disposer.register(disposable, () -> catchAndWarn(messages::close));
else {
builder.append("Paths index is null"); trigrams = new VcsLogMessagesTrigramIndex(logId, fatalErrorHandler, disposable);
} users = new VcsLogUserIndex(logId, userRegistry, fatalErrorHandler, disposable);
paths = new VcsLogPathsIndex(logId, roots, fatalErrorHandler, disposable);
}
catch (Throwable t) {
Disposer.dispose(disposable);
throw t;
}
}
private static void catchAndWarn(@NotNull ThrowableRunnable<IOException> runnable) {
try {
runnable.run();
} }
catch (IOException e) { catch (IOException e) {
LOG.error(e); LOG.warn(e);
} }
}
LOG.info(builder.toString()); private static void cleanup(@NotNull String logId) {
IOUtil.deleteAllFilesStartingWith(PersistentUtil.getStorageFile(COMMITS, logId, getVersion()));
IOUtil.deleteAllFilesStartingWith(PersistentUtil.getStorageFile(MESSAGES, logId, MESSAGES_VERSION));
VcsLogMessagesTrigramIndex.getStorageFiles(logId).forEach(IOUtil::deleteAllFilesStartingWith);
VcsLogUserIndex.getStorageFiles(logId).forEach(IOUtil::deleteAllFilesStartingWith);
VcsLogPathsIndex.getStorageFiles(logId).forEach(IOUtil::deleteAllFilesStartingWith);
} }
} }
platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogUserIndex.java
+12 -13
View File
@@ -31,26 +31,33 @@ import gnu.trove.THashMap;
import gnu.trove.TIntHashSet; import gnu.trove.TIntHashSet;
import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.NotNull;
import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.Collection; import java.util.Collection;
import java.util.Collections;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import static com.intellij.vcs.log.data.index.VcsLogPersistentIndex.getVersion;
public class VcsLogUserIndex extends VcsLogFullDetailsIndex<Void> { public class VcsLogUserIndex extends VcsLogFullDetailsIndex<Void> {
private static final Logger LOG = Logger.getInstance(VcsLogUserIndex.class); private static final Logger LOG = Logger.getInstance(VcsLogUserIndex.class);
private static final String USERS = "users";
@NotNull private final VcsUserRegistryImpl myUserRegistry; @NotNull private final VcsUserRegistryImpl myUserRegistry;
public VcsLogUserIndex(@NotNull String logId, public VcsLogUserIndex(@NotNull String logId,
@NotNull VcsUserRegistryImpl userRegistry, @NotNull VcsUserRegistryImpl userRegistry,
@NotNull FatalErrorHandler consumer, @NotNull FatalErrorHandler consumer,
@NotNull Disposable disposableParent) throws IOException { @NotNull Disposable disposableParent) throws IOException {
super(logId, "users", VcsLogPersistentIndex.getVersion(), new UserIndexer(userRegistry), ScalarIndexExtension.VOID_DATA_EXTERNALIZER, super(logId, USERS, getVersion(), new UserIndexer(userRegistry), ScalarIndexExtension.VOID_DATA_EXTERNALIZER,
consumer, disposableParent); consumer, disposableParent);
myUserRegistry = userRegistry; myUserRegistry = userRegistry;
((UserIndexer)myIndexer).setFatalErrorConsumer(e -> { ((UserIndexer)myIndexer).setFatalErrorConsumer(e -> consumer.consume(this, e));
consumer.consume(this, e); }
markCorrupted();
}); @NotNull
public static Collection<File> getStorageFiles(@NotNull String logId) {
return Collections.singletonList(getStorageFile(USERS, logId, getVersion()));
} }
public TIntHashSet getCommitsForUsers(@NotNull Set<VcsUser> users) throws IOException, StorageException { public TIntHashSet getCommitsForUsers(@NotNull Set<VcsUser> users) throws IOException, StorageException {
@@ -61,14 +68,6 @@ public class VcsLogUserIndex extends VcsLogFullDetailsIndex<Void> {
return getCommitsWithAnyKey(ids); return getCommitsWithAnyKey(ids);
} }
@NotNull
public String getUserInfo(int commit) throws IOException {
Collection<Integer> keys = getKeysForCommit(commit);
assert keys != null;
assert keys.size() == 1;
return ObjectUtils.assertNotNull(myUserRegistry.getUserById(ObjectUtils.assertNotNull(ContainerUtil.getFirstItem(keys)))).toString();
}
private static class UserIndexer implements DataIndexer<Integer, Void, VcsFullCommitDetails> { private static class UserIndexer implements DataIndexer<Integer, Void, VcsFullCommitDetails> {
@NotNull private final VcsUserRegistryImpl myRegistry; @NotNull private final VcsUserRegistryImpl myRegistry;
@NotNull private Consumer<Exception> myFatalErrorConsumer = LOG::error; @NotNull private Consumer<Exception> myFatalErrorConsumer = LOG::error;
platform/vcs-log/impl/src/com/intellij/vcs/log/ui/actions/PrintIndexInfoAction.java
-88
View File
@@ -1,88 +0,0 @@
/*
* Copyright 2000-2016 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.vcs.log.ui.actions;
import com.intellij.openapi.actionSystem.AnActionEvent;
import com.intellij.openapi.actionSystem.Presentation;
import com.intellij.openapi.project.DumbAwareAction;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.ui.MessageType;
import com.intellij.openapi.util.registry.Registry;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.openapi.vcs.ui.VcsBalloonProblemNotifier;
import com.intellij.vcs.log.CommitId;
import com.intellij.vcs.log.VcsLog;
import com.intellij.vcs.log.VcsLogDataKeys;
import com.intellij.vcs.log.data.VcsLogData;
import com.intellij.vcs.log.data.index.VcsLogIndex;
import com.intellij.vcs.log.data.index.VcsLogPersistentIndex;
import com.intellij.vcs.log.impl.VcsProjectLog;
import java.util.List;
public class PrintIndexInfoAction extends DumbAwareAction {
@Override
public void update(AnActionEvent e) {
Presentation presentation = e.getPresentation();
if (!Registry.is("vcs.log.highlight.not.indexed")) {
presentation.setEnabledAndVisible(false);
}
else {
Project project = e.getProject();
VcsLog log = e.getData(VcsLogDataKeys.VCS_LOG);
if (project == null || log == null) {
presentation.setEnabledAndVisible(false);
}
else {
VcsLogData dataManager = VcsProjectLog.getInstance(project).getDataManager();
if (dataManager == null) {
presentation.setEnabledAndVisible(false);
}
else {
VcsLogIndex index = dataManager.getIndex();
if (!(index instanceof VcsLogPersistentIndex)) {
presentation.setEnabledAndVisible(false);
}
else {
presentation.setEnabledAndVisible(!log.getSelectedCommits().isEmpty());
}
}
}
}
}
@Override
public void actionPerformed(AnActionEvent e) {
Project project = e.getProject();
VcsLog log = e.getData(VcsLogDataKeys.VCS_LOG);
if (project == null || log == null) {
return;
}
VcsLogData dataManager = VcsProjectLog.getInstance(project).getDataManager();
if (dataManager == null) return;
VcsLogIndex index = dataManager.getIndex();
if (!(index instanceof VcsLogPersistentIndex)) return;
List<CommitId> commits = log.getSelectedCommits();
for (CommitId commit : commits) {
((VcsLogPersistentIndex)index).printDebugInfoForCommit(commit);
}
VcsBalloonProblemNotifier.showOverChangesView(project, "Index information for " +
commits.size() + " " +
StringUtil.pluralize("commit", commits.size()) +
" was written into log file", MessageType.INFO);
}
}
platform/vcs-log/impl/src/com/intellij/vcs/log/util/PersistentUtil.java
+5 -20
View File
@@ -75,27 +75,12 @@ public class PersistentUtil {
} }
@NotNull @NotNull
public static <T> PersistentSet<T> createPersistentSet(@NotNull KeyDescriptor<T> keyDescriptor, public static <T> PersistentSet<T> createPersistentSetOrFailIfBroken(@NotNull KeyDescriptor<T> keyDescriptor,
@NotNull String storageKind, @NotNull String storageKind,
@NotNull String logId, @NotNull String logId,
int version) throws IOException { int version) throws IOException {
File storageFile = getStorageFile(storageKind, logId, version); File storageFile = getStorageFile(storageKind, logId, version);
return new PersistentSetImpl<>(storageFile, keyDescriptor, Page.PAGE_SIZE, null, version);
return IOUtil.openCleanOrResetBroken(() ->
new PersistentSetImpl<>(storageFile, keyDescriptor, Page.PAGE_SIZE, null, version),
storageFile);
}
@NotNull
public static <V> PersistentHashMap<Integer, V> createPersistentHashMap(@NotNull DataExternalizer<V> externalizer,
@NotNull String storageKind,
@NotNull String logId,
int version) throws IOException {
File storageFile = getStorageFile(storageKind, logId, version);
return IOUtil.openCleanOrResetBroken(() ->
new PersistentHashMap<>(storageFile, new IntInlineKeyDescriptor(), externalizer, Page.PAGE_SIZE),
storageFile);
} }
@NotNull @NotNull