MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-22 04:43:02 +07:00
Code Issues Packages Projects Releases Wiki Activity

- process deleted files same way as regular ones in order to process index data invalidation during bulk indexing (IDEA-147138)

Browse Source 
- limiting disk access with nested executor services
This commit is contained in:
Maxim.Mossienko
2015-11-19 15:37:23 +01:00
parent 6d27da842f
commit 347d468af7
5 changed files with 128 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/lang-impl/src/com/intellij/util/indexing/DeletedVirtualFileStub.java
+67
View File
@@ -0,0 +1,67 @@
/*
* Copyright 2000-2015 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.util.indexing;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.openapi.vfs.VirtualFileWithId;
import com.intellij.testFramework.LightVirtualFile;
import org.jetbrains.annotations.NotNull;
/**
* Created by Maxim.Mossienko on 11/18/2015.
*/
class DeletedVirtualFileStub extends LightVirtualFile implements VirtualFileWithId {
private final int myFileId;
DeletedVirtualFileStub(VirtualFileWithId original) {
setOriginalFile((VirtualFile)original);
myFileId = -Math.abs(original.getId());
}
@Override
public int getId() {
return myFileId;
}
@Override
public boolean isValid() {
return false;
}
@Override
public boolean equals(Object obj) {
if (obj instanceof DeletedVirtualFileStub) {
return ((DeletedVirtualFileStub)obj).getId() == myFileId;
}
return super.equals(obj);
}
@Override
public int hashCode() {
return myFileId;
}
@Override
public String toString() {
return "invalid:" + getOriginalFile().toString();
}
@NotNull
@Override
public String getUrl() {
return "invalid:" + super.getUrl();
}
}
platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java
+45 -91
View File
@@ -86,7 +86,6 @@ import java.io.*;
import java.lang.ref.SoftReference;
import java.lang.ref.WeakReference;
import java.util.*;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
@@ -674,7 +673,11 @@ public class FileBasedIndexImpl extends FileBasedIndex {
finally {
LOG.info("START INDEX SHUTDOWN");
try {
myChangedFilesCollector.ensureAllInvalidateTasksCompleted();
for(VirtualFile file:myChangedFilesCollector.getAllFilesToUpdate()) {
if (!file.isValid()) {
removeDataFromIndicesForFile(file);
}
}
IndexingStamp.flushCaches();
for (ID<?, ?> indexId : myIndices.keySet()) {
@@ -696,6 +699,19 @@ public class FileBasedIndexImpl extends FileBasedIndex {
}
}
private void removeDataFromIndicesForFile(VirtualFile file) {
final int fileId = Math.abs(getIdMaskingNonIdBasedFile(file));
final List<ID<?, ?>> states = IndexingStamp.getNontrivialFileIndexedStates(fileId);
if (!states.isEmpty()) {
ProgressManager.getInstance().executeNonCancelableSection(new Runnable() {
@Override
public void run() {
myChangedFilesCollector.removeFileDataFromIndices(states, fileId);
}
});
}
}
private void flushAllIndices(final long modCount) {
if (HeavyProcessLatch.INSTANCE.isRunning()) {
return;
@@ -825,7 +841,6 @@ public class FileBasedIndexImpl extends FileBasedIndex {
myReentrancyGuard.set(Boolean.TRUE);
try {
myChangedFilesCollector.tryToEnsureAllInvalidateTasksCompleted();
if (isUpToDateCheckEnabled()) {
try {
if (ourRebuildStatus.get(indexId).get() != OK) {
@@ -1585,10 +1600,6 @@ public class FileBasedIndexImpl extends FileBasedIndex {
return pair.getSecond();
}
public int getNumberOfPendingInvalidations() {
return myChangedFilesCollector.getNumberOfPendingInvalidations();
}
public int getChangedFileCount() {
return myChangedFilesCollector.getAllFilesToUpdate().size();
}
@@ -1598,6 +1609,9 @@ public class FileBasedIndexImpl extends FileBasedIndex {
return ContainerUtil.findAll(myChangedFilesCollector.getAllFilesToUpdate(), new Condition<VirtualFile>() {
@Override
public boolean value(VirtualFile virtualFile) {
if (virtualFile instanceof DeletedVirtualFileStub) {
return true;
}
for (IndexableFileSet set : myIndexableSets) {
final Project proj = myIndexableSetToProjectMap.get(set);
if (proj != null && !proj.equals(project)) {
@@ -1617,7 +1631,6 @@ public class FileBasedIndexImpl extends FileBasedIndex {
}
void processRefreshedFile(@NotNull Project project, @NotNull final com.intellij.ide.caches.FileContent fileContent) {
myChangedFilesCollector.tryToEnsureAllInvalidateTasksCompleted();
myChangedFilesCollector.processFileImpl(project, fileContent); // ProcessCanceledException will cause re-adding the file to processing list
}
@@ -1627,12 +1640,15 @@ public class FileBasedIndexImpl extends FileBasedIndex {
// in this case we consider that current indexing (out of roots backed CacheUpdater) will cover its content
// todo this assumption isn't correct for vfs events happened between content loading and indexing itself
// proper fix will when events handling will be out of direct execution by EDT
doIndexFileContent(project, content);
if (file.isValid()) {
doIndexFileContent(project, content);
} else {
removeDataFromIndicesForFile(file);
}
myChangedFilesCollector.myFilesToUpdate.remove(file);
}
private void doIndexFileContent(@Nullable Project project, @NotNull com.intellij.ide.caches.FileContent content) {
myChangedFilesCollector.tryToEnsureAllInvalidateTasksCompleted();
final VirtualFile file = content.getVirtualFile();
FileType fileType = file.getFileType();
@@ -1736,7 +1752,7 @@ public class FileBasedIndexImpl extends FileBasedIndex {
static final Key<Boolean> ourPhysicalContentKey = Key.create("physical.content.flag");
private void updateSingleIndex(@NotNull ID<?, ?> indexId, final int inputId, @Nullable FileContent currentFC)
void updateSingleIndex(@NotNull ID<?, ?> indexId, final int inputId, @Nullable FileContent currentFC)
throws StorageException {
if (ourRebuildStatus.get(indexId).get() == REQUIRES_REBUILD && !myIsUnitTestMode) {
return; // the index is scheduled for rebuild, no need to update
@@ -1878,7 +1894,6 @@ public class FileBasedIndexImpl extends FileBasedIndex {
private final class ChangedFilesCollector extends VirtualFileAdapter implements BulkFileListener {
private final Set<VirtualFile> myFilesToUpdate = ContainerUtil.newConcurrentSet();
private final Queue<InvalidationTask> myFutureInvalidations = new ConcurrentLinkedQueue<InvalidationTask>();
private final ManagingFS myManagingFS = ManagingFS.getInstance();
@@ -2095,19 +2110,13 @@ public class FileBasedIndexImpl extends FileBasedIndex {
}
}
else if (!fileIndexedStatesToUpdate.isEmpty()) { // file was removed, its data should be (lazily) wiped for every index
final Collection<ID<?, ?>> finalFileIndexedStatesToUpdate = fileIndexedStatesToUpdate;
myFutureInvalidations.offer(new InvalidationTask(file) {
@Override
public void run() {
removeFileDataFromIndices(finalFileIndexedStatesToUpdate, fileId);
}
});
myFilesToUpdate.add(new DeletedVirtualFileStub((VirtualFileWithId)file));
}
IndexingStamp.flushCache(fileId);
}
private void removeFileDataFromIndices(@NotNull Collection<ID<?, ?>> affectedIndices, int inputId) {
void removeFileDataFromIndices(@NotNull Collection<ID<?, ?>> affectedIndices, int inputId) {
Throwable unexpectedError = null;
for (ID<?, ?> indexId : affectedIndices) {
try {
@@ -2133,59 +2142,6 @@ public class FileBasedIndexImpl extends FileBasedIndex {
}
}
public int getNumberOfPendingInvalidations() {
return myFutureInvalidations.size();
}
public void ensureAllInvalidateTasksCompleted() {
ensureAllInvalidateTasksCompleted(false);
}
public void tryToEnsureAllInvalidateTasksCompleted() {
ensureAllInvalidateTasksCompleted(true);
}
private void ensureAllInvalidateTasksCompleted(boolean doCheckCancelledBetweenInvalidations) {
final int size = getNumberOfPendingInvalidations();
if (size == 0) {
return;
}
if (doCheckCancelledBetweenInvalidations) {
while (true) {
InvalidationTask task = myFutureInvalidations.poll();
if (task == null) {
break;
}
ProgressManager.getInstance().executeNonCancelableSection(task);
ProgressManager.checkCanceled();
}
}
else {
ProgressManager.getInstance().executeNonCancelableSection(
new Runnable() {
@Override
public void run() {
final ProgressIndicator indicator = ProgressManager.getInstance().getProgressIndicator();
indicator.setText("");
int count = 0;
while (true) {
InvalidationTask task = myFutureInvalidations.poll();
if (task == null) {
break;
}
indicator.setFraction((double)count++ / size);
task.run();
}
}
}
);
}
}
private void iterateIndexableFiles(@NotNull final VirtualFile file, @NotNull final Processor<VirtualFile> processor) {
if (file.isDirectory()) {
final ContentIterator iterator = new ContentIterator() {
@@ -2217,9 +2173,7 @@ public class FileBasedIndexImpl extends FileBasedIndex {
private final VirtualFileUpdateTask myForceUpdateTask = new VirtualFileUpdateTask();
private final AtomicInteger myForceUpdateRequests = new AtomicInteger();
private void forceUpdate(@Nullable Project project, @Nullable final GlobalSearchScope filter, @Nullable final VirtualFile restrictedTo) {
myChangedFilesCollector.tryToEnsureAllInvalidateTasksCompleted();
void forceUpdate(@Nullable Project project, @Nullable final GlobalSearchScope filter, @Nullable final VirtualFile restrictedTo) {
Collection<VirtualFile> allFilesToUpdate = getAllFilesToUpdate();
if (!allFilesToUpdate.isEmpty()) {
@@ -2242,21 +2196,18 @@ public class FileBasedIndexImpl extends FileBasedIndex {
private boolean processFileImpl(Project project, @NotNull final com.intellij.ide.caches.FileContent fileContent) {
final VirtualFile file = fileContent.getVirtualFile();
if (myFilesToUpdate.contains(file)) {
if (file.isValid()) {
int fileId = getIdMaskingNonIdBasedFile(file);
try {
if (isTooLarge(file)) {
List<ID<?, ?>> nontrivialFileIndexedStates = IndexingStamp.getNontrivialFileIndexedStates(fileId);
removeFileDataFromIndices(ContainerUtil.intersection(nontrivialFileIndexedStates, myRequiringContentIndices), Math.abs(fileId));
}
else {
doIndexFileContent(project, fileContent);
}
try {
if (!file.isValid() || isTooLarge(file)) {
removeDataFromIndicesForFile(file);
}
finally {
IndexingStamp.flushCache(file);
else {
doIndexFileContent(project, fileContent);
}
}
finally {
IndexingStamp.flushCache(file);
}
myFilesToUpdate.remove(file);
return true;
}
@@ -2509,16 +2460,19 @@ public class FileBasedIndexImpl extends FileBasedIndex {
@Override
public void removeIndexableSet(@NotNull IndexableFileSet set) {
if (!myIndexableSetToProjectMap.containsKey(set)) return;
myChangedFilesCollector.ensureAllInvalidateTasksCompleted();
IndexingStamp.flushCaches();
myIndexableSets.remove(set);
myIndexableSetToProjectMap.remove(set);
for (VirtualFile file : myChangedFilesCollector.getAllFilesToUpdate()) {
if (getIndexableSetForFile(file) == null) {
if (!file.isValid()) {
removeDataFromIndicesForFile(file);
myChangedFilesCollector.myFilesToUpdate.remove(file);
} else if (getIndexableSetForFile(file) == null) { // todo remove data from indices for removed
myChangedFilesCollector.myFilesToUpdate.remove(file);
}
}
IndexingStamp.flushCaches();
}
@Override
platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexProjectHandler.java
+1 -1
View File
@@ -142,7 +142,7 @@ public class FileBasedIndexProjectHandler extends AbstractProjectComponent imple
}
final FileBasedIndexImpl index = (FileBasedIndexImpl)i;
if (index.getChangedFileCount() + index.getNumberOfPendingInvalidations() < 20) {
if (index.getChangedFileCount() < 20) {
return null;
}
platform/lang-impl/src/com/intellij/util/indexing/ProjectFilesCondition.java
+6 -1
View File
@@ -42,12 +42,17 @@ class ProjectFilesCondition implements Condition<VirtualFile> {
@Override
public boolean value(VirtualFile file) {
if (myIndexableFilesFilter != null && !myIndexableFilesFilter.containsFileId(((VirtualFileWithId)file).getId())) {
int fileId = ((VirtualFileWithId)file).getId();
if (myIndexableFilesFilter != null && fileId > 0 && !myIndexableFilesFilter.containsFileId(fileId)) {
if (myFilesFromOtherProjects >= MAX_FILES_TO_UPDATE_FROM_OTHER_PROJECT) return false;
++myFilesFromOtherProjects;
return true;
}
if (fileId < 0 && file instanceof DeletedVirtualFileStub) {
//file = ((FileBasedIndexImpl.MyLightVirtualFile)file).getOriginalFile();
return true;
}
if (FileBasedIndexImpl.belongsToScope(file, myRestrictedTo, myFilter)) return true;
if (myFilesFromOtherProjects < MAX_FILES_TO_UPDATE_FROM_OTHER_PROJECT) {
platform/platform-impl/src/com/intellij/openapi/project/CacheUpdateRunner.java
+9 -2
View File
@@ -31,8 +31,10 @@ import com.intellij.openapi.util.Key;
import com.intellij.openapi.util.registry.Registry;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.util.Consumer;
import com.intellij.util.concurrency.BoundedTaskExecutor;
import gnu.trove.THashSet;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.ide.PooledThreadExecutor;
import java.util.Collection;
import java.util.Set;
@@ -92,6 +94,11 @@ public class CacheUpdateRunner {
}
}
private static final BoundedTaskExecutor ourCacheUpdateExecutor = new BoundedTaskExecutor(
PooledThreadExecutor.INSTANCE,
indexingThreadCount()
);
private static boolean processSomeFilesWhileUserIsInactive(@NotNull FileContentQueue queue,
@NotNull Consumer<VirtualFile> progressUpdater,
final boolean processInReadAction,
@@ -131,7 +138,7 @@ public class CacheUpdateRunner {
AtomicBoolean ref = new AtomicBoolean();
finishedRefs[i] = ref;
Runnable process = new MyRunnable(innerIndicator, queue, ref, progressUpdater, processInReadAction, project, fileProcessor);
futures[i] = ApplicationManager.getApplication().executeOnPooledThread(process);
futures[i] = ourCacheUpdateExecutor.submit(process);
}
isFinished.set(waitForAll(finishedRefs, futures));
}
@@ -221,7 +228,7 @@ public class CacheUpdateRunner {
final VirtualFile file = fileContent.getVirtualFile();
try {
myProgressUpdater.consume(file);
if (file.isValid() && !file.isDirectory() && !Boolean.TRUE.equals(file.getUserData(FAILED_TO_INDEX))) {
if (!file.isDirectory() && !Boolean.TRUE.equals(file.getUserData(FAILED_TO_INDEX))) {
myProcessor.consume(fileContent);
}
}