diff --git a/java/java-impl/src/com/intellij/psi/RefQueueIndex.java b/java/java-impl/src/com/intellij/psi/RefQueueIndex.java new file mode 100644 index 000000000000..8691781301b6 --- /dev/null +++ b/java/java-impl/src/com/intellij/psi/RefQueueIndex.java @@ -0,0 +1,119 @@ +/* + * Copyright 2000-2014 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.psi; + +import com.intellij.openapi.project.Project; +import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.psi.impl.file.impl.ResolveScopeManagerImpl; +import com.intellij.util.indexing.*; +import com.intellij.util.io.DataExternalizer; +import com.intellij.util.io.KeyDescriptor; +import org.jetbrains.annotations.NotNull; + +import java.io.DataInput; +import java.io.DataOutput; +import java.util.Collections; +import java.util.Map; + +// all it does is take files it was fed and queue them to the resolve +public class RefQueueIndex extends FileBasedIndexExtension { + private static final ID ID = com.intellij.util.indexing.ID.create("RefQueueIndex"); + + @NotNull + @Override + public ID getName() { + return ID; + } + + @NotNull + @Override + public DataIndexer getIndexer() { + return new DataIndexer() { + @NotNull + @Override + public Map map(@NotNull FileContent inputData) { + if (ResolveScopeManagerImpl.ENABLED_REF_BACK) { + Project project = inputData.getProject(); + RefResolveService.getInstance(project).queue(Collections.singletonList(inputData.getFile()), "Cache updater"); + } + return Collections.emptyMap(); + } + }; + } + + @NotNull + @Override + public KeyDescriptor getKeyDescriptor() { + return new KeyDescriptor() { + @Override + public void save(@NotNull DataOutput out, Void value) { + + } + + @Override + public Void read(@NotNull DataInput in) { + return null; + } + + @Override + public int getHashCode(Void value) { + return 0; + } + + @Override + public boolean isEqual(Void val1, Void val2) { + return false; + } + }; + } + + @NotNull + @Override + public DataExternalizer getValueExternalizer() { + return new DataExternalizer() { + @Override + public void save(@NotNull DataOutput out, Void value) { + + } + + @Override + public Void read(@NotNull DataInput in) { + return null; + } + }; + } + + @NotNull + @Override + public FileBasedIndex.InputFilter getInputFilter() { + return new FileBasedIndex.InputFilter() { + @Override + public boolean acceptInput(@NotNull VirtualFile file) { + return true; + } + }; + } + + @Override + public boolean dependsOnFileContent() { + return true; + } + + @Override + public int getVersion() { + return 0; + } +} diff --git a/java/java-impl/src/com/intellij/psi/RefResolveServiceImpl.java b/java/java-impl/src/com/intellij/psi/RefResolveServiceImpl.java new file mode 100644 index 000000000000..b0afc8829b01 --- /dev/null +++ b/java/java-impl/src/com/intellij/psi/RefResolveServiceImpl.java @@ -0,0 +1,743 @@ +/* + * Copyright 2000-2014 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.psi; + +import com.intellij.concurrency.JobLauncher; +import com.intellij.openapi.Disposable; +import com.intellij.openapi.application.ApplicationAdapter; +import com.intellij.openapi.application.ApplicationManager; +import com.intellij.openapi.application.PathManager; +import com.intellij.openapi.application.ex.ApplicationEx; +import com.intellij.openapi.application.ex.ApplicationUtil; +import com.intellij.openapi.diagnostic.Logger; +import com.intellij.openapi.fileTypes.StdFileTypes; +import com.intellij.openapi.module.Module; +import com.intellij.openapi.progress.ProcessCanceledException; +import com.intellij.openapi.progress.ProgressIndicator; +import com.intellij.openapi.progress.Task; +import com.intellij.openapi.progress.util.ProgressIndicatorUtils; +import com.intellij.openapi.project.DumbService; +import com.intellij.openapi.project.IndexNotReadyException; +import com.intellij.openapi.project.Project; +import com.intellij.openapi.roots.ProjectFileIndex; +import com.intellij.openapi.startup.StartupManager; +import com.intellij.openapi.util.Computable; +import com.intellij.openapi.util.Disposer; +import com.intellij.openapi.util.io.FileUtil; +import com.intellij.openapi.util.text.StringUtil; +import com.intellij.openapi.vfs.*; +import com.intellij.openapi.vfs.newvfs.BulkFileListener; +import com.intellij.openapi.vfs.newvfs.NewVirtualFile; +import com.intellij.openapi.vfs.newvfs.RefreshQueueImpl; +import com.intellij.openapi.vfs.newvfs.events.VFileEvent; +import com.intellij.openapi.vfs.newvfs.persistent.FSRecords; +import com.intellij.openapi.vfs.newvfs.persistent.PersistentFS; +import com.intellij.psi.impl.PersistentIntList; +import com.intellij.psi.impl.file.impl.ResolveScopeManagerImpl; +import com.intellij.psi.search.GlobalSearchScope; +import com.intellij.psi.util.PsiUtilCore; +import com.intellij.util.ArrayUtil; +import com.intellij.util.ExceptionUtil; +import com.intellij.util.Function; +import com.intellij.util.Processor; +import com.intellij.util.containers.ConcurrentBitSet; +import com.intellij.util.containers.ConcurrentIntObjectMap; +import com.intellij.util.containers.ContainerUtil; +import com.intellij.util.containers.StripedLockIntObjectConcurrentHashMap; +import com.intellij.util.io.storage.HeavyProcessLatch; +import com.intellij.util.messages.MessageBus; +import gnu.trove.*; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.jetbrains.jps.model.java.JavaSourceRootType; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.text.DateFormat; +import java.util.*; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +public class RefResolveServiceImpl extends RefResolveService implements Runnable, Disposable { + private static final Logger LOG = Logger.getInstance("#com.intellij.psi.RefResolveService"); + private final AtomicInteger fileCount = new AtomicInteger(); + private final AtomicLong bytesSize = new AtomicLong(); + private final AtomicLong refCount = new AtomicLong(); + private final PersistentIntList storage; + private final Deque filesToResolve = new ArrayDeque(); + private final ConcurrentBitSet fileIsInQueue = new ConcurrentBitSet(); + private final ConcurrentBitSet fileIsResolved; + private final ApplicationEx myApplication; + private volatile boolean myDisposed; + private volatile boolean upToDate; + private final FileWriter log; + private final ProjectFileIndex myProjectFileIndex; + + + public RefResolveServiceImpl(final Project project, + final MessageBus messageBus, + final PsiManager psiManager, + StartupManager startupManager, + ApplicationEx application, + ProjectFileIndex projectFileIndex) throws IOException { + super(project); + myApplication = application; + myProjectFileIndex = projectFileIndex; + if (ResolveScopeManagerImpl.ENABLED_REF_BACK) { + File indexFile = new File(getStorageDirectory(), "index"); + File dataFile = new File(getStorageDirectory(), "data"); + fileIsResolved = ConcurrentBitSet.readFrom(new File(getStorageDirectory(), "bitSet")); + + final boolean initial = !indexFile.exists() || !dataFile.exists(); + storage = new PersistentIntList(indexFile, dataFile, initial); + Disposer.register(this, storage); + if (!application.isUnitTestMode()) { + startupManager.runWhenProjectIsInitialized(new Runnable() { + @Override + public void run() { + init(messageBus, psiManager); + } + }); + } + log = new FileWriter(new File(getStorageDirectory(), "log.txt")); + Disposer.register(this, new Disposable() { + @Override + public void dispose() { + try { + save(); + log.close(); + } + catch (IOException e) { + LOG.error(e); + } + } + }); + } + else { + log = null; + fileIsResolved = null; + storage = null; + } + } + + public static List toVf(@NotNull int[] ids) { + List res = new ArrayList(); + for (int id : ids) { + VirtualFile file = PersistentFS.getInstance().findFileById(id); + if (file != null) { + res.add(file); + } + } + return res; + } + + public static String toVfString(@NotNull int[] backIds) { + List list = toVf(backIds); + return toVfString(list); + } + + private static String toVfString(@NotNull List list) { + List sub = list.subList(0, Math.min(list.size(), 100)); + return list.size() + " files: " + StringUtil.join(sub, new Function() { + @Override + public String fun(VirtualFile file) { + return file.getName(); + } + }, ", ")+(list.size()==sub.size() ? "" : "..."); + } + + private void init(@NotNull MessageBus messageBus, @NotNull PsiManager psiManager) { + //if (true) { + // upToDate = false; + // return; + //} + messageBus.connect().subscribe(VirtualFileManager.VFS_CHANGES, new BulkFileListener.Adapter(){ + @Override + public void after(@NotNull List events) { + fileCount.set(0); + List files = ContainerUtil.mapNotNull(events, new Function() { + @Override + public VirtualFile fun(VFileEvent event) { + return event.getFile(); + } + }); + queue(files, "VFS events " + events.size()); + } + }); + psiManager.addPsiTreeChangeListener(new PsiTreeChangeAdapter() { + @Override + public void childrenChanged(@NotNull PsiTreeChangeEvent event) { + PsiFile file = event.getFile(); + VirtualFile virtualFile = PsiUtilCore.getVirtualFile(file); + if (virtualFile != null) { + queue(Collections.singletonList(virtualFile), event); + } + } + + @Override + public void propertyChanged(@NotNull PsiTreeChangeEvent event) { + childrenChanged(event); + } + }); + + messageBus.connect().subscribe(DumbService.DUMB_MODE, new DumbService.DumbModeListener() { + @Override + public void enteredDumbMode() { + wakeUp(); + } + + @Override + public void exitDumbMode() { + wakeUp(); + } + }); + myApplication.addApplicationListener(new ApplicationAdapter() { + @Override + public void writeActionFinished(Object action) { + wakeUp(); + } + }, this); + VirtualFileManager.getInstance().addVirtualFileManagerListener(new VirtualFileManagerListener() { + @Override + public void beforeRefreshStart(boolean asynchronous) { + wakeUp(); + } + + @Override + public void afterRefreshFinish(boolean asynchronous) { + wakeUp(); + } + }, this); + + startThread(); + } + + // return true if file was added to queue + private boolean queueIfNeeded(VirtualFile virtualFile, @NotNull Project project) { + return toResolve(virtualFile, project) && queueUpdate(virtualFile); + } + + private boolean toResolve(VirtualFile virtualFile, @NotNull Project project) { + if (virtualFile != null && virtualFile.isValid() && + project.isInitialized() && + myProjectFileIndex.isContentSourceFile(virtualFile) && + (virtualFile.isDirectory() || virtualFile.getFileType() == StdFileTypes.JAVA)) { + return true; + } + + // else mark it as resolved so we will not have to check it again + if (virtualFile instanceof VirtualFileWithId) { + int id = getAbsId(virtualFile); + fileIsResolved.set(id); + } + + return false; + } + + @NotNull + private File getStorageDirectory() { + String dirName = myProject.getName() + "."+Integer.toHexString(myProject.getPresentableUrl().hashCode()); + File dir = new File(PathManager.getSystemPath(), "refs/" + dirName); + FileUtil.createDirectory(dir); + return dir; + } + + + private void log(String m) { + //System.out.println(m); + logf(m); + } + + private void logf(String m) { + try { + log.write(DateFormat.getDateTimeInstance().format(new Date()) + " "+m+" ; gap="+storage.gap+"\n"); + } + catch (IOException e) { + LOG.error(e); + } + } + + private void flushLog() { + try { + log.flush(); + } + catch (IOException e) { + LOG.error(e); + } + } + + // return true if file was added to queue + private boolean queueUpdate(@NotNull VirtualFile file) { + synchronized (filesToResolve) { + if (!(file instanceof VirtualFileWithId)) return false; + int fileId = getAbsId(file); + countAndMarkUnresolved(file, new int[1]); + boolean alreadyAdded = fileIsInQueue.set(fileId); + if (!alreadyAdded) { + filesToResolve.add(file); + } + upToDate = false; + wakeUpUnderLock(); + return !alreadyAdded; + } + } + + private void wakeUp() { + synchronized (filesToResolve) { + wakeUpUnderLock(); + } + } + + private void wakeUpUnderLock() { + filesToResolve.notifyAll(); + } + + private void waitForQueue() throws InterruptedException { + synchronized (filesToResolve) { + filesToResolve.wait(1000); + } + } + + private void startThread() { + new Thread(this, "Ref resolve service").start(); + upToDate = true; + queueUnresolvedFilesSinceLastRestart(); + } + + private void queueUnresolvedFilesSinceLastRestart() { + PersistentFS fs = PersistentFS.getInstance(); + int maxId = FSRecords.getMaxId(); + TIntArrayList list = new TIntArrayList(); + for (int id= fileIsResolved.nextClearBit(1); id >= 0 && id < maxId; id = fileIsResolved.nextClearBit(id + 1)) { + int nextSetBit = fileIsResolved.nextSetBit(id); + int endOfRun = Math.min(maxId, nextSetBit == -1 ? maxId : nextSetBit); + do { + VirtualFile virtualFile = fs.findFileById(id); + if (queueIfNeeded(virtualFile, myProject)) { + list.add(id); + } + } + while (++id < endOfRun); + } + log("Initially added to resolve " + toVfString(list.toNativeArray())); + } + + @Override + public void dispose() { + myDisposed = true; + } + + private void save() throws IOException { + fileIsResolved.writeTo(new File(getStorageDirectory(), "bitSet")); + } + + @Override + public void run() { + while (!myDisposed) { + if (!hasSomething()) { + try { + waitForQueue(); + } + catch (InterruptedException e) { + break; + } + continue; + } + upToDate = false; + final CountDownLatch batchProcessedLatch = new CountDownLatch(1); + ApplicationManager.getApplication().invokeLater(new Runnable() { + @Override + public void run() { + new Task.Backgroundable(myProject, "Resolving files...", true) { + @Override + public void run(@NotNull final ProgressIndicator indicator) { + if (ApplicationManager.getApplication().isDisposed()) return; + try { + processBatch(indicator); + } + finally { + batchProcessedLatch.countDown(); + } + } + }.queue(); + } + }, myProject.getDisposed()); + + try { + batchProcessedLatch.await(); + } + catch (InterruptedException e) { + break; + } + + synchronized (filesToResolve) { + upToDate = filesToResolve.isEmpty(); + log("upToDate = " + upToDate); + } + flushLog(); + } + } + + private void processBatch(@NotNull final ProgressIndicator indicator) { + Set set; + int queuedSize; + synchronized (filesToResolve) { + queuedSize = filesToResolve.size(); + set = new THashSet(queuedSize); + // someone might have cleared this bit to mark file as processed + for (VirtualFile file : filesToResolve) { + if (fileIsInQueue.clear(getAbsId(file))) { + set.add(file); + } + } + filesToResolve.clear(); + } + final Set toProcess = Collections.synchronizedSet(set); + final ConcurrentIntObjectMap fileToForwardIds = new StripedLockIntObjectConcurrentHashMap(); + final int size = countAndMarkUnresolved(set); + if (size == 0) return; + log("Started to resolve "+ size + " files (was queued "+queuedSize+")"); + + indicator.setIndeterminate(false); + ProgressIndicatorUtils.forceWriteActionPriority(indicator, (Disposable)indicator); + long start = System.currentTimeMillis(); + Processor processor = new Processor() { + @Override + public boolean process(VirtualFile file) { + double fraction = 1 - toProcess.size() * 1.0 / size; + indicator.setFraction(fraction); + try { + VfsUtilCore.visitChildrenRecursively(file, new VirtualFileVisitor() { + @Override + public boolean visitFile(@NotNull VirtualFile file) { + if (!toResolve(file, myProject)) { + return true; + } + int fileId = getAbsId(file); + int i = size - toProcess.size(); + indicator.setText(i + "/" + size + ": Resolving " + file.getPresentableUrl()); + int[] forwardIds = processFile(file, fileId, indicator); + if (forwardIds == null) { + //queueUpdate(file); + return false; + } + toProcess.remove(file); + fileToForwardIds.put(fileId, forwardIds); + return true; + } + + @Nullable + @Override + public Iterable getChildrenIterable(@NotNull VirtualFile file) { + return ((NewVirtualFile)file).iterInDbChildren(); + } + }, RuntimeException.class); + } + catch (RuntimeException e) { + indicator.checkCanceled(); + } + return true; + } + }; + boolean success = true; + try { + success = JobLauncher + .getInstance().invokeConcurrentlyUnderProgress(new ArrayList(set), indicator, false, false, processor); + } + finally { + queue(toProcess, "re-added after fail. success=" + success); + storeIds(fileToForwardIds); + + long end = System.currentTimeMillis(); + log("Resolved batch of " + (size - toProcess.size()) + " from " + size + " files in " + ((end - start) / 1000) + "sec. (Gap: " + storage.gap+")"); + } + } + + private static int getAbsId(@NotNull VirtualFile file) { + return Math.abs(((VirtualFileWithId)file).getId()); + } + + private int countAndMarkUnresolved(@NotNull Collection files) { + final int[] count = new int[1]; + for (VirtualFile file : files) { + countAndMarkUnresolved(file, count); + } + return count[0]; + } + + private void countAndMarkUnresolved(@NotNull VirtualFile file, @NotNull final int[] count) { + if (file.isDirectory()) { + VfsUtilCore.visitChildrenRecursively(file, new VirtualFileVisitor() { + @Override + public boolean visitFile(@NotNull VirtualFile file) { + doCountAndMarkUnresolved(file, count); + return true; + } + + @Nullable + @Override + public Iterable getChildrenIterable(@NotNull VirtualFile file) { + return ((NewVirtualFile)file).iterInDbChildren(); + } + }); + } + else { + doCountAndMarkUnresolved(file, count); + } + } + + private void doCountAndMarkUnresolved(@NotNull VirtualFile file, @NotNull int[] count) { + if (toResolve(file, myProject)) { + count[0]++; + fileIsResolved.clear(getAbsId(file)); + } + } + + private boolean hasSomething() { + if (DumbService.isDumb(myProject) || + myApplication.isWriteActionInProgress() || + RefreshQueueImpl.isRefreshInProgress() || + HeavyProcessLatch.INSTANCE.isRunning()) { + return false; + } + synchronized (filesToResolve) { + return !filesToResolve.isEmpty(); + } + } + + // returns list of resolved files if updated successfully, or null if write action or dumb mode started + private int[] processFile(@NotNull final VirtualFile file, + int fileId, + @NotNull final ProgressIndicator indicator) { + final TIntHashSet forward; + try { + forward = calcForwardRefs(file, indicator); + } + catch (IndexNotReadyException e) { + return null; + } + catch (ApplicationUtil.CannotRunReadActionException e) { + return null; + } + catch (ProcessCanceledException e) { + throw e; + } + catch (Exception e) { + log(ExceptionUtil.getThrowableText(e)); + flushLog(); + return null; + } + + int[] forwardIds = forward.toArray(); + fileIsResolved.set(fileId); + logf(" ---- "+file.getPresentableUrl() + " processed. forwardIds: "+ toVfString(forwardIds)); + return forwardIds; + } + + private void storeIds(@NotNull ConcurrentIntObjectMap fileToForwardIds) { + int forwardSize = 0; + int backwardSize = 0; + final TIntObjectHashMap fileToBackwardIds = new TIntObjectHashMap(fileToForwardIds.size()); + for (StripedLockIntObjectConcurrentHashMap.IntEntry entry : fileToForwardIds.entries()) { + int fileId = entry.getKey(); + int[] forwardIds = entry.getValue(); + forwardSize += forwardIds.length; + for (int forwardId : forwardIds) { + TIntArrayList backIds = fileToBackwardIds.get(forwardId); + if (backIds == null) { + backIds = new TIntArrayList(); + fileToBackwardIds.put(forwardId, backIds); + } + backIds.add(fileId); + backwardSize++; + } + } + log("backwardSize = " + backwardSize); + log("forwardSize = " + forwardSize); + log("fileToForwardIds.size() = "+fileToForwardIds.size()); + log("fileToBackwardIds.size() = "+fileToBackwardIds.size()); + assert forwardSize == backwardSize; + + // wrap in read action so that sudden quit (in write action) would not interrupt us + ApplicationManager.getApplication().runReadAction(new Runnable() { + @Override + public void run() { + fileToBackwardIds.forEachEntry(new TIntObjectProcedure() { + @Override + public boolean execute(int fileId, TIntArrayList backIds) { + storage.addAll(fileId, backIds.toNativeArray()); + return true; + } + }); + } + }); + } + + + @NotNull + private TIntHashSet calcForwardRefs(@NotNull final VirtualFile virtualFile, @NotNull final ProgressIndicator indicator) + throws IndexNotReadyException, ApplicationUtil.CannotRunReadActionException { + if (myProject.isDisposed()) throw new ProcessCanceledException(); + if (fileCount.incrementAndGet() % 100 == 0) { + PsiManager.getInstance(myProject).dropResolveCaches(); + synchronized (storage) { + storage.flush(); + } + try { + log.flush(); + } + catch (IOException e) { + LOG.error(e); + } + } + + final TIntHashSet forward = new TIntHashSet(); + + final PsiFile psiFile = ApplicationUtil.tryRunReadAction(new Computable() { + @Override + public PsiFile compute() { + return PsiManager.getInstance(myProject).findFile(virtualFile); + } + }); + final int fileId = getAbsId(virtualFile); + if (psiFile != null) { + bytesSize.addAndGet(virtualFile.getLength()); + final Set resolved = new THashSet(); + ApplicationUtil.tryRunReadAction(new Runnable() { + @Override + public void run() { + indicator.checkCanceled(); + + psiFile.accept(new JavaRecursiveElementWalkingVisitor() { + @Override + public void visitReferenceElement(PsiJavaCodeReferenceElement reference) { + indicator.checkCanceled(); + PsiElement element = reference.resolve(); + if (element != null) { + resolved.add(element); + } + refCount.incrementAndGet(); + + super.visitReferenceElement(reference); + } + }); + } + }); + ApplicationUtil.tryRunReadAction(new Runnable() { + @Override + public void run() { + indicator.checkCanceled(); + for (PsiElement element : resolved) { + PsiFile file = element.getContainingFile(); + addIdAndSuperClasses(file, forward); + } + } + }); + } + + forward.remove(fileId); + return forward; + } + + private static void addIdAndSuperClasses(PsiFile file, @NotNull TIntHashSet forward) { + if (file instanceof PsiJavaFile && file.getName().equals("Object.class") && ((PsiJavaFile)file).getPackageName().equals("java.lang")) { + return; + } + VirtualFile virtualFile = PsiUtilCore.getVirtualFile(file); + if (virtualFile instanceof VirtualFileWithId && forward.add(getAbsId(virtualFile)) && file instanceof PsiClassOwner) { + for (PsiClass aClass : ((PsiClassOwner)file).getClasses()) { + for (PsiClass superClass : aClass.getSupers()) { + addIdAndSuperClasses(superClass.getContainingFile(), forward); + } + } + } + } + + @Override + @Nullable + public int[] getBackwardIds(@NotNull VirtualFileWithId file) { + if (!upToDate) return null; + int fileId = getAbsId((VirtualFile)file); + return storage.get(fileId); + } + + private String prevLog = ""; + private static final Set SOURCE_ROOTS = ContainerUtil.newTroveSet(JavaSourceRootType.SOURCE, JavaSourceRootType.TEST_SOURCE); + + @NotNull + @Override + public GlobalSearchScope restrictByBackwardIds(@NotNull final VirtualFile virtualFile, @NotNull GlobalSearchScope scope) { + final int[] backIds = RefResolveService.getInstance(myProject).getBackwardIds((VirtualFileWithId)virtualFile); + if (backIds == null) { + return scope; + } + String files = toVfString(backIds); + String log = "Restricting scope of " + virtualFile.getName() + " to " + files; + if (!log.equals(prevLog)) { + log(log); + flushLog(); + prevLog = log; + } + GlobalSearchScope restrictedByBackwardIds = new GlobalSearchScope() { + @Override + public boolean contains(@NotNull VirtualFile file) { + if (!(file instanceof VirtualFileWithId) + || file.equals(virtualFile) + || ArrayUtil.indexOf(backIds, getAbsId(file)) != -1) return true; + return false & !myProjectFileIndex.isUnderSourceRootOfType(file, SOURCE_ROOTS); // filter out source file which we know for sure does not reference the element + } + + @Override + public int compare(@NotNull VirtualFile file1, @NotNull VirtualFile file2) { + return 0; + } + + @Override + public boolean isSearchInModuleContent(@NotNull Module aModule) { + return true; + } + + @Override + public boolean isSearchInLibraries() { + return false; + } + }; + return scope.intersectWith(restrictedByBackwardIds); + } + + @Override + public boolean queue(@NotNull Collection files, Object reason) { + if (files.isEmpty()) { + return false; + } + boolean queued = false; + List added = new ArrayList(files.size()); + for (VirtualFile file : files) { + boolean wasAdded = queueIfNeeded(file, myProject); + if (wasAdded) { + added.add(file); + } + queued |= wasAdded; + } + if (queued) { + log("Queued to resolve (from " + reason + "): " + toVfString(added)); + flushLog(); + } + return queued; + } +} diff --git a/platform/indexing-api/src/com/intellij/psi/RefResolveService.java b/platform/indexing-api/src/com/intellij/psi/RefResolveService.java new file mode 100644 index 000000000000..f53316506e78 --- /dev/null +++ b/platform/indexing-api/src/com/intellij/psi/RefResolveService.java @@ -0,0 +1,50 @@ +/* + * Copyright 2000-2014 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.psi; + +import com.intellij.openapi.components.AbstractProjectComponent; +import com.intellij.openapi.project.Project; +import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.openapi.vfs.VirtualFileWithId; +import com.intellij.psi.search.GlobalSearchScope; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.Collection; + +public abstract class RefResolveService extends AbstractProjectComponent { + public RefResolveService(Project project) { + super(project); + } + + public static RefResolveService getInstance(Project project) { + return project.getComponent(RefResolveService.class); + } + + @Nullable("null means the service has not resolved all files and is not ready yet") + public abstract int[] getBackwardIds(@NotNull VirtualFileWithId file); + + /** + * @return subset of scope containing only files which reference the virtualFile + */ + @NotNull + public abstract GlobalSearchScope restrictByBackwardIds(@NotNull VirtualFile virtualFile, @NotNull GlobalSearchScope scope); + + /** + * @return add files to the resolve queue. until all files from there are resolved, the service is in incomplete state and returns null from getBackwardIds() + */ + public abstract boolean queue(@NotNull Collection files, Object reason); +} diff --git a/platform/indexing-impl/src/com/intellij/psi/impl/PersistentIntList.java b/platform/indexing-impl/src/com/intellij/psi/impl/PersistentIntList.java new file mode 100644 index 000000000000..79487d1bad3e --- /dev/null +++ b/platform/indexing-impl/src/com/intellij/psi/impl/PersistentIntList.java @@ -0,0 +1,366 @@ +/* + * Copyright 2000-2014 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.psi.impl; + +import com.intellij.openapi.Disposable; +import com.intellij.openapi.util.Ref; +import com.intellij.openapi.util.io.FileUtil; +import com.intellij.util.ArrayUtil; +import com.intellij.util.io.Bits; +import com.intellij.util.io.IntToIntBtree; +import com.intellij.util.io.PagedFileStorage; +import com.intellij.util.io.RandomAccessDataFile; +import gnu.trove.TIntHashSet; +import gnu.trove.TIntIntHashMap; +import gnu.trove.TIntIntProcedure; +import org.jetbrains.annotations.NotNull; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; + +/** + * the (int -> int[]) map which is persisted to the specified file. + */ +public class PersistentIntList implements Disposable { + public static final int MAX_DATA_BYTES = 500000000; + public static final int MAX_LIST_LENGTH = 100000; + private final IntToIntBtree index; + private RandomAccessDataFile data; + public int gap; // bytes lost due to fragmentation + private final int dataStart; // offset of real data; the bytes before are reserved for 'index' meta information, see persistsVarsTo() + + public PersistentIntList(@NotNull File indexFile, @NotNull File dataFile, boolean initial) throws IOException { + if (initial) { + FileUtil.writeToFile(dataFile, ArrayUtil.EMPTY_BYTE_ARRAY); + } + PagedFileStorage.StorageLockContext context = new PagedFileStorage.StorageLockContext(true); + context.lock(); + try { + data = new RandomAccessDataFile(dataFile); + index = new IntToIntBtree(4096, indexFile, context, initial); + dataStart = persistsVarsTo(data, initial); + } + finally { + context.unlock(); + } + } + + private int persistsVarsTo(@NotNull final RandomAccessDataFile data, boolean toDisk) { + return index.persistVars(new IntToIntBtree.BtreeDataStorage() { + @Override + public int persistInt(int offset, int value, boolean toDisk) { + if (toDisk) { + data.putInt(offset, value); + return value; + } + else { + return data.getInt(offset); + } + } + }, toDisk); + } + + @Override + public void dispose() { + index.withStorageLock(new Runnable() { + @Override + public void run() { + try { + persistsVarsTo(data, true); + index.doClose(); + } + catch (IOException e) { + throw new RuntimeException(e); + } + data.dispose(); + } + }); + } + + @NotNull + public int[] get(final int id) { + final Ref res = new Ref(); + + index.withStorageLock(new Runnable() { + @Override + public void run() { + final int[] ptrPtr = new int[1]; + boolean exists = index.get(id, ptrPtr); + if (!exists) { + ptrPtr[0] = 0; + } + int pointer = ptrPtr[0]; + if (pointer == 0) { + res.set(ArrayUtil.EMPTY_INT_ARRAY); + } + else { + assertPointer(pointer); + int listLength = data.getInt(pointer); + int capacity = data.getInt(pointer + 4); + assertListLength(listLength, capacity); + int[] result = new int[listLength]; + byte[] bytes = new byte[listLength * 4]; + data.get(pointer + 8, bytes, 0, bytes.length); + for (int i = 0; i < listLength; i++) { + result[i] = Bits.getInt(bytes, i*4); + } + res.set(result); + } + } + }); + return res.get(); + } + + // return true if was added + public boolean add(final int id, final int value) { + assert value > 0; + assert id > 0; + final boolean[] added = new boolean[1]; + index.withStorageLock(new Runnable() { + @Override + public void run() { + int[] ptrPtr = new int[1]; + index.get(id, ptrPtr); + final int pointer = ptrPtr[0]; + int[] stored; + int capacity; + final int listLength; + if (pointer == 0) { + stored = ArrayUtil.EMPTY_INT_ARRAY; + listLength = 0; + capacity = 2; + } + else { + assertPointer(pointer); + listLength = data.getInt(pointer); + capacity = data.getInt(pointer+4); + assertListLength(listLength,capacity); + stored = new int[listLength]; + for (int i = 0; i < listLength; i++) { + int v = data.getInt(pointer + (i + 2) * 4); + stored[i] = v; + if (v == value) return; + } + // append + if (capacity > listLength /*|| data.length() == pointer + 4 + 4 + 4*capacity*/) { + data.putInt(pointer + (listLength + 2) * 4, value); + data.putInt(pointer, listLength + 1); + if (capacity <= listLength) { + data.putInt(pointer+4, capacity + 1); + } + added[0] = true; + return; + } + // reallocate + gap += 4 + 4 + 4 * capacity; + } + + int storePointer = (int)data.length(); + data.putInt(storePointer, stored.length + 1); + int newCapacity = capacity < 10 ? capacity * 2 : (int)(capacity * 1.5); + assert newCapacity > stored.length + 1; + data.putInt(storePointer+4, newCapacity); + for (int i = 0; i < stored.length; i++) { + int v = stored[i]; + data.putInt(storePointer + (i+2)*4, v); + } + data.putInt(storePointer + (stored.length+2)*4, value); + for (int i = stored.length + 1; i < newCapacity; i++) { + data.putInt(storePointer + (i+2)*4, 0); // gap + } + index.put(id, storePointer); + if (storePointer > 10000000) { + int i = 0; + } + added[0] = true; + } + }); + + return added[0]; + } + + private static void assertListLength(int listLength, int capacity) { + assert 0 < listLength && listLength <= MAX_LIST_LENGTH : listLength; + assert 0 < capacity && capacity <= MAX_LIST_LENGTH : capacity; + assert capacity >= listLength : listLength + ", " + capacity; + assert capacity <= (listLength+1)*2 : listLength + ", " + capacity; + } + + public void addAll(final int id, @NotNull final int[] values) { + assertListLength(values.length, values.length); + assert id > 0; + Arrays.sort(values); + + index.withStorageLock(new Runnable() { + @Override + public void run() { + int[] ptrPtr = new int[1]; + index.get(id, ptrPtr); + final int pointer = ptrPtr[0]; + int capacity; + final int newListLength; + byte[] mergedBytes; + + if (pointer == 0) { + mergedBytes = toBytes(values); + newListLength = values.length; + capacity = 0; + } + else { + assertPointer(pointer); + int storedListLength = data.getInt(pointer); + capacity = data.getInt(pointer + 4); + assertListLength(storedListLength, capacity); + // try to merge inplace and if failed, reallocate at the end + byte[] storedBytes = new byte[storedListLength * 4]; + data.get(pointer + 8, storedBytes, 0, storedListLength * 4); + + mergedBytes = new byte[storedBytes.length + values.length * 4]; + int outPtr = 0; + int i = 0; + int j = 0; + while (i < storedListLength || j < values.length) { + int stored = i < storedListLength ? Bits.getInt(storedBytes, i * 4) : Integer.MAX_VALUE; + int value = j < values.length ? values[j] : Integer.MAX_VALUE; + if (stored < value) { + Bits.putInt(mergedBytes, outPtr, stored); + outPtr += 4; + i++; + } + else if (stored > value) { + Bits.putInt(mergedBytes, outPtr, value); + outPtr += 4; + j++; + } + else { + Bits.putInt(mergedBytes, outPtr, value); + outPtr += 4; + j++; + i++; + } + } + newListLength = outPtr / 4; + assertListLength(newListLength, newListLength); + if (newListLength <= capacity) { + storeArray(data, pointer, newListLength, capacity, mergedBytes); + return; + } + gap += capacity * 4 + 8; + } + // reallocate at the end + + int storePointer = (int)data.length(); + assertPointer(storePointer); + int oldCapacity = Math.max(capacity, newListLength); + int newCapacity = oldCapacity < 10 ? (oldCapacity + 1) * 2 : (int)(oldCapacity * 1.5); + assert newCapacity > newListLength + 1; + storeArray(data, storePointer, newListLength, newCapacity, mergedBytes); + index.put(id, storePointer); + } + }); + + int[] ids = get(id); + for (int i = 1; i < ids.length; i++) { + assert ids[i] > ids[i - 1] : ids[i-1] + ", " + ids[i]; + } + TIntHashSet set = new TIntHashSet(ids); + assert set.containsAll(values): "ids: "+Arrays.toString(ids)+";\n values:"+Arrays.toString(values); + } + + private static byte[] toBytes(@NotNull int[] values) { + byte[] mergedBytes = new byte[4 * values.length]; + for (int i = 0; i < values.length; i++) { + int value = values[i]; + Bits.putInt(mergedBytes, i * 4, value); + } + return mergedBytes; + } + + private static void storeArray(@NotNull RandomAccessDataFile data, + int storePointer, + int newListLength, + int newCapacity, + @NotNull byte[] mergedBytes) { + assertListLength(newListLength, newCapacity); + data.putInt(storePointer, newListLength); + data.putInt(storePointer + 4, newCapacity); + data.put(storePointer + 8, mergedBytes, 0, newListLength * 4); + byte[] fill = new byte[(newCapacity - newListLength) * 4]; + Arrays.fill(fill, (byte)-1); + data.put(storePointer + 8 + newListLength * 4, fill, 0, fill.length); + } + + private static void assertPointer(int pointer) { + assert 0 < pointer && pointer <= MAX_DATA_BYTES : pointer; + } + + public void flush() { + index.withStorageLock(new Runnable() { + @Override + public void run() { + persistsVarsTo(data, true); + index.doFlush(); + data.sync(); + //data.force(); + } + }); + } + + private void compactIfNecessary() { + if (gap < data.length() / 2) return; + index.withStorageLock(new Runnable() { + @Override + public void run() { + persistsVarsTo(data, true); + index.doFlush(); + data.sync(); + + try { + final RandomAccessDataFile newData = new RandomAccessDataFile(new File(data.getFile().getParentFile(), "newData")); + persistsVarsTo(newData, true); + final TIntIntHashMap map = new TIntIntHashMap(); + index.processMappings(new IntToIntBtree.KeyValueProcessor() { + @Override + public boolean process(int key, int value) throws IOException { + map.put(key, value); + return true; + } + }); + map.forEachEntry(new TIntIntProcedure() { + @Override + public boolean execute(int key, int value) { + int[] ids = get(key); + int pointer = (int)newData.length(); + byte[] bytes = toBytes(ids); + storeArray(newData, pointer, ids.length, (int)(ids.length * 1.3), bytes); + index.put(key, pointer); + return true; + } + }); + + data.dispose(); + data = newData; + gap = 0; + flush(); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + }); + } +} diff --git a/platform/indexing-impl/src/com/intellij/psi/impl/file/impl/ResolveScopeManagerImpl.java b/platform/indexing-impl/src/com/intellij/psi/impl/file/impl/ResolveScopeManagerImpl.java index b89042b68257..15112d3a100e 100644 --- a/platform/indexing-impl/src/com/intellij/psi/impl/file/impl/ResolveScopeManagerImpl.java +++ b/platform/indexing-impl/src/com/intellij/psi/impl/file/impl/ResolveScopeManagerImpl.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2013 JetBrains s.r.o. + * Copyright 2000-2014 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,6 +22,7 @@ import com.intellij.openapi.project.Project; import com.intellij.openapi.roots.*; import com.intellij.openapi.roots.impl.LibraryScopeCache; import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.openapi.vfs.VirtualFileWithId; import com.intellij.psi.*; import com.intellij.psi.impl.PsiManagerImpl; import com.intellij.psi.impl.ResolveScopeManager; @@ -37,14 +38,17 @@ import java.util.List; import java.util.Map; public class ResolveScopeManagerImpl extends ResolveScopeManager { - + /** + * if true then getUseScope() returns scope restricted to only relevant files which are stored in {@link RefResolveService} + */ + public static final boolean ENABLED_REF_BACK = /*ApplicationManager.getApplication().isUnitTestMode() ||*/ Boolean.getBoolean("ref.back"); private final Project myProject; private final ProjectRootManager myProjectRootManager; private final PsiManager myManager; private final Map myDefaultResolveScopesCache = new ConcurrentFactoryMap() { @Override - protected GlobalSearchScope create(VirtualFile key) { + protected GlobalSearchScope create(@NotNull VirtualFile key) { GlobalSearchScope scope = null; for(ResolveScopeProvider resolveScopeProvider: ResolveScopeProvider.EP_NAME.getExtensions()) { scope = resolveScopeProvider.getResolveScope(key, myProject); @@ -180,34 +184,37 @@ public class ResolveScopeManagerImpl extends ResolveScopeManager { @Override @NotNull public GlobalSearchScope getUseScope(@NotNull PsiElement element) { - VirtualFile vFile; + VirtualFile vDirectory; + final VirtualFile virtualFile; + final PsiFile containingFile; final GlobalSearchScope allScope = GlobalSearchScope.allScope(myManager.getProject()); if (element instanceof PsiDirectory) { - vFile = ((PsiDirectory)element).getVirtualFile(); + vDirectory = ((PsiDirectory)element).getVirtualFile(); + virtualFile = null; + containingFile = null; } else { - final PsiFile containingFile = element.getContainingFile(); + containingFile = element.getContainingFile(); if (containingFile == null) return allScope; - final VirtualFile virtualFile = containingFile.getVirtualFile(); + virtualFile = containingFile.getVirtualFile(); if (virtualFile == null) return allScope; - vFile = virtualFile.getParent(); + vDirectory = virtualFile.getParent(); } - if (vFile == null) return allScope; - ProjectFileIndex projectFileIndex = myProjectRootManager.getFileIndex(); - Module module = projectFileIndex.getModuleForFile(vFile); - if (module != null) { - boolean isTest = projectFileIndex.isInTestSourceContent(vFile); - return isTest - ? GlobalSearchScope.moduleTestsWithDependentsScope(module) - : GlobalSearchScope.moduleWithDependentsScope(module); + if (vDirectory == null) return allScope; + final ProjectFileIndex projectFileIndex = myProjectRootManager.getFileIndex(); + final Module module = projectFileIndex.getModuleForFile(vDirectory); + if (module == null) { + return containingFile == null || virtualFile.isDirectory() || allScope.contains(virtualFile) + ? allScope : GlobalSearchScope.fileScope(containingFile).uniteWith(allScope); } - else { - final PsiFile f = element.getContainingFile(); - final VirtualFile vf = f == null ? null : f.getVirtualFile(); - - return f == null || vf == null || vf.isDirectory() || allScope.contains(vf) - ? allScope : GlobalSearchScope.fileScope(f).uniteWith(allScope); + boolean isTest = projectFileIndex.isInTestSourceContent(vDirectory); + GlobalSearchScope scope = isTest + ? GlobalSearchScope.moduleTestsWithDependentsScope(module) + : GlobalSearchScope.moduleWithDependentsScope(module); + if (virtualFile instanceof VirtualFileWithId && ENABLED_REF_BACK) { + return RefResolveService.getInstance(myProject).restrictByBackwardIds(virtualFile, scope); } + return scope; } } diff --git a/platform/platform-resources/src/META-INF/LangExtensions.xml b/platform/platform-resources/src/META-INF/LangExtensions.xml index f309e93b4d6a..89f145caad93 100644 --- a/platform/platform-resources/src/META-INF/LangExtensions.xml +++ b/platform/platform-resources/src/META-INF/LangExtensions.xml @@ -166,8 +166,7 @@ - + @@ -460,6 +459,7 @@ + diff --git a/platform/platform-resources/src/componentSets/Lang.xml b/platform/platform-resources/src/componentSets/Lang.xml index 89a9b361a4f0..dfd2260bf1cb 100644 --- a/platform/platform-resources/src/componentSets/Lang.xml +++ b/platform/platform-resources/src/componentSets/Lang.xml @@ -62,6 +62,10 @@ com.intellij.psi.impl.PsiManagerImpl + + com.intellij.psi.RefResolveService + com.intellij.psi.RefResolveServiceImpl + com.intellij.psi.impl.file.impl.PsiVFSListener com.intellij.psi.impl.file.impl.PsiVFSListener