IDEA-56157 Sort out performance problem with Perforce changelists refresh

IDEA-67375 Files periodically loose their Perforce state

P4 local changes provider refactored.

1)We calculated read-only unversioned only on start; after that, we listen to local changes and invalidate changes
2) [for big amount of edited files] when asking "p4 have" for changed files (to get to know their revisions), keep have commands limited by number of arguments

platform/platform-api/src/com/intellij/execution/process/CapturingProcessHandler.java

@@ -39,7 +39,7 @@ public class CapturingProcessHandler extends OSProcessHandler {
   }
 
   public CapturingProcessHandler(final Process process, final Charset charset, final String commandLine) {
-    super(process, commandLine);
+    super(process, commandLine, charset);
     myCharset = charset;
     addProcessListener(new ProcessAdapter() {
       @Override

platform/platform-api/src/com/intellij/execution/process/OSProcessHandler.java

@@ -22,11 +22,16 @@ import com.intellij.openapi.vfs.encoding.EncodingManager;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
+import java.nio.charset.Charset;
 import java.util.concurrent.Future;
 
 public class OSProcessHandler extends BaseOSProcessHandler {
   private static final Logger LOG = Logger.getInstance("#com.intellij.execution.process.OSProcessHandler");
 
+  public OSProcessHandler(@NotNull final Process process, @Nullable final String commandLine, final Charset defaultCharset) {
+    super(process, commandLine, defaultCharset);
+  }
+
   public OSProcessHandler(@NotNull final Process process, @Nullable final String commandLine) {
     super(process, commandLine, EncodingManager.getInstance().getDefaultCharset());
   }

platform/testFramework/src/com/intellij/testFramework/vcs/MockDirtyScope.java

@@ -110,6 +110,11 @@ public class MockDirtyScope extends VcsModifiableDirtyScope {
     throw new UnsupportedOperationException();
   }
 
+  @Override
+  public void iterateExistingInsideScope(Processor<VirtualFile> vf) {
+    throw new UnsupportedOperationException();
+  }
+
   @Override
   public boolean isEmpty() {
     return myDirtyFiles.isEmpty();

platform/vcs-api/src/com/intellij/openapi/vcs/ProjectLevelVcsManager.java

@@ -42,6 +42,8 @@ import java.util.List;
 public abstract class ProjectLevelVcsManager {
   @NonNls public static final String FILE_VIEW_TOOL_WINDOW_ID = "File View";
 
+  public abstract void iterateVfUnderVcsRoot(VirtualFile file, Processor<VirtualFile> processor);
+
   /**
    * Returns the <code>ProjectLevelVcsManager<code> instance for the specified project.
    *

platform/vcs-api/src/com/intellij/openapi/vcs/changes/ChangeListManagerGate.java

@@ -19,6 +19,7 @@ import com.intellij.openapi.vcs.FileStatus;
 import com.intellij.openapi.vfs.VirtualFile;
 import org.jetbrains.annotations.Nullable;
 
+import java.io.File;
 import java.util.Collection;
 import java.util.List;
 
@@ -40,4 +41,5 @@ public interface ChangeListManagerGate {
   void moveChanges(final String toList, final Collection<Change> changes);
   void setListsToDisappear(final Collection<String> names);
   FileStatus getStatus(final VirtualFile file);
+  FileStatus getStatus(final File file);
 }

platform/vcs-api/src/com/intellij/openapi/vcs/changes/VcsDirtyScope.java

@@ -91,6 +91,8 @@ public abstract class VcsDirtyScope {
    * @param iterator an iterator to invoke
    */
   public abstract void iterate(Processor<FilePath> iterator);
+  
+  public abstract void iterateExistingInsideScope(Processor<VirtualFile> vf);
 
   public abstract boolean isEmpty();
 

platform/vcs-api/src/com/intellij/openapi/vcs/changes/VcsModifiableDirtyScope.java

@@ -17,6 +17,8 @@
 package com.intellij.openapi.vcs.changes;
 
 import com.intellij.openapi.vcs.FilePath;
+import com.intellij.openapi.vfs.VirtualFile;
+import com.intellij.util.Processor;
 import org.jetbrains.annotations.Nullable;
 
 public abstract class VcsModifiableDirtyScope extends VcsDirtyScope {

platform/vcs-impl/src/com/intellij/openapi/vcs/FilePathImpl.java

@@ -20,6 +20,7 @@ import com.intellij.openapi.fileEditor.FileDocumentManager;
 import com.intellij.openapi.fileTypes.FileType;
 import com.intellij.openapi.fileTypes.FileTypeManager;
 import com.intellij.openapi.project.Project;
+import com.intellij.openapi.util.Comparing;
 import com.intellij.openapi.util.io.FileUtil;
 import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.openapi.vfs.LocalFileSystem;
@@ -93,9 +94,15 @@ public class FilePathImpl implements FilePath {
       return false;
     }
     else {
+      if (! isSpecialName(myName) && ! isSpecialName(((FilePath)o).getName()) &&
+        (! Comparing.equal(myName, ((FilePath)o).getName()))) return false;
       return myFile.equals(((FilePath)o).getIOFile());
     }
   }
+  
+  private boolean isSpecialName(final String name) {
+    return ".".equals(name) || "..".equals(name);
+  }
 
   public void refresh() {
     if (!myNonLocal) {

platform/vcs-impl/src/com/intellij/openapi/vcs/changes/ChangeListWorker.java

@@ -505,6 +505,10 @@ public class ChangeListWorker implements ChangeListsWriteOperations {
     return myIdx.getStatus(file);
   }
 
+  public FileStatus getStatus(final File file) {
+    return myIdx.getStatus(file);
+  }
+
   public DeletedFilesHolder getLocallyDeleted() {
     return myLocallyDeleted.copy();
   }
@@ -771,6 +775,11 @@ public class ChangeListWorker implements ChangeListsWriteOperations {
     public FileStatus getStatus(VirtualFile file) {
       return myWorker.getStatus(file);
     }
+
+    @Override
+    public FileStatus getStatus(File file) {
+      return myWorker.getStatus(file);
+    }
   }
 
   // assumes after revisions are all not null

platform/vcs-impl/src/com/intellij/openapi/vcs/changes/ChangeListsIndexes.java

@@ -55,6 +55,10 @@ public class ChangeListsIndexes {
   public FileStatus getStatus(final VirtualFile file) {
     return myFileToStatus.get(new File(file.getPath()).getAbsolutePath());
   }
+  
+  public FileStatus getStatus(final File file) {
+    return myFileToStatus.get(file.getAbsolutePath());
+  }
 
   public void changeAdded(final Change change, final VcsKey key) {
     addChangeToIdx(change, key);

platform/vcs-impl/src/com/intellij/openapi/vcs/changes/VcsDirtyScopeImpl.java

@@ -270,6 +270,35 @@ public class VcsDirtyScopeImpl extends VcsModifiableDirtyScope {
     }
   }
 
+  @Override
+  public void iterateExistingInsideScope(Processor<VirtualFile> processor) {
+    if (myProject.isDisposed()) return;
+
+    for (VirtualFile root : myAffectedContentRoots) {
+      THashSet<FilePath> dirsByRoot = myDirtyDirectoriesRecursively.get(root);
+      if (dirsByRoot != null) {
+        for (FilePath dir : dirsByRoot) {
+          final VirtualFile vFile = dir.getVirtualFile();
+          if (vFile != null && vFile.isValid()) {
+            myVcsManager.iterateVfUnderVcsRoot(vFile, processor);
+          }
+        }
+      }
+    }
+
+    for (FilePath file : myDirtyFiles) {
+      if (file.getVirtualFile() != null) {
+        processor.process(file.getVirtualFile());
+      }
+      final VirtualFile vFile = file.getVirtualFile();
+      if (vFile != null && vFile.isValid() && vFile.isDirectory()) {
+        for (VirtualFile child : vFile.getChildren()) {
+          processor.process(child);
+        }
+      }
+    }
+  }
+
   @Override
   public boolean isEmpty() {
     return myDirtyDirectoriesRecursively.isEmpty() && myDirtyFiles.isEmpty();

platform/vcs-impl/src/com/intellij/openapi/vcs/changes/pending/MockChangeListManagerGate.java

@@ -22,6 +22,7 @@ import com.intellij.openapi.vcs.changes.ChangeListManagerGate;
 import com.intellij.openapi.vcs.changes.LocalChangeList;
 import com.intellij.openapi.vfs.VirtualFile;
 
+import java.io.File;
 import java.util.Collection;
 import java.util.List;
 
@@ -74,4 +75,9 @@ public class MockChangeListManagerGate implements ChangeListManagerGate {
   public FileStatus getStatus(VirtualFile file) {
     return null;
   }
+
+  @Override
+  public FileStatus getStatus(File file) {
+    return null;
+  }
 }

platform/vcs-impl/src/com/intellij/openapi/vcs/impl/ProjectLevelVcsManagerImpl.java

@@ -179,6 +179,11 @@ public class ProjectLevelVcsManagerImpl extends ProjectLevelVcsManagerEx impleme
     return AllVcses.getInstance(myProject).getDescriptor(name);
   }
 
+  @Override
+  public void iterateVfUnderVcsRoot(VirtualFile file, Processor<VirtualFile> processor) {
+    VcsRootIterator.iterateVfUnderVcsRoot(myProject, file, processor);
+  }
+
   public VcsDescriptor[] getAllVcss() {
     return AllVcses.getInstance(myProject).getAll();
   }

platform/vcs-impl/src/com/intellij/openapi/vcs/impl/VcsRootIterator.java

@@ -57,6 +57,11 @@ public class VcsRootIterator {
     return true;
   }
 
+  public static boolean iterateVfUnderVcsRoot(Project project, VirtualFile file, Processor<VirtualFile> processor) {
+    final MyRootIterator rootIterator = new MyRootIterator(project, file, null, processor, null);
+    return rootIterator.iterate();
+  }
+
   private static class MyRootFilter {
     private final VirtualFile myRoot;
     private final String myVcsName;
@@ -102,20 +107,22 @@ public class VcsRootIterator {
 
   public static boolean iterateVcsRoot(final Project project, final VirtualFile root, final Processor<FilePath> processor,
                                        @Nullable PairProcessor<VirtualFile, VirtualFile[]> directoryFilter) {
-    final MyRootIterator rootIterator = new MyRootIterator(project, root, processor, directoryFilter);
+    final MyRootIterator rootIterator = new MyRootIterator(project, root, processor, null, directoryFilter);
     return rootIterator.iterate();
   }
 
   private static class MyRootIterator {
     private final Processor<FilePath> myProcessor;
+    private final Processor<VirtualFile> myVfProcessor;
     @Nullable private final PairProcessor<VirtualFile, VirtualFile[]> myDirectoryFilter;
     private final LinkedList<VirtualFile> myQueue;
     private final MyRootFilter myRootPresentFilter;
     private final FileIndexFacade myExcludedFileIndex;
 
-    private MyRootIterator(final Project project, final VirtualFile root, final Processor<FilePath> processor,
+    private MyRootIterator(final Project project, final VirtualFile root, final Processor<FilePath> processor, final Processor<VirtualFile> vfProcessor,
                            @Nullable PairProcessor<VirtualFile, VirtualFile[]> directoryFilter) {
       myProcessor = processor;
+      myVfProcessor = vfProcessor;
       myDirectoryFilter = directoryFilter;
 
       final ProjectLevelVcsManager plVcsManager = ProjectLevelVcsManager.getInstance(project);
@@ -130,7 +137,7 @@ public class VcsRootIterator {
     public boolean iterate() {
       while (! myQueue.isEmpty()) {
         final VirtualFile current = myQueue.removeFirst();
-        if (! myProcessor.process(new FilePathImpl(current))) return false;
+        if (!process(current)) return false;
 
         if (current.isDirectory()) {
           final VirtualFile[] files = current.getChildren();
@@ -145,5 +152,13 @@ public class VcsRootIterator {
       }
       return true;
     }
+
+    private boolean process(VirtualFile current) {
+      if (myProcessor != null) {
+        return myProcessor.process(new FilePathImpl(current));
+      } else {
+        return myVfProcessor.process(current);
+      }
+    }
   }
 }

plugins/svn4idea/src/org/jetbrains/idea/svn17/integrate/AlienDirtyScope.java

@@ -76,6 +76,10 @@ public class AlienDirtyScope extends VcsDirtyScope {
   public void iterate(final Processor<FilePath> iterator) {
   }
 
+  @Override
+  public void iterateExistingInsideScope(Processor<VirtualFile> vf) {
+  }
+
   @Override
   public boolean isEmpty() {
     return myFiles.isEmpty() && myDirs.isEmpty();

plugins/svn4idea/testSource/org/jetbrains/idea/svn17/SvnChangesCorrectlyRefreshedTest.java

@@ -62,7 +62,7 @@ public class SvnChangesCorrectlyRefreshedTest extends SvnTestCase {
 
   private static void sleep300() {
     try {
-      Thread.sleep(300);
+      Thread.sleep(10000);
     }
     catch (InterruptedException ignore) { }
   }

plugins/svn4ideaOld/src/org/jetbrains/idea/svn/integrate/AlienDirtyScope.java

@@ -76,6 +76,10 @@ public class AlienDirtyScope extends VcsDirtyScope {
   public void iterate(final Processor<FilePath> iterator) {
   }
 
+  @Override
+  public void iterateExistingInsideScope(Processor<VirtualFile> vf) {
+  }
+
   @Override
   public boolean isEmpty() {
     return myFiles.isEmpty() && myDirs.isEmpty();