From e4ad3771b12c3eaaf93cb66cc9e17d607db3176e Mon Sep 17 00:00:00 2001 From: Maxim Shafirov Date: Wed, 24 Sep 2008 21:31:14 +0400 Subject: [PATCH] FileWatcher refactoring --- bin/fsnotifier | Bin 0 -> 13300 bytes .../openapi/vfs/watcher/ChangeKind.java | 14 + .../intellij/idea/CommandLineApplication.java | 2 +- .../openapi/vfs/impl/local/FileWatcher.java | 301 ++++++++++++++++++ .../vfs/impl/local/LocalFileSystemImpl.java | 144 +++------ .../src/componentSets/Platform.xml | 5 - tools/fsNotifier/mac/fsnotifier.c | 176 ++++++++++ tools/fsNotifier/mac/make.sh | 1 + 8 files changed, 540 insertions(+), 103 deletions(-) create mode 100644 bin/fsnotifier create mode 100644 platform-api/src/com/intellij/openapi/vfs/watcher/ChangeKind.java create mode 100644 platform-impl/src/com/intellij/openapi/vfs/impl/local/FileWatcher.java create mode 100644 tools/fsNotifier/mac/fsnotifier.c create mode 100644 tools/fsNotifier/mac/make.sh diff --git a/bin/fsnotifier b/bin/fsnotifier new file mode 100644 index 0000000000000000000000000000000000000000..9961c9174e84be30d136bebb78190e2d2d9315fd GIT binary patch literal 13300 zcmeHOZEPGz8J>%;#7Rx=+L$I3TCU{yQ7Q7d5K*Kca;`m2Ozqg%=Ohrw=zOzUcRJ>~T;|Mi!*w=>3C(6^z#AN|5DjLl$-pnnQoh@q~- z{0n?|5I25{`{2fr^rE;3k>MvYfqByL^)QlOxRSnUl@Nw<&LHHr&HC{>8G8l_@ttn0 z{yG+A=Y1?hPB*k-W!w{Nw(-J87(0l$7h=AKJ5ug`6QWc!6fIk#B@8kCKA9t4;}|(!`BrepEa`U>xH;czn!L`8;p5fgW}CkmGbM7 z%{JN-jC~l>?eAzKR|pch*+wU9P#T@@Xk$#*vBqzkZKM;JAtKS}9QX~itu?~4=tA@j z3=IyipQcU-DS(7K(pjSJb0S1lEmp9LUI1I^w_!c$`#N;{KDu~Kh|^}dDvaz{Ng>f@ z8)?Ka3VAxQQ4c9aX-_F9ir;;P`umPP$b0(y4APjO_+eueiz#+m>ohjo0E)2G$dqm<VYzV3sfhYvqDGqdJDaMRZK99~^&JxSJRzXM-o4GQ!`^GDC) z%fmkZBCgH4E3?l(EAK{YuZ&pB2c+=i0+-_gUn0Gq@z>b`UnIEDlWe6=Zj7*xF`A^I z@rw_@8?Rf_bvCuL@on7MxL>rU7n%7PYr4UVTW#K`MU#z3X^z)xKb`wyw03>29(AJI zZi&MtP1@2-gA~l|a7iEM(#D}JRs3j>XVf#_BP}dh(@PY{?hS#o*?j3BuRjz;LJ?`U zEp~~&J~JD;a96Y0T;Qve%K~4a;njz6vH8k|KwEgjPIt~V+N_m`RgYYlHTT_Y$sT`+ zalbXa%FJDkbqGXLme1lFe+xBh_CcEc24)?;+Qen0N$U>srP$Xr1W0zioN%mu&(1il zw7x2H1l`d6xU?*J-Vf+)qm7N2KXCHrm)$*1x_fN%_8MRIvwh6XcCNmyh3}=kTZOl@ zvEwV#tLHIgZg2eV-_7Q0Wg8QexUtW%PU8iMy#8Ug%tz`jyMbJT=(RM0_YKa>8ga+c zJ&k8*`X0xc^6tNLqlHHxT4x|0d-xO>aIEjsN;+Kd+XHOPC*8!3AH(t8@22KhS8tcg z;If5W;M6+S6L`6ylTvF>Hg=PBIp zD~Q6|GRI0m>u<3*`fYs09&JE3LBT=BS0L<3?u950QkodsX~gk{#kN|8vUT2WHiP#6 zc^lbZcC7ndYC0RgCy9}4rByaxCPse?kq2J_Y`lnzWWbsqI(bT_CVB36b89TTDZR_2 z9BU6^fU#D@BR?FFF&TZH{BZe58h1KYm-nnc1u>GRPm@L!*s*fBVsEi&-=Z~g(?0DY zxoM~5Dw=aH(vP?rf4Me#eSu%Y^FW*SvORs>9=LYXP4n!_O>2QIY+9R7|6lPY?fP{t z4oKTqYNPWDe3stiVf%_bJ#P=pZfQFLBC_=3&jI{q0PlcptijdI&&TP2JR`AN zdOzvPQNx3nUAe@y0Dd&E=9>Zhy#RhPfPWakmjZZBV%P6)0`htQuLN*sf-%%ywV~uf z)*J7^NcG-Ss)6bfhB_ODG9y{vjT$`mL3Ge0_mgV7K_#^`Xk-cWd(b~BF%^0X-7v?* zSh14FUrmS!rBqck?@tIs!K~y&Nmb97RVId&s;U_Ss9-jfd~dO&xHZu_6V*Y$eac(Gy+Tc+MLq;X_n4b-aRNZ{YHTP|hhi(@@qGv!&8l zHg|>zr7~I6)C$0)QZYyk_24?iYfNZx!{D)4VY-pk3?{mJyEH93)r}bgrn`HG%}T$j zRu3yix2b8c5D;RHiQW-ThDQvjlzqJo!jasBk~d4r@uD%|ncJi`;yM@b*NxJ2BP;D8 z#_m#9*N2sY0?#>0RVbRK633t(q=Q myDirtyPaths = new ArrayList(); + private List myDirtyRecursivePaths = new ArrayList(); + private List myDirtyDirs = new ArrayList(); + private List myManualWatchRoots = new ArrayList(); + + private List myRecursiveWatchRoots = new ArrayList(); + private List myFlatWatchRoots = new ArrayList(); + + private Process notifierProcess; + private BufferedReader notifierReader; + private BufferedWriter notifierWriter; + + private static FileWatcher ourInstance = new FileWatcher(); + private int attemptCount = 0; + private static final int MAX_PROCESS_LAUNCH_ATTEMPT_COUNT = 10; + private boolean isShuttingDown = false; + + public static FileWatcher getInstance() { + return ourInstance; + } + + private FileWatcher() { + try { + if (!"true".equals(System.getProperty(PROPERTY_WATCHER_DISABLED))) { + startupProcess(); + } + } + catch (IOException e) { + // Ignore + } + + if (notifierProcess != null) { + LOG.info("Native file watcher is operational."); + new WatchForChangesThread().start(); + + Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { + public void run() { + try { + isShuttingDown = true; + shutdownProcess(); + } + catch (IOException e) { + // Do nothing. + } + } + })); + } + else { + LOG.info("Native file watcher failed to startup."); + } + } + + public List getDirtyPaths() { + synchronized (LOCK) { + final List result = myDirtyPaths; + myDirtyPaths = new ArrayList(); + return result; + } + } + + public List getDirtyRecursivePaths() { + synchronized (LOCK) { + final List result = myDirtyRecursivePaths; + myDirtyRecursivePaths = new ArrayList(); + return result; + } + + } + + public List getDirtyDirs() { + synchronized (LOCK) { + final List result = myDirtyDirs; + myDirtyDirs = new ArrayList(); + return result; + } + } + + public List getManualWatchRoots() { + synchronized (LOCK) { + return Collections.unmodifiableList(myManualWatchRoots); + } + } + + public void setWatchRoots(List recursive, List flat) { + synchronized (LOCK) { + try { + if (myRecursiveWatchRoots.equals(recursive) && myFlatWatchRoots.equals(flat)) return; + + myRecursiveWatchRoots = recursive; + myFlatWatchRoots = flat; + + writeLine(ROOTS_COMMAND); + for (String path : recursive) { + writeLine(path); + } + for (String path : flat) { + writeLine("|" + path); + } + writeLine("#"); + } + catch (IOException e) { + LOG.error(e); + } + } + } + + private void setManualWatchRoots(List roots) { + synchronized (LOCK) { + myManualWatchRoots = roots; + } + } + + private void startupProcess() throws IOException { + if (isShuttingDown) return; + + if (attemptCount++ > MAX_PROCESS_LAUNCH_ATTEMPT_COUNT) { + throw new IOException("Can't launch process anymore"); + } + + shutdownProcess(); + + notifierProcess = Runtime.getRuntime().exec(new String[]{PathManager.getBinPath() + "/fsnotifier"}); + notifierReader = new BufferedReader(new InputStreamReader(notifierProcess.getInputStream())); + notifierWriter = new BufferedWriter(new OutputStreamWriter(notifierProcess.getOutputStream())); + } + + private void shutdownProcess() throws IOException { + if (notifierProcess != null) { + writeLine(EXIT_COMMAND); + + notifierProcess = null; + notifierReader = null; + notifierWriter = null; + } + } + + public boolean isOperational() { + return notifierProcess != null; + } + + private class WatchForChangesThread extends Thread { + + public WatchForChangesThread() { + //noinspection HardCodedStringLiteral + super("WatchForChangesThread"); + } + + public void run() { + try { + while (true) { + if (ApplicationManager.getApplication().isDisposeInProgress() || notifierProcess == null || isShuttingDown) return; + + final String command = readLine(); + if (command == null) { + // Unexpected process exit, relaunch attempt + startupProcess(); + continue; + } + + if (GIVEUP_COMMAND.equals(command)) { + shutdownProcess(); + return; + } + if (RESET_COMMAND.equals(command)) { + reset(); + } + else if (UNWATCHEABLE_COMMAND.equals(command)) { + List roots = new ArrayList(); + do { + final String path = readLine(); + if (path == null || "#".equals(path)) break; + roots.add(path); + } + while (true); + + setManualWatchRoots(roots); + } + else { + String path = readLine(); + if (path == null) { + // Unexpected process exit, relaunch attempt + startupProcess(); + continue; + } + + if (isWatcheable(path)) { + try { + onPathChange(ChangeKind.valueOf(command), path); + } + catch (IllegalArgumentException e) { + LOG.error("Illegal watcher command: " + command); + } + } + } + } + } + catch (IOException e) { + LOG.info("Watcher terminated and attempt to restart has failed. Exiting watching thread.", e); + } + } + } + + private void writeLine(String line) throws IOException { + notifierWriter.write(line); + notifierWriter.newLine(); + notifierWriter.flush(); + } + + private String readLine() throws IOException { + return notifierReader.readLine(); + } + + private boolean isWatcheable(final String path) { + if (path == null) return false; + + synchronized (LOCK) { + for (String root : myRecursiveWatchRoots) { + if (FileUtil.startsWith(path, root)) return true; + } + + for (String root : myFlatWatchRoots) { + if (FileUtil.pathsEqual(path, root)) return true; + if (FileUtil.pathsEqual(new File(path).getParentFile().getPath(), root)) return true; + } + } + + return false; + } + + private void onPathChange(final ChangeKind changeKind, final String path) { + synchronized (LOCK) { + switch (changeKind) { + case STATS: + case CHANGE: + myDirtyPaths.add(path); + break; + + case CREATE: + case DELETE: + myDirtyPaths.add(new File(path).getParentFile().getPath()); + break; + + case DIRTY: + myDirtyDirs.add(path); + break; + + case RECDIRTY: + myDirtyRecursivePaths.add(path); + break; + + case RESET: + reset(); + break; + } + } + } + + private void reset() { + synchronized (LOCK) { + myDirtyPaths.clear(); + myDirtyDirs.clear(); + myDirtyRecursivePaths.clear(); + + for (VirtualFile root : ManagingFS.getInstance().getLocalRoots()) { + ((NewVirtualFile)root).markDirtyRecursively(); + } + } + } +} \ No newline at end of file diff --git a/platform-impl/src/com/intellij/openapi/vfs/impl/local/LocalFileSystemImpl.java b/platform-impl/src/com/intellij/openapi/vfs/impl/local/LocalFileSystemImpl.java index a513ca612a91..ec2ac11fcb39 100644 --- a/platform-impl/src/com/intellij/openapi/vfs/impl/local/LocalFileSystemImpl.java +++ b/platform-impl/src/com/intellij/openapi/vfs/impl/local/LocalFileSystemImpl.java @@ -22,7 +22,6 @@ import com.intellij.util.concurrency.JBReentrantReadWriteLock; import com.intellij.util.concurrency.LockFactory; import com.intellij.util.containers.HashSet; import com.intellij.util.io.fs.IFile; -import com.intellij.vfs.local.win32.FileWatcher; import org.jetbrains.annotations.NonNls; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -41,11 +40,8 @@ public final class LocalFileSystemImpl extends LocalFileSystem implements Applic private final List myRootsToWatch = new ArrayList(); private WatchRequest[] myCachedNormalizedRequests = null; - private final Set myDirtyFiles = new HashSet(); // dirty files when FileWatcher is available - private final Set myDeletedFiles = new HashSet(); - private final List myHandlers = new ArrayList(); - private List myManualWatchRoots = new ArrayList(); + private final FileWatcher myWatcher; private static class WatchRequestImpl implements WatchRequest { public final String myRootPath; @@ -109,9 +105,8 @@ public final class LocalFileSystemImpl extends LocalFileSystem implements Applic } public LocalFileSystemImpl() { - if (FileWatcher.isAvailable()) { - FileWatcher.initialize(); - new WatchForChangesThread().start(); + myWatcher = FileWatcher.getInstance(); + if (myWatcher.isOperational()) { new StoreRefreshStatusThread().start(); } } @@ -141,20 +136,12 @@ public final class LocalFileSystemImpl extends LocalFileSystem implements Applic } myRootsToWatch.clear(); - myDirtyFiles.clear(); - myDeletedFiles.clear(); final File file = new File(FileUtil.getTempDirectory()); String path = file.getCanonicalPath().replace(File.separatorChar, '/'); addRootToWatch(path, true); } - private static void updateFileWatcher() { - if (FileWatcher.isAvailable()) { - FileWatcher.interruptWatcher(); - } - } - public String getProtocol() { return PROTOCOL; } @@ -291,21 +278,15 @@ public final class LocalFileSystemImpl extends LocalFileSystem implements Applic } private void storeRefreshStatusToFiles() { - if (FileWatcher.isAvailable()) { - markPathsDirty(getAndClear(myDirtyFiles)); - markPathsDirty(getAndClear(myDeletedFiles)); + if (FileWatcher.getInstance().isOperational()) { + // TODO: different ways to marky dirty for all these cases + markPathsDirty(FileWatcher.getInstance().getDirtyPaths()); + markFlatDirsDirty(FileWatcher.getInstance().getDirtyDirs()); + markRecursiveDirsDirty(FileWatcher.getInstance().getDirtyRecursivePaths()); } } - private static String[] getAndClear(final Set set) { - synchronized (set) { - final String[] copy = set.toArray(new String[set.size()]); - set.clear(); - return copy; - } - } - - private void markPathsDirty(final String[] dirtyFiles) { + private void markPathsDirty(final List dirtyFiles) { for (String dirtyFile : dirtyFiles) { String path = dirtyFile.replace(File.separatorChar, '/'); VirtualFile file = findFileByPathIfCached(path); @@ -315,11 +296,35 @@ public final class LocalFileSystemImpl extends LocalFileSystem implements Applic } } + private void markFlatDirsDirty(final List dirtyFiles) { + for (String dirtyFile : dirtyFiles) { + String path = dirtyFile.replace(File.separatorChar, '/'); + VirtualFile file = findFileByPathIfCached(path); + if (file instanceof NewVirtualFile) { + final NewVirtualFile nvf = (NewVirtualFile)file; + nvf.markDirty(); + for (VirtualFile child : nvf.getCachedChildren()) { + ((NewVirtualFile)child).markDirty(); + } + } + } + } + + private void markRecursiveDirsDirty(final List dirtyFiles) { + for (String dirtyFile : dirtyFiles) { + String path = dirtyFile.replace(File.separatorChar, '/'); + VirtualFile file = findFileByPathIfCached(path); + if (file instanceof NewVirtualFile) { + ((NewVirtualFile)file).markDirtyRecursively(); + } + } + } + public void markSuspicousFilesDirty(List files) { storeRefreshStatusToFiles(); - if (FileWatcher.isAvailable()) { - for (String root : myManualWatchRoots) { + if (myWatcher.isOperational()) { + for (String root : myWatcher.getManualWatchRoots()) { final VirtualFile suspicousRoot = findFileByPathIfCached(root); if (suspicousRoot != null) { ((NewVirtualFile)suspicousRoot).markDirtyRecursively(); @@ -371,84 +376,30 @@ public final class LocalFileSystemImpl extends LocalFileSystem implements Applic return path.replace(File.separatorChar, '/'); } - private class WatchForChangesThread extends Thread { - public WatchForChangesThread() { - //noinspection HardCodedStringLiteral - super("WatchForChangesThread"); - } - - public void run() { - updateFileWatcher(); - try { - while (true) { - if (ApplicationManager.getApplication().isDisposeInProgress()) return; - - FileWatcher.ChangeInfo[] infos = FileWatcher.waitForChange(); - - if (infos == null) { - setUpFileWatcher(); - } - else { - for (FileWatcher.ChangeInfo info : infos) { - if (info == null) continue; - - String path = info.getFilePath(); - int changeType = info.getChangeType(); - if (changeType == FileWatcher.FILE_MODIFIED) { - synchronized (myDirtyFiles) { - myDirtyFiles.add(path); - } - } - else if (changeType == FileWatcher.FILE_ADDED || changeType == FileWatcher.FILE_RENAMED_NEW_NAME) { - synchronized (myDirtyFiles) { - String parent = new File(path).getParent(); - if (parent != null) { - myDirtyFiles.add(parent); - } - } - } - else if (changeType == FileWatcher.FILE_REMOVED || changeType == FileWatcher.FILE_RENAMED_OLD_NAME) { - synchronized (myDeletedFiles) { - myDeletedFiles.add(path); - } - } - } - } - } - } - catch (IOException e) { - LOG.info("Watcher terminated and attempt to restart has failed. Exiting watching thread.", e); - } - } - } - private void setUpFileWatcher() { final Application application = ApplicationManager.getApplication(); if (application.isDisposeInProgress()) return; - if (FileWatcher.isAvailable()) { + if (myWatcher.isOperational()) { application.runReadAction(new Runnable() { public void run() { WRITE_LOCK.lock(); try { final WatchRequest[] watchRequests = normalizeRootsForRefresh(); - String[] dirPaths = new String[watchRequests.length]; - boolean[] toWatchRecursively = new boolean[watchRequests.length]; - int cnt = 0; + List myRecursiveRoots = new ArrayList(); + List myFlatRoots = new ArrayList(); + for (WatchRequest root : watchRequests) { - dirPaths[cnt] = root.getFileSystemRootPath(); - toWatchRecursively[cnt] = root.isToWatchRecursively(); - cnt++; + if (root.isToWatchRecursively()) { + myRecursiveRoots.add(root.getFileSystemRootPath()); + } + else { + myFlatRoots.add(root.getFileSystemRootPath()); + } } - final Vector watchManual = new Vector(); - FileWatcher.setup(dirPaths, toWatchRecursively, watchManual); - - myManualWatchRoots = new ArrayList(); - for (int i = 0; i < watchManual.size(); i++) { - myManualWatchRoots.add(watchManual.elementAt(i)); - } + myWatcher.setWatchRoots(myRecursiveRoots, myFlatRoots); } finally { WRITE_LOCK.unlock(); @@ -469,7 +420,6 @@ public final class LocalFileSystemImpl extends LocalFileSystem implements Applic } public void run() { - //noinspection InfiniteLoopStatement while (true) { final Application application = ApplicationManager.getApplication(); if (application == null || application.isDisposed()) break; @@ -897,7 +847,7 @@ public final class LocalFileSystemImpl extends LocalFileSystem implements Applic } }; - if (asynchronous && FileWatcher.isAvailable()) { + if (asynchronous && myWatcher.isOperational()) { RefreshQueue.getInstance().refresh(true, true, heavyRefresh, ManagingFS.getInstance().getRoots(this)); } else { diff --git a/platform-resources/src/componentSets/Platform.xml b/platform-resources/src/componentSets/Platform.xml index a464d0102027..ab26913cb72d 100644 --- a/platform-resources/src/componentSets/Platform.xml +++ b/platform-resources/src/componentSets/Platform.xml @@ -30,11 +30,6 @@ com.intellij.openapi.vfs.newvfs.persistent.PersistentFS - - com.intellij.openapi.vfs.watcher.FileSystemTracker - com.intellij.openapi.vfs.impl.watcher.FileSystemTrackerImpl - - com.intellij.openapi.vfs.pointers.VirtualFilePointerManager com.intellij.openapi.vfs.impl.VirtualFilePointerManagerImpl diff --git a/tools/fsNotifier/mac/fsnotifier.c b/tools/fsNotifier/mac/fsnotifier.c new file mode 100644 index 000000000000..bae724d3b6fb --- /dev/null +++ b/tools/fsNotifier/mac/fsnotifier.c @@ -0,0 +1,176 @@ +#include +#include + +static int ReportMountedFileSystems() + // If fsBuf is too small to account for all volumes, getfsstat will + // silently truncate the returned information. Worse yet, it returns + // the number of volumes it passed back, not the number of volumes present, + // so you can't tell if the list was truncated. + // + // So, in order to get an accurate snapshot of the volume list, I call + // getfsstat with a NULL fsBuf to get a count (fsCountOrig), then allocate a + // buffer that holds (fsCountOrig + 1) items, then call getfsstat again with + // that buffer. If the list was silently truncated, the second count (fsCount) + // will be (fsCountOrig + 1), and we loop to try again. +{ + int err; + int fsCountOrig; + int fsCount; + struct statfs * fsBuf; + bool done; + + + fsBuf = NULL; + fsCount = 0; + + done = false; + do { + // Get the initial count. + err = 0; + fsCountOrig = getfsstat(NULL, 0, MNT_WAIT); + if (fsCountOrig < 0) { + err = errno; + } + + // Allocate a buffer for fsCountOrig + 1 items. + if (err == 0) { + if (fsBuf != NULL) { + free(fsBuf); + } + fsBuf = malloc((fsCountOrig + 1) * sizeof(*fsBuf)); + if (fsBuf == NULL) { + err = ENOMEM; + } + } + + // Get the list. + if (err == 0) { + fsCount = getfsstat(fsBuf, (int) ((fsCountOrig + 1) * sizeof(*fsBuf)), MNT_WAIT); + if (fsCount < 0) { + err = errno; + } + } + + // We got the full list if the number of items returned by the kernel + // is strictly less than the buffer that we allocated (fsCountOrig + 1). + if (err == 0) { + if (fsCount <= fsCountOrig) { + done = true; + } + } + } while ( (err == 0) && ! done ); + + int i; + int mountCounts = 0; + for (i = 0; i < fsCount; i++) { + if ((fsBuf[i].f_flags & MNT_LOCAL) == 0 || (fsBuf[i].f_flags & MNT_JOURNALED) == 0) { + if (mountCounts == 0) { + printf("UNWATCHEABLE\n"); + } + printf("%s\n", fsBuf[i].f_mntonname); + mountCounts++; + } + } + + if (mountCounts > 0) { + printf("#\n"); + fflush(stdout); + } + + free(fsBuf); + fsBuf = NULL; + + return err; +} + +void callback(ConstFSEventStreamRef streamRef, + void *clientCallBackInfo, + size_t numEvents, + void *eventPaths, + const FSEventStreamEventFlags eventFlags[], + const FSEventStreamEventId eventIds[]) { + char **paths = eventPaths; + + int i; + for (i=0; i