PY-27260: For Django Manage.py: improve file tracking

* Track only files in folders by glob pattern ("migrations" is default value)
* Do not even open dot folders (.venv, .idea etc) to improve performance

python/helpers/pycharm/_jb_utils.py

@@ -1,32 +1,41 @@
 # coding=utf-8
+
 __author__ = 'Ilya.Kazakevich'
-import sys, os
+import fnmatch
+import os
+import sys
+
 
 class FileChangesTracker(object):
     """
      On the instantiation the class records the timestampts of files stored in the folder.
      #get_changed_files() return the list of files that have a timestamp different from the one they had during the class instantiation
 
+
     """
-    def __init__(self, folder):
-        self.old_files = self._get_changes_from(folder)
+
+    def __init__(self, folder, pattern="*"):
+        self.old_files = self._get_changes_from(folder, pattern)
         self.folder = folder
+        self.pattern = pattern
 
     def get_changed_files(self):
         assert self.folder, "No changes recorded"
-        new_files = self._get_changes_from(self.folder)
+        new_files = self._get_changes_from(self.folder, pattern=self.pattern)
         return filter(lambda f: f not in self.old_files or self.old_files[f] != new_files[f], new_files.keys())
 
     @staticmethod
-    def _get_changes_from(folder):
+    def _get_changes_from(folder, pattern):
         result = {}
-        for tmp_folder, _, files in os.walk(folder):
-            for file in map(lambda f: os.path.join(tmp_folder, f), files):
-                try:
-                    result.update({file: os.path.getmtime(file)})
-                except OSError:  # on Windows long path may lead to it: PY-23386
-                    message = "PyCharm can't check if the following file been updated: {0}\n".format(str(file))
-                    sys.stderr.write(message)
+        for tmp_folder, sub_dirs, files in os.walk(folder):
+            sub_dirs[:] = [s for s in sub_dirs if not s.startswith(".")]
+            if fnmatch.fnmatch(os.path.basename(tmp_folder), pattern):
+                for file in map(lambda f: os.path.join(tmp_folder, f), files):
+                    try:
+                        result.update({file: os.path.getmtime(file)})
+                    except OSError:  # on Windows long path may lead to it: PY-23386
+                        message = "PyCharm can't check if the following file been updated: {0}\n".format(str(file))
+                        sys.stderr.write(message)
         return result
 
 
@@ -45,11 +54,13 @@ class OptionDescription(object):
     """
     Wrapper for argparse/optparse option (see VersionAgnosticUtils#get_options)
     """
+
     def __init__(self, name, description, action=None):
         self.name = name
         self.description = description
         self.action = action
 
+
 class VersionAgnosticUtils(object):
     """
     "six" emulator: this class fabrics appropriate tool to use regardless python version.
@@ -96,10 +107,9 @@ class _Py2Utils(VersionAgnosticUtils):
         if isinstance(obj, unicode):
             return obj
         try:
-            return unicode(obj) # Obj may have its own __unicode__
+            return unicode(obj)  # Obj may have its own __unicode__
         except (UnicodeDecodeError, AttributeError):
-            return unicode(str(obj).decode("utf-8")) # or it may have __str__
-
+            return unicode(str(obj).decode("utf-8"))  # or it may have __str__
 
     def get_options(self, *args):
         import optparse

python/helpers/pycharm/django_manage.py

@@ -1,10 +1,10 @@
 #!/usr/bin/env python
-import sys
 import os
+import sys
 
-from pycharm_run_utils import adjust_django_sys_path
-from fix_getpass import fixGetpass
 from _jb_utils import FileChangesTracker, jb_escape_output
+from fix_getpass import fixGetpass
+from pycharm_run_utils import adjust_django_sys_path
 
 try:
     from runpy import run_module
@@ -15,6 +15,7 @@ adjust_django_sys_path()
 base_path = sys.argv.pop()
 
 manage_file = os.getenv('PYCHARM_DJANGO_MANAGE_MODULE')
+track_files_pattern = os.environ.get('PYCHARM_TRACK_FILES_PATTERN', None)
 if not manage_file:
     manage_file = 'manage'
 
@@ -39,10 +40,20 @@ if __name__ == "__main__":
     command = sys.argv[1]
     if command in ["syncdb", "createsuperuser"]:  # List of commands that need stdin to be cheated
         sys.stdin = _PseudoTTY(sys.stdin)
-    file_changes_tracker = FileChangesTracker(os.getcwd())
-    run_module(manage_file, None, '__main__', True)
-    # Report files affected/created by commands. This info is used on Java side.
-    changed_files = file_changes_tracker.get_changed_files()
-    if changed_files:
-        print("\n" + jb_escape_output(",".join(changed_files)))
 
+
+    def run_command():
+        run_module(manage_file, None, '__main__', True)
+
+
+    if track_files_pattern:
+        print("Tracking file by folder pattern: ", track_files_pattern)
+        file_changes_tracker = FileChangesTracker(os.getcwd(), track_files_pattern)
+        run_command()
+        # Report files affected/created by commands. This info is used on Java side.
+        changed_files = list(file_changes_tracker.get_changed_files())
+        if changed_files:
+            print("\n" + jb_escape_output(",".join(changed_files)))
+    else:
+        print("File tracking disabled")
+        run_command()

python/src/com/jetbrains/python/remote/NewFilesProcessor.java

@@ -22,6 +22,7 @@ import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.openapi.vcs.AbstractVcs;
 import com.intellij.openapi.vcs.checkin.CheckinEnvironment;
 import com.intellij.openapi.vfs.LocalFileSystem;
+import com.intellij.util.ArrayUtil;
 import com.intellij.vcsUtil.VcsUtil;
 import com.jetbrains.extensions.ModuleExtKt;
 import org.jetbrains.annotations.NotNull;
@@ -53,7 +54,10 @@ public final class NewFilesProcessor {
     assert sdk != null : String.format("Sdk can't be null on module %s", module);
     final PyProjectSynchronizer synchronizer = PythonRemoteInterpreterManager.getSynchronizerInstance(sdk);
 
-    final String[] fileNames = files.split(",");
+    final String[] fileNames = StringUtil.split(files, ",").toArray(ArrayUtil.EMPTY_STRING_ARRAY);
+    if (fileNames.length == 0) {
+      return "";
+    }
     if (synchronizer != null) { // We are on remote side, lets pull files from python first
       synchronizer.syncProject(module, PySyncDirection.REMOTE_TO_LOCAL, success -> {
         if (!success) {
@@ -72,7 +76,7 @@ public final class NewFilesProcessor {
     }
 
 
-    return (fileNames.length == 0 ? "" : String.format("Following files were affected \n %s", StringUtil.join(fileNames, "\n")));
+    return String.format("Following files were affected \n %s", StringUtil.join(fileNames, "\n"));
   }
 
   /**