diff --git a/python/helpers/remote_sync.py b/python/helpers/remote_sync.py index e46af0524d31..da9a977ee474 100644 --- a/python/helpers/remote_sync.py +++ b/python/helpers/remote_sync.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import argparse import json import os +import re import sys import zipfile from collections import defaultdict @@ -18,6 +19,29 @@ _bytes_that_never_appears_in_text = (set(range(7)) | set(range(28, 32)) | {127}) +BINARY_FILE_PATTERNS = [ + # plotlywidget/static/index.js.map is 8.7 MiB. + # Many map files from notebook are near 2 MiB. + r'.*\.js\.map$', + + # uvloop/loop.c contains 6.4 MiB of code. + # Some header files from tensorflow has size more than 1 MiB. + r'.*\.h$', + r'.*\.c$', + + # Test data of pycrypto, many files are near 1 MiB. + r'.*\.rsp$', + + # No need to read these files even if they are small. + r'.*\.py[cdo]$', + + r'.*\.dll$', + # https://unix.stackexchange.com/a/293782/16197 + r'.*\.so(\.\d+){0,3}$', +] +_binary_file_patterns = [re.compile(p) for p in BINARY_FILE_PATTERNS] + + if six.PY2: from io import open @@ -50,22 +74,8 @@ def is_source_file(path): # Want to see that files regardless of their encoding. if path.endswith(('-nspkg.pth', '.html', '.pxd', '.py', '.pyi', '.pyx')): return True - has_bad_extension = path.endswith(( - # plotlywidget/static/index.js.map is 8.7 MiB. - # Many map files from notebook are near 2 MiB. - '.js.map', - - # uvloop/loop.c contains 6.4 MiB of code. - # Some header files from tensorflow has size more than 1 MiB. - '.h', '.c', - - # Test data of pycrypto, many files are near 1 MiB. - '.rsp', - - # No need to read these files even if they are small. - '.dll', '.pyc', '.pyd', '.pyo', '.so', - )) - if has_bad_extension: + _, filename = os.path.split(path) + if any(p.match(filename) for p in _binary_file_patterns): return False return is_text_file(path) diff --git a/python/helpers/tests/data/remote_sync/versioned_dot_so_libraries_ignored/root/lib.py b/python/helpers/tests/data/remote_sync/versioned_dot_so_libraries_ignored/root/lib.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/helpers/tests/data/remote_sync/versioned_dot_so_libraries_ignored/root/lib.so b/python/helpers/tests/data/remote_sync/versioned_dot_so_libraries_ignored/root/lib.so new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/helpers/tests/data/remote_sync/versioned_dot_so_libraries_ignored/root/lib.so.8 b/python/helpers/tests/data/remote_sync/versioned_dot_so_libraries_ignored/root/lib.so.8 new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/helpers/tests/data/remote_sync/versioned_dot_so_libraries_ignored/root/lib.so.8.1 b/python/helpers/tests/data/remote_sync/versioned_dot_so_libraries_ignored/root/lib.so.8.1 new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/helpers/tests/data/remote_sync/versioned_dot_so_libraries_ignored/root/lib.so.8.1.0 b/python/helpers/tests/data/remote_sync/versioned_dot_so_libraries_ignored/root/lib.so.8.1.0 new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/helpers/tests/data/remote_sync/versioned_dot_so_libraries_ignored/root/lib.so.py b/python/helpers/tests/data/remote_sync/versioned_dot_so_libraries_ignored/root/lib.so.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/helpers/tests/test_remote_sync.py b/python/helpers/tests/test_remote_sync.py index b6fbca9aff64..cb60fd964a9c 100644 --- a/python/helpers/tests/test_remote_sync.py +++ b/python/helpers/tests/test_remote_sync.py @@ -556,6 +556,30 @@ class RemoteSyncTest(HelpersTestCase): universal_newlines=True) self.assertIn('usage: remote_sync.py', output) + def test_versioned_dot_so_libraries_ignored(self): + self.collect_sources(['root']) + self.assertJsonEquals(self.resolve_in_temp_dir('.state.json'), { + 'roots': [ + { + 'path': 'root', + 'zip_name': 'root.zip', + 'valid_entries': { + 'lib.py': { + 'mtime': self.mtime('root/lib.py'), + }, + 'lib.so.py': { + 'mtime': self.mtime('root/lib.so.py'), + }, + }, + 'invalid_entries': [] + } + ] + }) + self.assertZipContentEquals(self.resolve_in_temp_dir('root.zip'), """ + lib.py + lib.so.py + """) + def collect_sources(self, roots_inside_test_data, output_dir=None, state_json=None, project_roots=()): if output_dir is None: