From 9ac2281e82568862fff2fb32c08d9c5aa993f448 Mon Sep 17 00:00:00 2001 From: Mikhail Golubev Date: Wed, 27 Feb 2019 15:24:56 +0300 Subject: [PATCH] Disable version and modification checks for SDK skeletons by env flag to avoid any inconsistencies in how versions are compared on Java and Python sides. At the moment generator will be run only if existing SDK skeletons are missing or considered invalid, only then cache are actually taken into account. Generator itself doesn't check SDK skeletons. GitOrigin-RevId: ecdd61591464162936b90e3730a848d4dd30c9f6 --- python/helpers/generator3.py | 10 +++++++++- .../pycharm_generator_utils/constants.py | 1 + .../test/test_generation.py | 18 ++++++++++++++---- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/python/helpers/generator3.py b/python/helpers/generator3.py index 0aaeb91bd81b..508dadc3c207 100644 --- a/python/helpers/generator3.py +++ b/python/helpers/generator3.py @@ -31,6 +31,14 @@ def is_test_mode(): return ENV_TEST_MODE_FLAG in os.environ +# Future generator mode where all the checks will be performed on Python side. +# Now it works in transitional mode where validity of existing SDK skeletons is checked on +# Java side (see PySkeletonRefresher), and generator itself inspects only the cache. +@cached +def is_standalone_mode(): + return ENV_STANDALONE_MODE_FLAG in os.environ + + _helpers_dir = os.path.dirname(os.path.abspath(__file__)) @@ -529,7 +537,7 @@ def process_one(name, mod_file_name, doing_builtins, sdk_skeletons_dir): global_cache_dir = os.path.join(python_stubs_dir, CACHE_DIR_NAME) mod_cache_dir = build_cache_dir_path(global_cache_dir, name, mod_file_name) # At the moment this is actually enforced on Java-side - if not should_update_skeleton(sdk_skeletons_dir, name, mod_file_name): + if is_standalone_mode() and not should_update_skeleton(sdk_skeletons_dir, name, mod_file_name): return True if should_update_skeleton(mod_cache_dir, name, mod_file_name): diff --git a/python/helpers/pycharm_generator_utils/constants.py b/python/helpers/pycharm_generator_utils/constants.py index 7d573d56a632..3c6af7e0741b 100644 --- a/python/helpers/pycharm_generator_utils/constants.py +++ b/python/helpers/pycharm_generator_utils/constants.py @@ -811,6 +811,7 @@ REQUIRED_GEN_VERSION_LINE = re.compile(r'(?P\S+)\s+(?P\d+\.\d+)') BLACKLIST_VERSION_LINE = re.compile(r'(?P{mod_path}|[^=]+) = (?P\d+\.\d+) (?P{mod_mtime}|\d+)') ENV_TEST_MODE_FLAG = 'GENERATOR3_TEST_MODE' +ENV_STANDALONE_MODE_FLAG = 'GENERATOR3_STANDALONE_MODE' ENV_VERSION = 'GENERATOR3_VERSION' ENV_REQUIRED_GEN_VERSION_FILE = 'GENERATOR3_REQUIRED_GEN_VERSION_FILE' diff --git a/python/helpers/pycharm_generator_utils/test/test_generation.py b/python/helpers/pycharm_generator_utils/test/test_generation.py index 154a6a5f7767..1b4657b1a847 100644 --- a/python/helpers/pycharm_generator_utils/test/test_generation.py +++ b/python/helpers/pycharm_generator_utils/test/test_generation.py @@ -12,6 +12,7 @@ from pycharm_generator_utils.constants import ( ENV_VERSION, ENV_REQUIRED_GEN_VERSION_FILE, CACHE_DIR_NAME, + ENV_STANDALONE_MODE_FLAG, ) from pycharm_generator_utils.test import GeneratorTestCase @@ -34,7 +35,8 @@ class SkeletonCachingTest(GeneratorTestCase): def run_generator(self, mod_qname=None, mod_path=None, builtins=False, extra_syspath_entry=None, gen_version=None, - required_gen_version_file_path=None): + required_gen_version_file_path=None, + extra_env=None): output_dir = self.temp_skeletons_dir if not extra_syspath_entry: @@ -50,6 +52,9 @@ class SkeletonCachingTest(GeneratorTestCase): if required_gen_version_file_path: env[ENV_REQUIRED_GEN_VERSION_FILE] = required_gen_version_file_path + if extra_env: + env.update(extra_env) + if _run_generator_in_separate_process: generator3_path = os.path.abspath(generator3.__file__) base, ext = os.path.splitext(generator3_path) @@ -190,7 +195,8 @@ class SkeletonCachingTest(GeneratorTestCase): # We can't safely updated cache from SDK skeletons (backwards) because of binaries declaring # multiple modules. Skeletons for them are scattered across SDK skeletons directory, and we can't # collect them reliably. - self.check_generator_output('mod', mod_path='mod.py', gen_version='0.2', custom_required_gen=True) + self.check_generator_output('mod', mod_path='mod.py', gen_version='0.2', custom_required_gen=True, + standalone_mode=True) def test_cache_skeleton_reused_when_sdk_skeleton_is_missing(self): self.check_generator_output('mod', mod_path='mod.py', gen_version='0.2', custom_required_gen=True) @@ -208,7 +214,8 @@ class SkeletonCachingTest(GeneratorTestCase): self.check_generator_output('mod', mod_path='mod.py', gen_version='0.3', custom_required_gen=True) def test_cache_skeleton_not_regenerated_when_sdk_skeleton_generation_failed_for_same_version_and_same_binary(self): - self.check_generator_output('mod', mod_path='mod.py', gen_version='0.1', custom_required_gen=True) + self.check_generator_output('mod', mod_path='mod.py', gen_version='0.1', custom_required_gen=True, + standalone_mode=True) def test_cache_skeleton_regenerated_when_sdk_skeleton_generation_failed_for_modified_binary(self): self.check_generator_output('mod', mod_path='mod.py', gen_version='0.1', custom_required_gen=True) @@ -227,10 +234,13 @@ class SkeletonCachingTest(GeneratorTestCase): def test_binary_declares_extra_module_that_fails(self): self.check_generator_output('mod', mod_path='mod.py') - def check_generator_output(self, mod_name, mod_path=None, mod_root=None, custom_required_gen=False, **kwargs): + def check_generator_output(self, mod_name, mod_path=None, mod_root=None, + custom_required_gen=False, standalone_mode=False, **kwargs): if custom_required_gen: kwargs.setdefault('required_gen_version_file_path', os.path.join(self.test_data_dir, 'required_gen_version')) + if standalone_mode: + kwargs.setdefault('extra_env', {})[ENV_STANDALONE_MODE_FLAG] = 'True' if not mod_root: mod_root = self.test_data_dir