Merge a module skeleton with an existing package __init__.py in cache if any

We've already done it in the opposite direction turning existing
module skeletons into packages' __init__.py while generating skeletons
for submodules. It turned out, it can happen in reverse order as well:
for "pyexpat" builtin skeletons for its submodules are generated earlier
that "pyexpat" itself if processed.

GitOrigin-RevId: 76631e901bb2476c59a0fce61c55cccc4408a9a8

python/helpers/pycharm_generator_utils/module_redeclarator.py

@@ -149,7 +149,16 @@ class ModuleRedeclarator(object):
                 self.footer_buf.flush(init)
         else:
             last_pkg_dir = build_pkg_structure(self.cache_dir, '.'.join(qname_parts[:-1]))
-            skeleton_path = os.path.join(last_pkg_dir, qname_parts[-1] + '.py')
+            # In some rare cases submodules of a binary might have been generated earlier than the module
+            # for the binary itself. For instance, it happens for "pyexpat" built-in module which
+            # submodules "pyexpat.errors" and "pyexpat.model" are processed together with "_elementtree"
+            # and "pickle" before "pyexpat" and thus empty pyexpat/__init__.py for them should be replaced
+            # with the skeleton for the main module itself later on.
+            existing_pkg_init = os.path.join(last_pkg_dir, qname_parts[-1], '__init__.py')
+            if os.path.exists(existing_pkg_init):
+                skeleton_path = existing_pkg_init
+            else:
+                skeleton_path = os.path.join(last_pkg_dir, qname_parts[-1] + '.py')
             with fopen(skeleton_path, "w") as mod:
                 for buf in (self.header_buf, self.imports_buf, self.functions_buf, self.classes_buf):
                     buf.flush(mod)

python/helpers/tests/generator3_tests/__init__.py

@@ -78,6 +78,12 @@ class GeneratorTestCase(unittest.TestCase):
                 raise AssertionError(
                     '%r != %r' % (actual_child, expected_child))
 
+    def assertNonEmptyFile(self, path):
+        with open(path) as f:
+            content = f.read()
+            self.assertTrue(content and not content.isspace(),
+                            "File {!r} is empty or contains only whitespaces".format(path))
+
     @contextmanager
     def comparing_dirs(self, subdir='', tmp_subdir=''):
         before_dir = os.path.join(self.test_data_dir, subdir, 'before')

python/helpers/tests/generator3_tests/test_generation.py

@@ -237,11 +237,21 @@ class SkeletonCachingTest(GeneratorTestCase):
     def test_origin_stamp_in_skeleton_header_is_updated_on_copying(self):
         self.check_generator_output('mod', mod_path='mod.py')
 
-    def test_pyexpat_skeletons_layout(self):
+    def test_single_pyexpat_skeletons_layout(self):
         self.run_generator('pyexpat')
         self.assertFalse(os.path.exists(os.path.join(self.temp_skeletons_dir, 'pyexpat.py')))
         self.assertTrue(os.path.isdir(os.path.join(self.temp_skeletons_dir, 'pyexpat')))
-        self.assertTrue(open(os.path.join(self.temp_skeletons_dir, 'pyexpat', '__init__.py')).read())
+        self.assertNonEmptyFile(os.path.join(self.temp_skeletons_dir, 'pyexpat', '__init__.py'))
+        self.assertTrue(os.path.exists(os.path.join(self.temp_skeletons_dir, 'pyexpat', 'model.py')))
+        self.assertTrue(os.path.exists(os.path.join(self.temp_skeletons_dir, 'pyexpat', 'errors.py')))
+
+    # TODO figure out why this is not true for some interpreters
+    @unittest.skipUnless('pyexpat' in sys.builtin_module_names, "pyexpat must be a built-in module")
+    def test_pyexpat_layout_in_builtins(self):
+        self.run_generator(builtins=True)
+        self.assertFalse(os.path.exists(os.path.join(self.temp_skeletons_dir, 'pyexpat.py')))
+        self.assertTrue(os.path.isdir(os.path.join(self.temp_skeletons_dir, 'pyexpat')))
+        self.assertNonEmptyFile(os.path.join(self.temp_skeletons_dir, 'pyexpat', '__init__.py'))
         self.assertTrue(os.path.exists(os.path.join(self.temp_skeletons_dir, 'pyexpat', 'model.py')))
         self.assertTrue(os.path.exists(os.path.join(self.temp_skeletons_dir, 'pyexpat', 'errors.py')))