[python] Rewrite pydev/build_tools

Fix problems with macOS cross-compile x86_64 and arm64

Merge-request: IJ-MR-166083
Merged-by: Egor Eliseev <Egor.Eliseev@jetbrains.com>
(cherry picked from commit 719e67cfb160ffb93258344463ad1763a679f0e6)

GitOrigin-RevId: d00badbeaff9460558bad8dd435e086db6e79f54

python/helpers/pydev/build_tools/build.py

@@ -109,14 +109,9 @@ def build():
 
     os.chdir(root_dir)
 
-    env = None
-    if sys.platform == 'win32':
-        # "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\bin\vcvars64.bat"
-        # set MSSdk=1
-        # set DISTUTILS_USE_SDK=1
-        # set VS100COMNTOOLS=C:\Program Files (x86)\Microsoft Visual Studio 9.0\Common7\Tools
+    env = os.environ.copy()
 
-        env = os.environ.copy()
+    if sys.platform == 'win32':
         if sys.version_info[:2] in ((2, 7), (3, 6), (3, 7), (3, 8), (3, 9), (3, 10),
                                     (3, 11), (3, 12)):
             import setuptools  # We have to import it first for the compiler to be found.
@@ -155,59 +150,115 @@ def build():
         macos_cross_compile(additional_args, env)
     else:
         args = [
-            sys.executable,
-            os.path.join(os.path.dirname(__file__), '..', 'setup_cython.py'),
-            'build_ext',
-            '--inplace',
-        ] + additional_args
+                   sys.executable,
+                   os.path.join(os.path.dirname(__file__), '..', 'setup_cython.py'),
+                   'build_ext',
+                   '--inplace',
+               ] + additional_args
 
         print('Calling args: %s' % (args,))
-        subprocess.check_call(args, env=env,)
+        subprocess.check_call(args, env=env)
 
 
+def check_arch(filepath, expected_arch):
+    try:
+        result = subprocess.check_output(['lipo', '-info', filepath],
+                                         stderr=subprocess.STDOUT).decode()
+        if expected_arch not in result:
+            raise RuntimeError(
+                'Wrong architecture for {filepath}. '.format(filepath=filepath) +
+                'Expected {expected_arch}, got: {result}'.format(
+                    expected_arch=expected_arch, result=result)
+            )
+    except subprocess.CalledProcessError as e:
+        print('Failed to check architecture of {filepath}: {output}'.format(
+            filepath=filepath, output=e.output.decode()))
+        raise
+
 def macos_cross_compile(additional_args, env):
     # Cross-compilation for x86 and M1.
     tempdir = tempfile.mkdtemp()
-    args = [
-               sys.executable,
-               os.path.join(os.path.dirname(__file__), '..', 'setup_cython.py'),
-               'build_ext',
-               '--inplace',
-               '--build-lib=%s/lib.x86' % tempdir,
-               '--build-temp=%s/temp.x86' % tempdir,
-               '--target=x86_64-apple-macos10.12',
-           ] + additional_args
-    print('Calling args: %s' % (args,))
-    subprocess.check_call(args, env=env, )
-    args = [
-               sys.executable,
-               os.path.join(os.path.dirname(__file__), '..', 'setup_cython.py'),
-               'build_ext',
-               '--inplace',
-               '--build-lib=%s/lib.arm64' % tempdir,
-               '--build-temp=%s/temp.arm64' % tempdir,
-               '--target=arm64-apple-macos11',
-           ] + additional_args
-    print('Calling args: %s' % (args,))
-    subprocess.check_call(args, env=env, )
-    # See: https://developer.apple.com/documentation/apple-silicon/building-a-universal-macos-binary.
-    for ext_dir_x86, ext_dir_arm64 in zip(glob.glob('%s/lib.x86/*' % tempdir),
-                                          glob.glob('%s/lib.arm64/*' % tempdir)):
-        for shared_lib_x86, shared_lib_arm64 in zip(
-                glob.glob('%s/*' % ext_dir_x86), glob.glob('%s/*' % ext_dir_arm64)):
-            args = [
-                'lipo',
-                '-create',
-                '-output',
-                '%s/%s/%s' % (root_dir, os.path.basename(ext_dir_x86),
-                              os.path.basename(shared_lib_x86)),
-                shared_lib_x86,
-                shared_lib_arm64,
-            ]
+    try:
+        # x86_64
+        x86_env = env.copy()
+        x86_env['ARCHFLAGS'] = '-arch x86_64'
+        x86_env['MACOSX_DEPLOYMENT_TARGET'] = '14.3'
 
-            print('Building universal binary: %s' % (args,))
-            subprocess.check_call(args, env=env, )
+        args = [
+                   sys.executable,
+                   os.path.join(os.path.dirname(__file__), '..', 'setup_cython.py'),
+                   'build_ext',
+                   '--inplace',
+                   '--build-lib=%s/lib.x86' % tempdir,
+                   '--build-temp=%s/temp.x86' % tempdir,
+                   '--target=x86_64-apple-macos14.3',
+                   ] + additional_args
+        print('Calling args for x86_64: %s' % (args,))
+        subprocess.check_call(args, env=x86_env)
 
+        # arm64
+        arm_env = env.copy()
+        arm_env['ARCHFLAGS'] = '-arch arm64'
+        arm_env['MACOSX_DEPLOYMENT_TARGET'] = '14.3'
+
+        args = [
+                   sys.executable,
+                   os.path.join(os.path.dirname(__file__), '..', 'setup_cython.py'),
+                   'build_ext',
+                   '--inplace',
+                   '--build-lib=%s/lib.arm64' % tempdir,
+                   '--build-temp=%s/temp.arm64' % tempdir,
+                   '--target=arm64-apple-macos14.3',
+                   ] + additional_args
+        print('Calling args for arm64: %s' % (args,))
+        subprocess.check_call(args, env=arm_env)
+
+        def ensure_dir_exists(path):
+            if not os.path.exists(path):
+                os.makedirs(path)
+
+        for ext_dir_x86, ext_dir_arm64 in zip(glob.glob('%s/lib.x86/*' % tempdir),
+                                              glob.glob('%s/lib.arm64/*' % tempdir)):
+            for shared_lib_x86, shared_lib_arm64 in zip(
+                    glob.glob('%s/*' % ext_dir_x86), glob.glob('%s/*' % ext_dir_arm64)):
+
+                if not os.path.exists(shared_lib_x86):
+                    raise RuntimeError('x86_64 binary not found: {shared_lib_x86}'.format(shared_lib_x86=shared_lib_x86))
+                if not os.path.exists(shared_lib_arm64):
+                    raise RuntimeError('arm64 binary not found: {shared_lib_arm64}'.format(shared_lib_arm64=shared_lib_arm64))
+
+                check_arch(shared_lib_x86, 'x86_64')
+                check_arch(shared_lib_arm64, 'arm64')
+
+                output_path = '%s/%s/%s' % (root_dir, os.path.basename(ext_dir_x86),
+                                            os.path.basename(shared_lib_x86))
+
+                ensure_dir_exists(os.path.dirname(output_path))
+
+                args = [
+                    'lipo',
+                    '-create',
+                    '-output',
+                    output_path,
+                    shared_lib_x86,
+                    shared_lib_arm64,
+                ]
+
+                print('Building universal binary:')
+                print(' - Output: %s' % output_path)
+                print(' - x86_64: %s' % shared_lib_x86)
+                print(' - arm64:  %s' % shared_lib_arm64)
+
+                try:
+                    subprocess.check_call(args, env=env)
+                except subprocess.CalledProcessError as e:
+                    print('Failed to create universal binary:')
+                    print(' - Command: {0}'.format(" ".join(args)))
+                    print(' - Return code: {0}'.format(e.returncode))
+                    raise
+    finally:
+        import shutil
+        shutil.rmtree(tempdir, ignore_errors=True)
 
 if __name__ == '__main__':
     use_cython = os.getenv('PYDEVD_USE_CYTHON', None)