Python tests: add script to patch .exe files in Scripts directory to make environment transferable.

See `README.txt`

GitOrigin-RevId: a4df2516360e96c03c507c471f662b97fe3bc51c

python/setup-test-environment/build.gradle.kts

@@ -53,10 +53,20 @@ envs {
   // I don't think that it's desired behaviour to install pythons for tests user-wide what will be done
   // if we don't force these options (also there may be conflicts with existing installations)
   zipRepository = URL(System.getenv().getOrDefault("PYCHARM_ZIP_REPOSITORY",
-                                                   "https://packages.jetbrains.team/files/p/py/python-archives-windows/"))
+    "https://packages.jetbrains.team/files/p/py/python-archives-windows/"))
   shouldUseZipsFromRepository = isWindows
 }
 
+tasks.register("copy_buildserver_win_fix") {
+  // these files are required to fix paths on Windows, see their readme
+  doLast {
+    copy {
+      from("buildserver_win_fix")
+      into(pythonsDirectory)
+    }
+  }
+}
+
 tasks.register<Exec>("kill_python_processes") {
   onlyIf { isWindows }
 
@@ -73,7 +83,7 @@ tasks.register<Delete>("clean") {
 }
 
 tasks.register("build") {
-  dependsOn(tasks.matching { it.name.startsWith("setup_") }, "clean")
+  dependsOn(tasks.matching { it.name.startsWith("setup_") }, "clean", "copy_buildserver_win_fix")
 }
 
 fun createPython(
@@ -124,13 +134,13 @@ fun createPython(
   // the task serves as aggregator so that one could just execute `./gradlew setup_python_123`
   // to build some specific environment
   project.tasks.create("setup_$id") {
-    setDependsOn(listOf("clean", "populate_links_$id"))
+    setDependsOn(listOf("clean", "populate_links_$id", "copy_buildserver_win_fix"))
   }
 }
 
 createPython("py312_django_latest", "3.12",
-             listOf("django", "behave-django", "behave", "pytest", "untangle", "djangorestframework"),
-             listOf("python3.12", "django", "django20", "behave", "behave-django", "django2", "pytest", "untangle"))
+  listOf("django", "behave-django", "behave", "pytest", "untangle", "djangorestframework"),
+  listOf("python3.12", "django", "django20", "behave", "behave-django", "django2", "pytest", "untangle"))
 
 val qtTags = mutableListOf<String>()
 val qtPackages = mutableListOf<String>()
@@ -140,32 +150,32 @@ if (isUnix && !isMacOs) { //qt is for Linux only
 }
 
 createPython("py27", "2.7",
-             listOf(),
-             listOf("python2.7"))
+  listOf(),
+  listOf("python2.7"))
 
 createPython("py38", "3.8",
-             listOf("ipython==7.8", "django==2.2", "behave", "jinja2", "tox>=2.0", "nose", "pytest", "django-nose", "behave-django",
-                    "pytest-xdist", "untangle", "numpy", "pandas") + qtPackages,
-             listOf("python3.8", "python3", "ipython", "ipython780", "skeletons", "django", "behave", "behave-django", "tox", "jinja2",
-                    "packaging", "pytest", "nose", "django-nose", "behave-django", "django2", "xdist", "untangle", "pandas") + qtTags)
+  listOf("ipython==7.8", "django==2.2", "behave", "jinja2", "tox>=2.0", "nose", "pytest", "django-nose", "behave-django",
+    "pytest-xdist", "untangle", "numpy", "pandas") + qtPackages,
+  listOf("python3.8", "python3", "ipython", "ipython780", "skeletons", "django", "behave", "behave-django", "tox", "jinja2",
+    "packaging", "pytest", "nose", "django-nose", "behave-django", "django2", "xdist", "untangle", "pandas") + qtTags)
 
 createPython("python3.9", "3.9",
-             listOf("pytest", "pytest-xdist"),
-             listOf("python3.9", "python3", "pytest", "xdist", "packaging"))
+  listOf("pytest", "pytest-xdist"),
+  listOf("python3.9", "python3", "pytest", "xdist", "packaging"))
 
 createPython("python3.10", "3.10",
-             listOf("untangle"), listOf("python3.10", "untangle"))
+  listOf("untangle"), listOf("python3.10", "untangle"))
 
 createPython("python3.11", "3.11",
-             listOf("black == 23.1.0", "joblib", "tensorflow", "poetry"),
-             listOf("python3.11", "black", "poetry", "joblib", "tensorflow"))
+  listOf("black == 23.1.0", "joblib", "tensorflow", "poetry"),
+  listOf("python3.11", "black", "poetry", "joblib", "tensorflow"))
 
 createPython("python3.12", "3.12",
-             listOf("teamcity-messages", "Twisted", "pytest", "poetry")
-             // TODO: maybe switch to optional dependency Twisted[windows-platform]
-             // https://docs.twisted.org/en/stable/installation/howto/optional.html
-             + if (isWindows) listOf("pypiwin32") else listOf(), //win32api is required for pypiwin32
-             listOf("python3", "poetry", "python3.12", "messages", "twisted", "pytest"))
+  listOf("teamcity-messages", "Twisted", "pytest", "poetry")
+    // TODO: maybe switch to optional dependency Twisted[windows-platform]
+    // https://docs.twisted.org/en/stable/installation/howto/optional.html
+    + if (isWindows) listOf("pypiwin32") else listOf(), //win32api is required for pypiwin32
+  listOf("python3", "poetry", "python3.12", "messages", "twisted", "pytest"))
 
 // set CONDA_PATH to conda binary location to be able to run tests
 createPython("conda", "Miniconda3-py312_24.5.0-0", listOf(), listOf("conda"), type = PythonType.CONDA)

python/setup-test-environment/buildserver_win_fix/README.txt

@@ -0,0 +1,6 @@
+The problem:
+https://mail.python.org/pipermail/python-win32/2024-December/014944.html
+
+We build environment on Agent1, then download it to Agent2.
+Scripts (i.e. `pip.exe`) have shebangs hardcoded.
+To fix it, run `fix_path.cmd`

python/setup-test-environment/buildserver_win_fix/fix_path.cmd

@@ -0,0 +1,18 @@
+@echo off
+@REM Runs `fix_path.py` against every python either in the same dir or in PYCHARM_PYTHONS
+SETLOCAL EnableDelayedExpansion
+SET FIX="%~dp0\fix_path.py"
+
+IF "%PYCHARM_PYTHONS%"=="" SET PYCHARM_PYTHONS="%~dp0"
+
+FOR /D %%d in ("%PYCHARM_PYTHONS%\*") do (
+    @rem skip conda
+    echo "%%d" | find  "conda" > nul
+    if !ERRORLEVEL!==1 (
+        @rem skip 2.7
+        echo "%%d" | find  "27" > nul
+        if !ERRORLEVEL!==1 (
+          %%d\python.exe  %FIX%
+        )
+    )
+)

python/setup-test-environment/buildserver_win_fix/fix_path.py

@@ -0,0 +1,49 @@
+import sys
+import sysconfig
+from pathlib import Path
+
+
+# On Windows you have lots of scripts (i.e `Scripts\pip.exe`). They are usually have hardcoded path to the python.
+# It makes them non-transferable.
+# This script replaces hard-coded paths to the one from the current interpreter.
+
+# When download `.zip` file with pythons from buildserver, run this script on each python to make sure scripts are ok
+
+
+def get_script_content(script_path: Path) -> bytes:
+    with open(script_path, "r+b") as f:
+        return f.read()
+
+
+def put_script_data(script_path: Path, script_data: bytes):
+    with open(script_path, "w+b") as f:
+        return f.write(script_data)
+
+
+def find_shebang_start_index(where_to_search_shebang: bytes) -> int:
+    # Almost always shebang starts here (right after the last PE section)
+    default_offset: int = 0x1A600
+    if len(where_to_search_shebang) > default_offset and where_to_search_shebang[default_offset] == '#':
+        return default_offset
+
+    # Not found in default offset? Let's search
+    for drive_letter_code in range(ord('a'), ord('z')):
+        for drive_letter in [chr(drive_letter_code), chr(drive_letter_code).upper()]:
+            try:
+                return where_to_search_shebang.index(f"#!{drive_letter}:\\".encode('UTF-8'))
+            except ValueError:
+                pass
+
+
+if __name__ == "__main__":
+    new_shebang = f"#!{sys.executable}".encode('UTF-8')
+    scripts_dir = sysconfig.get_path('scripts')
+    for script in Path(scripts_dir).glob("*.exe"):
+        script_content = get_script_content(script)
+        shebang_start = find_shebang_start_index(script_content)
+        assert shebang_start, f"No python found in {script}"
+        shebang_end = script_content.find(b'\n', shebang_start)
+        assert shebang_end, "No shebang end found: broken binary?"
+        old_shebang = script_content[shebang_start:shebang_end]
+        new_script_content = script_content.replace(old_shebang, new_shebang)
+        put_script_data(script, new_script_content)