PY-45771 Add Gradle tasks for checking debugger C files are up to date

(cherry picked from commit 10467d6f5d3cc4d3512b82a1cadba50eb12e3860)

IDEA-CR-69615

GitOrigin-RevId: ec86b3aa8f43feeddb09976167867920b34cad76

python/build/build-debug-binaries/src/build.gradle

@@ -1,5 +1,8 @@
 import org.apache.tools.ant.taskdefs.condition.Os
 
+import java.nio.file.Paths
+import java.util.regex.Pattern
+
 plugins {
   id "com.jetbrains.python.envs" version "0.0.30"
 }
@@ -98,6 +101,62 @@ task buildBinaries() {
   }
 }
 
+task checkVersionOfCFiles() {
+  description "Checks if the binary version in C files matches the regular version."
+  doLast {
+    File additionalThreadInfoRegular = Paths.get(project.ext.pydevDir.path, "_pydevd_bundle", "pydevd_additional_thread_info_regular.py").toFile()
+    File cFile = Paths.get(project.ext.pydevDir.path, "_pydevd_bundle", "pydevd_cython.c").toFile()
+    Integer regularVersion = extractVersion(additionalThreadInfoRegular, ~/version = (\d+)/)
+    Integer cVersion = extractVersion(cFile, ~/\s+version = (\d+)/)
+    assert regularVersion == cVersion : "The version '$regularVersion' in ${additionalThreadInfoRegular.name} does not matches the '$cVersion' " +
+                                        "version in ${cFile.name}"
+  }
+}
+
+task checkCFilesAreUpToDate() {
+  description "Checks if the C files are up to date by checking their creation time and binary version."
+  dependsOn checkVersionOfCFiles
+  doLast {
+    String pydevdRoot = project.ext.pydevDir.path
+
+    def mappings = [
+      ["_pydevd_bundle", "pydevd_cython"],
+      ["_pydevd_frame_eval", "pydevd_frame_evaluator_common"],
+      ["_pydevd_frame_eval", "pydevd_frame_evaluator"],
+      ["_pydevd_frame_eval", "pydevd_frame_evaluator_py39"]
+    ]
+
+    for (mapping in mappings) {
+      def (dir, extName) = mapping
+      File pxdFile = Paths.get(pydevdRoot, dir, extName + ".pxd").toFile()
+      File pyxFile = Paths.get(pydevdRoot, dir, extName + ".pyx").toFile()
+      File cFile = Paths.get(pydevdRoot, dir, extName + ".c").toFile()
+
+      def checkTimespamps = { File mainFile, File dependentFile ->
+        assert dependentFile.lastModified() > mainFile.lastModified() : "${dependentFile.name} is older than ${mainFile.name}"
+      }
+
+      checkTimespamps(pxdFile, cFile)
+      checkTimespamps(pyxFile, cFile)
+    }
+
+    println "The C files are up to date."
+  }
+}
+
+Integer extractVersion(File file, Pattern versionPattern) {
+  file.withReader {reader ->
+    def line
+    while ((line = reader.readLine()) != null) {
+      def matcher = line =~ versionPattern
+      if (matcher.find()) {
+        return Integer.parseInt(matcher.group(1))
+      }
+    }
+    throw new RuntimeException("Can't find version in ${file.path}")
+  }
+}
+
 task wrapper(type: Wrapper) doLast {
   gradleVersion = '4.2'
-}
+}

python/helpers/pydev/pycharm-readme.rst

@@ -0,0 +1,61 @@
+Synchronization with PyDev.Debugger
+===================================
+
+The sources in this folder are PyCharm version of the debugger `PyDev.Debugger`_.
+They should be in sync, so after testing all commits should be
+back-ported to the original repo.
+
+.. _PyDev.Debugger: https://github.com/fabioz/PyDev.Debugger.git
+
+How to do it?
+
+a) You should have PyDev.Debugger repo clone and fetch there IDEA commits related to this folder with changed paths from python/pydev/* to *
+b) Then cherry-pick all necessary commits resolving merge conflicts
+
+Steps:
+
+1) git clone https://github.com/fabioz/PyDev.Debugger.git
+2) <clone IDEA Community repo>
+and in this folder:
+3) git remote rm origin
+4) git tag -l | xargs git tag -d
+5) git filter-branch --subdirectory-filter python/helpers/pydev --prune-empty --index-filter 'SHA=$(git write-tree); rm $GIT_INDEX_FILE && git read-tree --prefix= $SHA' -- --all
+in PyDev.Debugger folder:
+6) git remote add temp-repo <path/to/idea/folder>
+7) git fetch temp-repo
+8) Cherry-pick all the changes
+
+Note about C Files
+==================
+
+PyCharm does not ship with pre-built Cython extensions for
+Linux. Instead, the extensions are built on site. The mentioned C
+files are:
+
+- ``pydevd_cython.c`` from ``_pydevd_bundle``
+- ``pydevd_frame_evaluator_common.c``
+- ``pydevd_frame_evaluator.c``
+- ``pydevd_frame_evaluator_py39.c`` (for Python 3.9 and above)
+
+The latest three are located in ``_pydevd_frame_eval`` package.
+
+These C files are generated by the Cython but they **aren't saved**
+during the standard extension build process initiated by calling the
+``buildBinaries`` task from ``build-debug-binaries`` Gradle
+project. To regenerate the C files the following two commands
+should be called from the ``pydev`` root directory:
+
+::
+
+   PYTHONPATH=python build_tools/build.py --no-remove-binaries
+   PYTHONPATH=python3.9 build_tools/build.py --no-remove-binaries
+
+ Note, that it's important to run the build script twice: one for the
+ versions older than 3.9 and another for Python 3.9 and above.
+
+After that the ``checkCFilesAreUpToDate`` Gradle task can be run to
+check the C files are up to date, e.g.:
+
+::
+
+   ./gradlew -q checkCFileAreUpToDate

python/helpers/pydev/pycharm-readme.txt

@@ -1,20 +0,0 @@
-The sources in this folder are PyCharm version of the debugger PyDev.Debugger.
-They should be in sync, so after testing all commits should be back-ported to the original repo.
-
-How to do it?
-
-a) You should have PyDev.Debugger repo clone and fetch there IDEA commits related to this folder with changed paths from python/pydev/* to *
-b) Then cherry-pick all necessary commits resolving merge conflicts
-
-Steps:
-
-1) git clone https://github.com/fabioz/PyDev.Debugger.git
-2) <clone IDEA Community repo>
-and in this folder:
-3) git remote rm origin
-4) git tag -l | xargs git tag -d
-5) git filter-branch --subdirectory-filter python/helpers/pydev --prune-empty --index-filter 'SHA=$(git write-tree); rm $GIT_INDEX_FILE && git read-tree --prefix= $SHA' -- --all
-in PyDev.Debugger folder:
-6) git remote add temp-repo <path/to/idea/folder>
-7) git fetch temp-repo
-8) Cherry-pick all the changes