openide/python/setup-test-environment/build.gradle

// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.

import org.apache.tools.ant.taskdefs.condition.Os

import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths

plugins {
  id "com.jetbrains.python.envs" version "0.0.31"
}

boolean isWindows = Os.isFamily(Os.FAMILY_WINDOWS)
boolean isUnix = Os.isFamily(Os.FAMILY_UNIX)

/**
 * Installs python interpreters for env. tests using conda or CPython.
 * Utilizes following env variables:
 *
 * PYCHARM_PYTHONS -- path to install cPythons
 * PYCHARM_PYTHON_VIRTUAL_ENVS -- path to install condas
 *
 * PYCHARM_USE_CONDA -- use conda (cPython will be used if not set)
 *
 * PYCHARM_ZIP_REPOSITORY -- to download unpacked pythons for Windows (default cpython does not support unattended installation)
 * Recommended value: https://repo.labs.intellij.net/pycharm/python-archives-windows/
 *
 * Pitfall: TMP var on windows points to very long path inside of user local dir and may lead to errors.
 * It is recommended to create "c:\temp\" with full write access and set TMP="c:\temp\" on Windows.
 *
 * When -DdjangoTrunkOnly passed, only install django_latest for Django Trunk tests
 *
 * ``PyEnvTestSettings`` class uses these vars also.
 *
 *
 */

envs {
  String python27version = "2.7.18"
  String python36version = (isWindows) ? "3.6.8" : "3.6.15" // 3.6.8 is broken on modern libc and clang, but latest version for Windows
  String python37version = "3.7.9"
  String python38version = "3.8.10"
  String python39version = "3.9.13"
  String python310version = "3.10.8"
  String python311version = "3.11.0"
  String python312version = "3.12.0rc3"

  bootstrapDirectory = new File(System.getenv().getOrDefault("PYCHARM_PYTHONS", new File(buildDir, 'pythons').path))
  envsDirectory = new File(System.getenv().getOrDefault("PYCHARM_PYTHON_VIRTUAL_ENVS", new File(buildDir, 'envs').path))

  def djangoTrunkOnly = System.getProperty("djangoTrunkOnly") ?: false


  if (isWindows) {
    // On Windows you almost always want to use cache server except you are outside of JB
    def zipRepositoryURL = new URL(
      System.getenv().getOrDefault("PYCHARM_ZIP_REPOSITORY", "https://repo.labs.intellij.net/pycharm/python-archives-windows/"))
    try {
      zipRepositoryURL.getContent()
    }
    catch (Exception ignored) {
      zipRepositoryURL = null
      System.err.println("Cache server is only accessible from JB network. Will try to build python from scratch. Use cache if possible")
    }
    if (zipRepositoryURL != null) {
      zipRepository = zipRepositoryURL
      shouldUseZipsFromRepository = true
    }
  }

  Boolean shouldUseCondaInterpreters = System.getenv().getOrDefault("PYCHARM_USE_CONDA", "false").toBoolean()

  Closure createPython = { String pythonName,
                           String version,
                           List<String> packages = null,
                           String tags = null,
                           Boolean createLink = false,
                           Boolean forceConda = false ->
    File pythonDirectory
    if (shouldUseCondaInterpreters || forceConda) {
      condaenv pythonName, version, packages
      pythonDirectory = new File(envsDirectory, pythonName)
    }
    else {
      python pythonName, version, packages
      pythonDirectory = new File(bootstrapDirectory, pythonName)
    }

    print("Python dir: $pythonDirectory\n")
    project.tasks.create("Misc for $pythonName") {
      shouldRunAfter 'build_envs'

      doLast {
        if (tags) new File(pythonDirectory, "tags.txt").write(tags)

        String versionMajor = version.split(/\./)[0]
        String versionMinor = version.split(/\./)[1]
        versionMinor = versionMinor.endsWith("-dev") ? versionMinor.substring(0, versionMinor.indexOf("-dev"))
                                                     : versionMinor
        // Pregenerate pyc
        String linkName = "python$versionMajor.$versionMinor"
        def path = pythonDirectory.toPath()
        def python = path.resolve(isWindows ? "python.exe" : "bin/${linkName}").toString()
        def lib = path.resolve("lib").toString()
        def command = "import compileall; compileall.compile_dir('${lib}', force=True, quiet=True)"

        def result = new StringBuilder()
        def error = new StringBuilder()
        def process = "${python} -c \"$command\"".execute()
        process.consumeProcessErrorStream(error)
        process.consumeProcessOutputStream(result)
        process.waitForOrKill(60000)

        if (createLink) {
          Closure createLinkClosure = { Path link, Path existing ->
            if (!link.toFile().exists()) Files.createLink(link, existing)
          }

          if (isUnix && !shouldUseCondaInterpreters) {
            createLinkClosure(Paths.get(pythonDirectory.toString(), linkName),
                              Paths.get(pythonDirectory.toString(), "bin/$linkName"))
          }
          else if (isWindows) {
            createLinkClosure(Paths.get(pythonDirectory.toString(), "${linkName}.exe"),
                              Paths.get(pythonDirectory.toString(), "python.exe"))
          }
        }
      }
    }
  }


  createPython("py36_django_latest", python36version, ["django", "behave-django", "behave", "pytest", "untangle"],
               "python3.6\ndjango\ndjango20\nbehave\nbehave-django\ndjango2\npytest\nuntangle", true)
  if (djangoTrunkOnly) {
    return
  }

  if (isUnix) {
    // For TensorFlow
    createPython("py37",
                 python37version,
                 ["tensorflow", "pyqt5==5.12", "PySide2==5.12.1"],
                 "tensorflow\npython3.7\nqt",
                 true)
  }
  else {
    // qt is for unix only
    createPython("py37",
                 python37version,
                 ["tensorflow"],
                 "tensorflow\npython3.7",
                 true)
  }

  createPython("py38",
               python38version,
               // The version of Jinja2 is fixed due to a bug in recent versions which breaks the debugging.
               // See: https://github.com/pallets/jinja/pull/1178.
               // The version of NumPy is fixed due to https://tinyurl.com/y3dm3h86.
               ["ipython==7.8", "django==2.2", "behave", "jinja2==3.1.2", "tox>=2.0", "nose", "pytest", "django-nose", "behave-django",
                "pytest-xdist", "untangle", "numpy==1.19.3", "pandas"],
               "python3.8\npython3\nipython\nipython780\nskeletons\ndjango\nbehave\nbehave-django\ntox\njinja2\npackaging" +
               "\npytest\nnose\ndjango-nose\nbehave-django\ndjango2\nxdist\nuntangle\npandas",
               true)

  createPython("python3.9",
               python39version,
               ["pytest", "pytest-xdist"],
               "python3.9\npython3\npytest\nxdist\npackaging",
               true)

  createPython("python3.10",
               python310version,
               [],
               "python3.10",
               true)

  createPython("python3.11",
               python311version,
               ["black == 23.1.0"],
               "python3.11\nblack",
               true)

  createPython("python3.12",
               python312version,
               ["teamcity-messages"],
               "python3\npython3.12\nmessages")

  createPython("py27",
               python27version,
               ["tox>=3.8.3", "nose", "pytest", "Twisted", "behave", "teamcity-messages",
                "untangle"] + (isWindows ? ['pypiwin32'] : []), //win32api is required for pypiwin32
               "python2.7\nnose\npytest\nbehave\npackaging\ntox\ntwisted\ndjango-nose\nuntangle\nmessages",
               true)
}

/**
 * Kills any process named "python" using windows powershell.
 * Works on Win7+.
 */
task killPythonWindows(type: Exec) {
  onlyIf { isWindows }

  commandLine 'powershell', '"Get-Process | where {$_.Name -ieq \\"python\\"} | Stop-Process"'
}

task prepare(type: Delete) {
  // Python leaked from previous test may prevent this script from
  // deleting folder because you can't delete file opened by process on Windows
  dependsOn killPythonWindows

  doFirst {
    new File(envs.bootstrapDirectory, "py36_django_latest").with { djangoLatestFolder ->
      //     if (djangoLatestFolder.exists() && djangoLatestFolder.lastModified() < System.currentTimeMillis() - 24 * 60 * 60 * 1000) {
      // older then a day
      println "Cleaning django_latest at" + djangoLatestFolder
      delete djangoLatestFolder
      //   }
    }

//    if (!envs.bootstrapDirectory.exists() || (System.getenv("NO_CLEAN") == null && envs.envsDirectory.exists() &&
    //                                            envs.envsDirectory.lastModified() < project.buildscript.sourceFile.lastModified())) {
    // clean the cache if the build script was modified later
    println "Cleaning cached environments at " + envs.envsDirectory
    delete envs.envsDirectory
    println "Cleaning cached pythons at " + envs.bootstrapDirectory
    delete envs.bootstrapDirectory
    //}
  }
}

task build {
  mustRunAfter prepare
  dependsOn prepare, 'build_envs', tasks.findAll { it.name.startsWith("Misc") }
}