IDEA-CR-40781 extract universal stub generation to separate step

python/build/groovy/org/jetbrains/intellij/build/pycharm/PyCharmBuildOptions.groovy

@@ -0,0 +1,21 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package org.jetbrains.intellij.build.pycharm
+
+import com.intellij.util.SystemProperties
+import groovy.transform.CompileStatic
+
+/**
+ * @author Aleksey.Rostovskiy
+ */
+@CompileStatic
+class PyCharmBuildOptions {
+  /**
+   * Pass 'true' to this system property to skip universal stubs generation and use prebuilt ones.
+   */
+  static final boolean usePrebuiltStubs = SystemProperties.getBooleanProperty("intellij.build.pycharm.use.prebuilt.stubs", false)
+
+  /**
+   * Path to a zip file containing generated universal stubs.
+   */
+  static final String prebuiltStubsArchive = System.getProperty("intellij.build.pycharm.prebuilt.stubs.archive")
+}

python/build/groovy/org/jetbrains/intellij/build/pycharm/PyCharmPropertiesBase.groovy

@@ -15,12 +15,14 @@
  */
 package org.jetbrains.intellij.build.pycharm
 
+import groovy.io.FileType
 import org.jetbrains.intellij.build.*
 
 /**
  * @author nik
  */
 abstract class PyCharmPropertiesBase extends ProductProperties {
+
   PyCharmPropertiesBase() {
     baseFileName = "pycharm"
     reassignAltClickToMultipleCarets = true
@@ -46,44 +48,102 @@ abstract class PyCharmPropertiesBase extends ProductProperties {
       }
     }
 
-    new PyPrebuiltIndicesGenerator().generateResources(context)
+    File folderWithUnzipContent = new File(context.paths.temp, "unzips")
+    folderWithUnzipContent.mkdirs()
+    unzipArchives(context, folderWithUnzipContent)
 
+    File temporaryIndexFolder = new File(context.paths.temp, "index")
+    temporaryIndexFolder.deleteDir()
+    temporaryIndexFolder.mkdirs()
+
+    boolean forceGenerate = false
+    if (PyCharmBuildOptions.usePrebuiltStubs) {
+      File stubsArchive = new File(context.paths.projectHome, PyCharmBuildOptions.prebuiltStubsArchive)
+      context.ant.echo("Unzip prebuilt stubs ${stubsArchive.absolutePath}")
+      context.ant.unzip(src: stubsArchive.absolutePath, dest: "${temporaryIndexFolder.absolutePath}")
+
+      try {
+        String stubsVersionPath = new FileNameFinder()
+          .getFileNames("${temporaryIndexFolder.absolutePath}/Python", "sdk-stubs.version").
+          first()
+        int stubsFromArchiveVersion = new File(stubsVersionPath).readLines()[0].toInteger()
+        context.ant.echo("Stubs version $stubsFromArchiveVersion")
+        int stubVersion = getStubVersion(context)
+        context.ant.echo("But should be $stubVersion")
+
+        forceGenerate = stubsFromArchiveVersion != stubVersion
+      }
+      catch (ignored) {
+        forceGenerate = true
+      }
+    }
+
+    if (forceGenerate || !PyCharmBuildOptions.usePrebuiltStubs) {
+      temporaryIndexFolder.deleteDir()
+      temporaryIndexFolder.mkdir()
+      generateUniversalStubs(context, folderWithUnzipContent, temporaryIndexFolder)
+    }
+
+    context.ant.echo("Generate indices")
+    generateIndices(context, folderWithUnzipContent, temporaryIndexFolder)
+    context.ant.echo("Copy indices")
     context.ant.copy(todir: "$targetDirectory/index", failonerror: !context.options.isInDevelopmentMode) {
-      fileset(dir: "$context.paths.temp/index", erroronmissingdir: !context.options.isInDevelopmentMode) {
+      fileset(dir: temporaryIndexFolder.absolutePath, erroronmissingdir: !context.options.isInDevelopmentMode) {
         include(name: "**")
       }
     }
+
+    folderWithUnzipContent.deleteDir()
+    temporaryIndexFolder.deleteDir()
   }
 
   @Override
   String getEnvironmentVariableBaseName(ApplicationInfoProperties applicationInfo) {
     "PYCHARM"
   }
-}
 
-class PyPrebuiltIndicesGenerator implements ResourcesGenerator {
-  @Override
-  File generateResources(BuildContext context) {
+  private void unzipArchives(BuildContext context, File destination) {
+    File tempFolder = new File(context.paths.temp, "zips")
+    tempFolder.mkdirs()
+    tempFolder.deleteOnExit()
+    context.ant.copy(todir: tempFolder.absolutePath, failonerror: !context.options.isInDevelopmentMode) {
+      fileset(dir: "$context.paths.projectHome/python-distributions", erroronmissingdir: !context.options.isInDevelopmentMode) {
+        include(name: "*.zip")
+      }
+      fileset(dir: "$context.paths.projectHome/skeletons", erroronmissingdir: !context.options.isInDevelopmentMode) {
+        include(name: "*.zip")
+      }
+    }
+
+    tempFolder.eachFileRecurse(FileType.FILES) {
+      if (it.name.endsWith('.zip')) {
+        context.ant.unzip(src: it.absolutePath, dest: "${destination.absolutePath}/${it.name}")
+      }
+    }
+  }
+
+  private int getStubVersion(BuildContext context) {
+    List<String> buildClasspath = context.getModuleRuntimeClasspath(context.findModule("intellij.python.tools"), false)
+
+    context.ant.java(classname: "com.jetbrains.python.tools.GetPyStubsVersionKt", fork: true, outputproperty: "stubsVersion") {
+      classpath {
+        buildClasspath.each {
+          pathelement(location: it)
+        }
+      }
+    }
+    return context.ant.project.properties.stubsVersion as int
+  }
+
+  private void generateUniversalStubs(BuildContext context, File from, File to) {
     CompilationTasks.create(context).compileModules(["intellij.python.tools"])
     List<String> buildClasspath = context.getModuleRuntimeClasspath(context.findModule("intellij.python.tools"), false)
 
-    String zipPath = "$context.paths.temp/zips"
-    String outputPath = "$context.paths.temp/index"
-
     context.messages.block("Generate universal stubs") {
-      context.ant.copy(todir: zipPath, failonerror: !context.options.isInDevelopmentMode) {
-        fileset(dir: "$context.paths.projectHome/python-distributions", erroronmissingdir: !context.options.isInDevelopmentMode) {
-          include(name: "*.zip")
-        }
-        fileset(dir: "$context.paths.projectHome/skeletons", erroronmissingdir: !context.options.isInDevelopmentMode) {
-          include(name: "*.zip")
-        }
-      }
-
-      context.ant.java(classname: "com.jetbrains.python.tools.PyPrebuiltIndicesGeneratorKt", fork: true) {
+      context.ant.java(classname: "com.jetbrains.python.tools.PythonUniversalStubsBuilderKt", fork: true) {
         jvmarg(line: "-ea -Xmx1000m")
-        arg(value: zipPath)
-        arg(value: outputPath)
+        arg(value: from.absolutePath)
+        arg(value: to.absolutePath)
         classpath {
           buildClasspath.each {
             pathelement(location: it)
@@ -91,7 +151,21 @@ class PyPrebuiltIndicesGenerator implements ResourcesGenerator {
         }
       }
     }
+  }
 
-    return new File(outputPath)
+  private void generateIndices(BuildContext context, File from, File to) {
+    CompilationTasks.create(context).compileModules(["intellij.python.tools"])
+    List<String> buildClasspath = context.getModuleRuntimeClasspath(context.findModule("intellij.python.tools"), false)
+
+    context.ant.java(classname: "com.jetbrains.python.tools.IndicesBuilderKt", fork: true) {
+      jvmarg(line: "-ea -Xmx1000m")
+      arg(value: from.absolutePath)
+      arg(value: to.absolutePath)
+      classpath {
+        buildClasspath.each {
+          pathelement(location: it)
+        }
+      }
+    }
   }
 }

python/tools/intellij.python.tools.iml

@@ -18,5 +18,6 @@
     <orderEntry type="module" module-name="intellij.pycharm.community" exported="" scope="RUNTIME" />
     <orderEntry type="module" module-name="intellij.platform.bootstrap" />
     <orderEntry type="library" name="KotlinScriptRuntime" level="project" />
+    <orderEntry type="module" module-name="intellij.pycharm.community.build" />
   </component>
 </module>

python/tools/src/com/jetbrains/python/tools/BuildZippedSkeletons.kt

@@ -62,9 +62,9 @@ fun main(args: Array<String>) {
 
 
       val dirPacked = File(skeletonsDir.parent, DefaultPregeneratedSkeletonsProvider.getPregeneratedSkeletonsName(sdk, refresher.generatorVersion, true, true))
-      val zip = ZipOutputStream(FileOutputStream(dirPacked))
-      ZipUtil.addDirToZipRecursively(zip, dirPacked, skeletonsDir, "", null, null)
-      zip.close()
+      ZipOutputStream(FileOutputStream(dirPacked)).use {
+        ZipUtil.addDirToZipRecursively(it, dirPacked, skeletonsDir, "", null, null)
+      }
     }
   }
   catch (e: Exception) {

python/tools/src/com/jetbrains/python/tools/GetPyStubsVersion.kt

@@ -0,0 +1,12 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.jetbrains.python.tools
+
+import com.jetbrains.python.psi.PyFileElementType
+
+
+/**
+ * This script is needed for PyCharm installation process to check indexer version of pre-built stubs
+ */
+fun main(args: Array<String>) {
+  println(PyFileElementType.INSTANCE.stubVersion)
+}

python/tools/src/com/jetbrains/python/tools/IndicesBuilder.kt

@@ -0,0 +1,35 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.jetbrains.python.tools
+
+import com.intellij.index.PrebuiltIndexAwareIdIndexer
+import com.jetbrains.python.psi.impl.stubs.PyPrebuiltStubsProvider
+import org.jetbrains.index.id.IdIndexGenerator
+
+
+/**
+ * @author Aleksey.Rostovskiy
+ */
+fun main(args: Array<String>) {
+  require(args.size == 2) {
+    "Usage: IndicesBuilderKt <input folder with files> <output folder to store indices>"
+  }
+  IndicesBuilder.build(args[0], "${args[1]}/${PyPrebuiltStubsProvider.NAME}")
+}
+
+object IndicesBuilder: PyGeneratorBase() {
+  fun build(root: String, outputPath: String) {
+    try {
+      app
+
+      val files = rootFiles(root)
+      IdIndexGenerator("$outputPath/${PrebuiltIndexAwareIdIndexer.ID_INDEX_FILE_NAME}")
+        .buildIdIndexForRoots(files)
+    }
+    catch (e: Throwable) {
+      e.printStackTrace()
+    }
+    finally {
+      tearDown()
+    }
+  }
+}

python/tools/src/com/jetbrains/python/tools/PyGeneratorBase.kt

@@ -0,0 +1,66 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.jetbrains.python.tools
+
+import com.intellij.ide.BootstrapClassLoaderUtil
+import com.intellij.idea.IdeaTestApplication
+import com.intellij.openapi.application.PathManager
+import com.intellij.openapi.application.WriteAction
+import com.intellij.openapi.util.io.FileUtil
+import com.intellij.openapi.vfs.LocalFileSystem
+import com.intellij.openapi.vfs.VirtualFile
+import com.intellij.util.PlatformUtils
+import com.intellij.util.io.ZipUtil
+import com.intellij.util.ui.UIUtil
+import java.io.File
+
+
+/**
+ * @author Aleksey.Rostovskiy
+ */
+open class PyGeneratorBase {
+  protected val app: IdeaTestApplication by lazy {
+    System.setProperty(PlatformUtils.PLATFORM_PREFIX_KEY, PlatformUtils.PYCHARM_CE_PREFIX)
+    System.setProperty(PathManager.PROPERTY_PLUGINS_PATH, FileUtil.createTempDirectory("pystubs", "plugins").absolutePath)
+    System.setProperty(PathManager.PROPERTY_SYSTEM_PATH, FileUtil.createTempDirectory("pystubs", "system").absolutePath)
+    System.setProperty(PathManager.PROPERTY_CONFIG_PATH, FileUtil.createTempDirectory("pystubs", "config").absolutePath)
+    Thread.currentThread().contextClassLoader = BootstrapClassLoaderUtil.initClassLoader()
+    IdeaTestApplication.getInstance()
+  }
+
+  protected fun rootFiles(root: String): ArrayList<VirtualFile>{
+    val dir = File(root)
+    if (!dir.exists() || !dir.isDirectory) {
+      throw IllegalStateException("$root doesn't exist or isn't a directory")
+    }
+
+    return ArrayList(dir.listFiles().map { it->
+      LocalFileSystem.getInstance().findFileByIoFile(it)!!
+    })
+  }
+
+  protected fun unzipArchivesToRoots(root: String): ArrayList<VirtualFile> {
+    val dir = File(root)
+    if (!dir.exists() || !dir.isDirectory) {
+      throw IllegalStateException("$root doesn't exist or isn't a directory")
+    }
+    val files = dir.listFiles { _, name -> name.endsWith(".zip") }.map { zip ->
+      val unzipRoot = File(root, zip.nameWithoutExtension)
+      println("Extracting $zip")
+      ZipUtil.extract(zip, unzipRoot, null)
+      unzipRoot
+    }
+    return ArrayList(files.map { it ->
+      LocalFileSystem.getInstance().findFileByIoFile(it)!!
+    })
+  }
+
+  protected fun tearDown() {
+    UIUtil.invokeAndWaitIfNeeded(Runnable {
+      WriteAction.run<Throwable> {
+        app.dispose()
+      }
+    })
+    FileUtil.delete(File(System.getProperty(PathManager.PROPERTY_PLUGINS_PATH)))
+    FileUtil.delete(File(System.getProperty(PathManager.PROPERTY_SYSTEM_PATH)))
+  }
+}

python/tools/src/com/jetbrains/python/tools/PyPrebuiltIndicesGenerator.kt

@@ -1,74 +0,0 @@
-// Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
-package com.jetbrains.python.tools
-
-import com.intellij.ide.BootstrapClassLoaderUtil
-import com.intellij.idea.IdeaTestApplication
-import com.intellij.index.PrebuiltIndexAwareIdIndexer
-import com.intellij.openapi.application.PathManager
-import com.intellij.openapi.application.WriteAction
-import com.intellij.openapi.util.io.FileUtil
-import com.intellij.openapi.vfs.LocalFileSystem
-import com.intellij.psi.stubs.PrebuiltStubsProviderBase
-import com.intellij.testFramework.PlatformTestCase
-import com.intellij.util.ReflectionUtil
-import com.intellij.util.io.ZipUtil
-import com.intellij.util.ui.UIUtil
-import com.jetbrains.python.psi.impl.stubs.PyPrebuiltStubsProvider
-import org.jetbrains.index.id.IdIndexGenerator
-import java.io.File
-import java.util.*
-
-fun main(args: Array<String>) {
-  buildIndices(args[0], "${args[1]}/${PyPrebuiltStubsProvider.NAME}")
-}
-
-fun buildIndices(root: String, outputPath: String) {
-  val app = createApp()
-  try {
-    FileUtil.delete(File(outputPath))
-
-    val roots = unzipArchivesToRoots(root)
-
-    val rootFiles = ArrayList(roots.map { it -> LocalFileSystem.getInstance().findFileByIoFile(it)!! })
-
-    PyStubsGenerator("$outputPath/${PrebuiltStubsProviderBase.SDK_STUBS_STORAGE_NAME}").buildStubsForRoots(rootFiles)
-
-    IdIndexGenerator("$outputPath/${PrebuiltIndexAwareIdIndexer.ID_INDEX_FILE_NAME}").buildIdIndexForRoots(rootFiles)
-  }
-  catch (e: Throwable) {
-    e.printStackTrace()
-  }
-  finally {
-    UIUtil.invokeAndWaitIfNeeded(Runnable {
-      WriteAction.run<Throwable> {
-        app.dispose()
-      }
-    })
-    FileUtil.delete(File(System.getProperty(PathManager.PROPERTY_PLUGINS_PATH)))
-    FileUtil.delete(File(System.getProperty(PathManager.PROPERTY_SYSTEM_PATH)))
-    System.exit(0)
-  }
-}
-
-fun unzipArchivesToRoots(root: String): List<File> {
-  val dir = File(root)
-  if (!dir.exists() || !dir.isDirectory) {
-    throw IllegalStateException("$root doesn't exist or isn't a directory")
-  }
-  return dir.listFiles { _, name -> name.endsWith(".zip") }.map { zip ->
-    val unzipRoot = File(root, zip.nameWithoutExtension)
-    println("Extracting $zip")
-    ZipUtil.extract(zip, unzipRoot, null)
-    unzipRoot
-  }
-}
-
-private fun createApp(): IdeaTestApplication {
-  val candidates = ReflectionUtil.getField(PlatformTestCase::class.java, null, Array<String>::class.java, "PREFIX_CANDIDATES")
-  candidates[0] = "PyCharm"
-  System.setProperty(PathManager.PROPERTY_PLUGINS_PATH, FileUtil.createTempDirectory("pystubs", "plugins").absolutePath)
-  System.setProperty(PathManager.PROPERTY_SYSTEM_PATH, FileUtil.createTempDirectory("pystubs", "system").absolutePath)
-  System.setProperty(PathManager.PROPERTY_CONFIG_PATH, FileUtil.createTempDirectory("pystubs", "config").absolutePath)
-  Thread.currentThread().contextClassLoader = BootstrapClassLoaderUtil.initClassLoader()
-  return IdeaTestApplication.getInstance()
-}

python/tools/src/com/jetbrains/python/tools/PythonUniversalStubsBuilder.kt

@@ -0,0 +1,78 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.jetbrains.python.tools
+
+import com.intellij.openapi.util.io.FileUtil
+import com.intellij.psi.stubs.PrebuiltStubsProviderBase
+import com.intellij.util.io.ZipUtil
+import com.jetbrains.python.psi.impl.stubs.PyPrebuiltStubsProvider
+import org.jetbrains.intellij.build.pycharm.PyCharmBuildOptions
+import java.io.File
+import java.io.FileOutputStream
+import java.util.zip.ZipOutputStream
+
+
+/**
+ * @author Aleksey.Rostovskiy
+ */
+fun main(args: Array<String>) {
+  if (args.size != 2) {
+    val zipsDirectory = System.getProperty("pycharm.zips.directory")
+    val prebuiltStubsArchive = PyCharmBuildOptions.getPrebuiltStubsArchive()
+
+    if (zipsDirectory.isNullOrBlank() || prebuiltStubsArchive.isNullOrBlank()) {
+      throw IllegalArgumentException(
+        "Usage: PythonUniversalStubsBuilderKt <input folder with files> <output folder to store universal stubs>"
+      )
+    }
+    PythonUniversalStubsBuilder.generateArchive(zipsDirectory, prebuiltStubsArchive)
+  }
+  else {
+    PythonUniversalStubsBuilder.generateStubs(args[0], "${args[1]}/${PyPrebuiltStubsProvider.NAME}")
+  }
+}
+
+object PythonUniversalStubsBuilder : PyGeneratorBase() {
+  fun generateStubs(root: String, outputPath: String) {
+    try {
+      app
+
+      val files = rootFiles(root)
+      PyStubsGenerator("$outputPath/${PrebuiltStubsProviderBase.SDK_STUBS_STORAGE_NAME}")
+        .buildStubsForRoots(files)
+    }
+    catch (e: Throwable) {
+      e.printStackTrace()
+    }
+    finally {
+      tearDown()
+    }
+  }
+
+  fun generateArchive(zipsDirectory: String, prebuiltStubsArchive: String) {
+    val tmpFolder = FileUtil.createTempDirectory("stubs", null)
+    tmpFolder.delete()
+    tmpFolder.mkdirs()
+    tmpFolder.deleteOnExit()
+
+    try {
+      app
+
+      val unzippedFiles = unzipArchivesToRoots(zipsDirectory)
+      PyStubsGenerator("${tmpFolder.absolutePath}/${PrebuiltStubsProviderBase.SDK_STUBS_STORAGE_NAME}")
+        .buildStubsForRoots(unzippedFiles)
+    }
+    catch (e: Throwable) {
+      e.printStackTrace()
+    }
+    finally {
+      tearDown()
+    }
+
+    println("Generate archive $prebuiltStubsArchive")
+    val archive = File(prebuiltStubsArchive)
+    ZipOutputStream(FileOutputStream(archive)).use {
+      ZipUtil.addFileOrDirRecursively(it, archive, tmpFolder, PyPrebuiltStubsProvider.NAME, null, null)
+    }
+  }
+
+}