From 277c0cde030613b1ea443ddd8e7e541a4a86b4c0 Mon Sep 17 00:00:00 2001 From: Aleksey Rostovskiy Date: Tue, 27 Nov 2018 12:12:52 +0300 Subject: [PATCH] IDEA-CR-40781 extract universal stub generation to separate step --- .../build/pycharm/PyCharmBuildOptions.groovy | 21 ++++ .../pycharm/PyCharmPropertiesBase.groovy | 118 ++++++++++++++---- python/tools/intellij.python.tools.iml | 1 + .../python/tools/BuildZippedSkeletons.kt | 6 +- .../python/tools/GetPyStubsVersion.kt | 12 ++ .../jetbrains/python/tools/IndicesBuilder.kt | 35 ++++++ .../jetbrains/python/tools/PyGeneratorBase.kt | 66 ++++++++++ .../tools/PyPrebuiltIndicesGenerator.kt | 74 ----------- .../tools/PythonUniversalStubsBuilder.kt | 78 ++++++++++++ 9 files changed, 312 insertions(+), 99 deletions(-) create mode 100644 python/build/groovy/org/jetbrains/intellij/build/pycharm/PyCharmBuildOptions.groovy create mode 100644 python/tools/src/com/jetbrains/python/tools/GetPyStubsVersion.kt create mode 100644 python/tools/src/com/jetbrains/python/tools/IndicesBuilder.kt create mode 100644 python/tools/src/com/jetbrains/python/tools/PyGeneratorBase.kt delete mode 100644 python/tools/src/com/jetbrains/python/tools/PyPrebuiltIndicesGenerator.kt create mode 100644 python/tools/src/com/jetbrains/python/tools/PythonUniversalStubsBuilder.kt diff --git a/python/build/groovy/org/jetbrains/intellij/build/pycharm/PyCharmBuildOptions.groovy b/python/build/groovy/org/jetbrains/intellij/build/pycharm/PyCharmBuildOptions.groovy new file mode 100644 index 000000000000..d4f84066a6ff --- /dev/null +++ b/python/build/groovy/org/jetbrains/intellij/build/pycharm/PyCharmBuildOptions.groovy @@ -0,0 +1,21 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package org.jetbrains.intellij.build.pycharm + +import com.intellij.util.SystemProperties +import groovy.transform.CompileStatic + +/** + * @author Aleksey.Rostovskiy + */ +@CompileStatic +class PyCharmBuildOptions { + /** + * Pass 'true' to this system property to skip universal stubs generation and use prebuilt ones. + */ + static final boolean usePrebuiltStubs = SystemProperties.getBooleanProperty("intellij.build.pycharm.use.prebuilt.stubs", false) + + /** + * Path to a zip file containing generated universal stubs. + */ + static final String prebuiltStubsArchive = System.getProperty("intellij.build.pycharm.prebuilt.stubs.archive") +} diff --git a/python/build/groovy/org/jetbrains/intellij/build/pycharm/PyCharmPropertiesBase.groovy b/python/build/groovy/org/jetbrains/intellij/build/pycharm/PyCharmPropertiesBase.groovy index a7be3de81541..38252f45106b 100644 --- a/python/build/groovy/org/jetbrains/intellij/build/pycharm/PyCharmPropertiesBase.groovy +++ b/python/build/groovy/org/jetbrains/intellij/build/pycharm/PyCharmPropertiesBase.groovy @@ -15,12 +15,14 @@ */ package org.jetbrains.intellij.build.pycharm +import groovy.io.FileType import org.jetbrains.intellij.build.* /** * @author nik */ abstract class PyCharmPropertiesBase extends ProductProperties { + PyCharmPropertiesBase() { baseFileName = "pycharm" reassignAltClickToMultipleCarets = true @@ -46,44 +48,102 @@ abstract class PyCharmPropertiesBase extends ProductProperties { } } - new PyPrebuiltIndicesGenerator().generateResources(context) + File folderWithUnzipContent = new File(context.paths.temp, "unzips") + folderWithUnzipContent.mkdirs() + unzipArchives(context, folderWithUnzipContent) + File temporaryIndexFolder = new File(context.paths.temp, "index") + temporaryIndexFolder.deleteDir() + temporaryIndexFolder.mkdirs() + + boolean forceGenerate = false + if (PyCharmBuildOptions.usePrebuiltStubs) { + File stubsArchive = new File(context.paths.projectHome, PyCharmBuildOptions.prebuiltStubsArchive) + context.ant.echo("Unzip prebuilt stubs ${stubsArchive.absolutePath}") + context.ant.unzip(src: stubsArchive.absolutePath, dest: "${temporaryIndexFolder.absolutePath}") + + try { + String stubsVersionPath = new FileNameFinder() + .getFileNames("${temporaryIndexFolder.absolutePath}/Python", "sdk-stubs.version"). + first() + int stubsFromArchiveVersion = new File(stubsVersionPath).readLines()[0].toInteger() + context.ant.echo("Stubs version $stubsFromArchiveVersion") + int stubVersion = getStubVersion(context) + context.ant.echo("But should be $stubVersion") + + forceGenerate = stubsFromArchiveVersion != stubVersion + } + catch (ignored) { + forceGenerate = true + } + } + + if (forceGenerate || !PyCharmBuildOptions.usePrebuiltStubs) { + temporaryIndexFolder.deleteDir() + temporaryIndexFolder.mkdir() + generateUniversalStubs(context, folderWithUnzipContent, temporaryIndexFolder) + } + + context.ant.echo("Generate indices") + generateIndices(context, folderWithUnzipContent, temporaryIndexFolder) + context.ant.echo("Copy indices") context.ant.copy(todir: "$targetDirectory/index", failonerror: !context.options.isInDevelopmentMode) { - fileset(dir: "$context.paths.temp/index", erroronmissingdir: !context.options.isInDevelopmentMode) { + fileset(dir: temporaryIndexFolder.absolutePath, erroronmissingdir: !context.options.isInDevelopmentMode) { include(name: "**") } } + + folderWithUnzipContent.deleteDir() + temporaryIndexFolder.deleteDir() } @Override String getEnvironmentVariableBaseName(ApplicationInfoProperties applicationInfo) { "PYCHARM" } -} -class PyPrebuiltIndicesGenerator implements ResourcesGenerator { - @Override - File generateResources(BuildContext context) { + private void unzipArchives(BuildContext context, File destination) { + File tempFolder = new File(context.paths.temp, "zips") + tempFolder.mkdirs() + tempFolder.deleteOnExit() + context.ant.copy(todir: tempFolder.absolutePath, failonerror: !context.options.isInDevelopmentMode) { + fileset(dir: "$context.paths.projectHome/python-distributions", erroronmissingdir: !context.options.isInDevelopmentMode) { + include(name: "*.zip") + } + fileset(dir: "$context.paths.projectHome/skeletons", erroronmissingdir: !context.options.isInDevelopmentMode) { + include(name: "*.zip") + } + } + + tempFolder.eachFileRecurse(FileType.FILES) { + if (it.name.endsWith('.zip')) { + context.ant.unzip(src: it.absolutePath, dest: "${destination.absolutePath}/${it.name}") + } + } + } + + private int getStubVersion(BuildContext context) { + List buildClasspath = context.getModuleRuntimeClasspath(context.findModule("intellij.python.tools"), false) + + context.ant.java(classname: "com.jetbrains.python.tools.GetPyStubsVersionKt", fork: true, outputproperty: "stubsVersion") { + classpath { + buildClasspath.each { + pathelement(location: it) + } + } + } + return context.ant.project.properties.stubsVersion as int + } + + private void generateUniversalStubs(BuildContext context, File from, File to) { CompilationTasks.create(context).compileModules(["intellij.python.tools"]) List buildClasspath = context.getModuleRuntimeClasspath(context.findModule("intellij.python.tools"), false) - String zipPath = "$context.paths.temp/zips" - String outputPath = "$context.paths.temp/index" - context.messages.block("Generate universal stubs") { - context.ant.copy(todir: zipPath, failonerror: !context.options.isInDevelopmentMode) { - fileset(dir: "$context.paths.projectHome/python-distributions", erroronmissingdir: !context.options.isInDevelopmentMode) { - include(name: "*.zip") - } - fileset(dir: "$context.paths.projectHome/skeletons", erroronmissingdir: !context.options.isInDevelopmentMode) { - include(name: "*.zip") - } - } - - context.ant.java(classname: "com.jetbrains.python.tools.PyPrebuiltIndicesGeneratorKt", fork: true) { + context.ant.java(classname: "com.jetbrains.python.tools.PythonUniversalStubsBuilderKt", fork: true) { jvmarg(line: "-ea -Xmx1000m") - arg(value: zipPath) - arg(value: outputPath) + arg(value: from.absolutePath) + arg(value: to.absolutePath) classpath { buildClasspath.each { pathelement(location: it) @@ -91,7 +151,21 @@ class PyPrebuiltIndicesGenerator implements ResourcesGenerator { } } } + } - return new File(outputPath) + private void generateIndices(BuildContext context, File from, File to) { + CompilationTasks.create(context).compileModules(["intellij.python.tools"]) + List buildClasspath = context.getModuleRuntimeClasspath(context.findModule("intellij.python.tools"), false) + + context.ant.java(classname: "com.jetbrains.python.tools.IndicesBuilderKt", fork: true) { + jvmarg(line: "-ea -Xmx1000m") + arg(value: from.absolutePath) + arg(value: to.absolutePath) + classpath { + buildClasspath.each { + pathelement(location: it) + } + } + } } } \ No newline at end of file diff --git a/python/tools/intellij.python.tools.iml b/python/tools/intellij.python.tools.iml index fdd887bdf1d5..7323806ea4c6 100644 --- a/python/tools/intellij.python.tools.iml +++ b/python/tools/intellij.python.tools.iml @@ -18,5 +18,6 @@ + \ No newline at end of file diff --git a/python/tools/src/com/jetbrains/python/tools/BuildZippedSkeletons.kt b/python/tools/src/com/jetbrains/python/tools/BuildZippedSkeletons.kt index 1b657a6f10dc..6fcb8f3b4133 100644 --- a/python/tools/src/com/jetbrains/python/tools/BuildZippedSkeletons.kt +++ b/python/tools/src/com/jetbrains/python/tools/BuildZippedSkeletons.kt @@ -62,9 +62,9 @@ fun main(args: Array) { val dirPacked = File(skeletonsDir.parent, DefaultPregeneratedSkeletonsProvider.getPregeneratedSkeletonsName(sdk, refresher.generatorVersion, true, true)) - val zip = ZipOutputStream(FileOutputStream(dirPacked)) - ZipUtil.addDirToZipRecursively(zip, dirPacked, skeletonsDir, "", null, null) - zip.close() + ZipOutputStream(FileOutputStream(dirPacked)).use { + ZipUtil.addDirToZipRecursively(it, dirPacked, skeletonsDir, "", null, null) + } } } catch (e: Exception) { diff --git a/python/tools/src/com/jetbrains/python/tools/GetPyStubsVersion.kt b/python/tools/src/com/jetbrains/python/tools/GetPyStubsVersion.kt new file mode 100644 index 000000000000..487fe81944e2 --- /dev/null +++ b/python/tools/src/com/jetbrains/python/tools/GetPyStubsVersion.kt @@ -0,0 +1,12 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.jetbrains.python.tools + +import com.jetbrains.python.psi.PyFileElementType + + +/** + * This script is needed for PyCharm installation process to check indexer version of pre-built stubs + */ +fun main(args: Array) { + println(PyFileElementType.INSTANCE.stubVersion) +} diff --git a/python/tools/src/com/jetbrains/python/tools/IndicesBuilder.kt b/python/tools/src/com/jetbrains/python/tools/IndicesBuilder.kt new file mode 100644 index 000000000000..6607803bf6c0 --- /dev/null +++ b/python/tools/src/com/jetbrains/python/tools/IndicesBuilder.kt @@ -0,0 +1,35 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.jetbrains.python.tools + +import com.intellij.index.PrebuiltIndexAwareIdIndexer +import com.jetbrains.python.psi.impl.stubs.PyPrebuiltStubsProvider +import org.jetbrains.index.id.IdIndexGenerator + + +/** + * @author Aleksey.Rostovskiy + */ +fun main(args: Array) { + require(args.size == 2) { + "Usage: IndicesBuilderKt " + } + IndicesBuilder.build(args[0], "${args[1]}/${PyPrebuiltStubsProvider.NAME}") +} + +object IndicesBuilder: PyGeneratorBase() { + fun build(root: String, outputPath: String) { + try { + app + + val files = rootFiles(root) + IdIndexGenerator("$outputPath/${PrebuiltIndexAwareIdIndexer.ID_INDEX_FILE_NAME}") + .buildIdIndexForRoots(files) + } + catch (e: Throwable) { + e.printStackTrace() + } + finally { + tearDown() + } + } +} diff --git a/python/tools/src/com/jetbrains/python/tools/PyGeneratorBase.kt b/python/tools/src/com/jetbrains/python/tools/PyGeneratorBase.kt new file mode 100644 index 000000000000..b668d7c44133 --- /dev/null +++ b/python/tools/src/com/jetbrains/python/tools/PyGeneratorBase.kt @@ -0,0 +1,66 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.jetbrains.python.tools + +import com.intellij.ide.BootstrapClassLoaderUtil +import com.intellij.idea.IdeaTestApplication +import com.intellij.openapi.application.PathManager +import com.intellij.openapi.application.WriteAction +import com.intellij.openapi.util.io.FileUtil +import com.intellij.openapi.vfs.LocalFileSystem +import com.intellij.openapi.vfs.VirtualFile +import com.intellij.util.PlatformUtils +import com.intellij.util.io.ZipUtil +import com.intellij.util.ui.UIUtil +import java.io.File + + +/** + * @author Aleksey.Rostovskiy + */ +open class PyGeneratorBase { + protected val app: IdeaTestApplication by lazy { + System.setProperty(PlatformUtils.PLATFORM_PREFIX_KEY, PlatformUtils.PYCHARM_CE_PREFIX) + System.setProperty(PathManager.PROPERTY_PLUGINS_PATH, FileUtil.createTempDirectory("pystubs", "plugins").absolutePath) + System.setProperty(PathManager.PROPERTY_SYSTEM_PATH, FileUtil.createTempDirectory("pystubs", "system").absolutePath) + System.setProperty(PathManager.PROPERTY_CONFIG_PATH, FileUtil.createTempDirectory("pystubs", "config").absolutePath) + Thread.currentThread().contextClassLoader = BootstrapClassLoaderUtil.initClassLoader() + IdeaTestApplication.getInstance() + } + + protected fun rootFiles(root: String): ArrayList{ + val dir = File(root) + if (!dir.exists() || !dir.isDirectory) { + throw IllegalStateException("$root doesn't exist or isn't a directory") + } + + return ArrayList(dir.listFiles().map { it-> + LocalFileSystem.getInstance().findFileByIoFile(it)!! + }) + } + + protected fun unzipArchivesToRoots(root: String): ArrayList { + val dir = File(root) + if (!dir.exists() || !dir.isDirectory) { + throw IllegalStateException("$root doesn't exist or isn't a directory") + } + val files = dir.listFiles { _, name -> name.endsWith(".zip") }.map { zip -> + val unzipRoot = File(root, zip.nameWithoutExtension) + println("Extracting $zip") + ZipUtil.extract(zip, unzipRoot, null) + unzipRoot + } + return ArrayList(files.map { it -> + LocalFileSystem.getInstance().findFileByIoFile(it)!! + }) + } + + protected fun tearDown() { + UIUtil.invokeAndWaitIfNeeded(Runnable { + WriteAction.run { + app.dispose() + } + }) + FileUtil.delete(File(System.getProperty(PathManager.PROPERTY_PLUGINS_PATH))) + FileUtil.delete(File(System.getProperty(PathManager.PROPERTY_SYSTEM_PATH))) + } +} \ No newline at end of file diff --git a/python/tools/src/com/jetbrains/python/tools/PyPrebuiltIndicesGenerator.kt b/python/tools/src/com/jetbrains/python/tools/PyPrebuiltIndicesGenerator.kt deleted file mode 100644 index f51b74ae05ca..000000000000 --- a/python/tools/src/com/jetbrains/python/tools/PyPrebuiltIndicesGenerator.kt +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. -package com.jetbrains.python.tools - -import com.intellij.ide.BootstrapClassLoaderUtil -import com.intellij.idea.IdeaTestApplication -import com.intellij.index.PrebuiltIndexAwareIdIndexer -import com.intellij.openapi.application.PathManager -import com.intellij.openapi.application.WriteAction -import com.intellij.openapi.util.io.FileUtil -import com.intellij.openapi.vfs.LocalFileSystem -import com.intellij.psi.stubs.PrebuiltStubsProviderBase -import com.intellij.testFramework.PlatformTestCase -import com.intellij.util.ReflectionUtil -import com.intellij.util.io.ZipUtil -import com.intellij.util.ui.UIUtil -import com.jetbrains.python.psi.impl.stubs.PyPrebuiltStubsProvider -import org.jetbrains.index.id.IdIndexGenerator -import java.io.File -import java.util.* - -fun main(args: Array) { - buildIndices(args[0], "${args[1]}/${PyPrebuiltStubsProvider.NAME}") -} - -fun buildIndices(root: String, outputPath: String) { - val app = createApp() - try { - FileUtil.delete(File(outputPath)) - - val roots = unzipArchivesToRoots(root) - - val rootFiles = ArrayList(roots.map { it -> LocalFileSystem.getInstance().findFileByIoFile(it)!! }) - - PyStubsGenerator("$outputPath/${PrebuiltStubsProviderBase.SDK_STUBS_STORAGE_NAME}").buildStubsForRoots(rootFiles) - - IdIndexGenerator("$outputPath/${PrebuiltIndexAwareIdIndexer.ID_INDEX_FILE_NAME}").buildIdIndexForRoots(rootFiles) - } - catch (e: Throwable) { - e.printStackTrace() - } - finally { - UIUtil.invokeAndWaitIfNeeded(Runnable { - WriteAction.run { - app.dispose() - } - }) - FileUtil.delete(File(System.getProperty(PathManager.PROPERTY_PLUGINS_PATH))) - FileUtil.delete(File(System.getProperty(PathManager.PROPERTY_SYSTEM_PATH))) - System.exit(0) - } -} - -fun unzipArchivesToRoots(root: String): List { - val dir = File(root) - if (!dir.exists() || !dir.isDirectory) { - throw IllegalStateException("$root doesn't exist or isn't a directory") - } - return dir.listFiles { _, name -> name.endsWith(".zip") }.map { zip -> - val unzipRoot = File(root, zip.nameWithoutExtension) - println("Extracting $zip") - ZipUtil.extract(zip, unzipRoot, null) - unzipRoot - } -} - -private fun createApp(): IdeaTestApplication { - val candidates = ReflectionUtil.getField(PlatformTestCase::class.java, null, Array::class.java, "PREFIX_CANDIDATES") - candidates[0] = "PyCharm" - System.setProperty(PathManager.PROPERTY_PLUGINS_PATH, FileUtil.createTempDirectory("pystubs", "plugins").absolutePath) - System.setProperty(PathManager.PROPERTY_SYSTEM_PATH, FileUtil.createTempDirectory("pystubs", "system").absolutePath) - System.setProperty(PathManager.PROPERTY_CONFIG_PATH, FileUtil.createTempDirectory("pystubs", "config").absolutePath) - Thread.currentThread().contextClassLoader = BootstrapClassLoaderUtil.initClassLoader() - return IdeaTestApplication.getInstance() -} diff --git a/python/tools/src/com/jetbrains/python/tools/PythonUniversalStubsBuilder.kt b/python/tools/src/com/jetbrains/python/tools/PythonUniversalStubsBuilder.kt new file mode 100644 index 000000000000..26e68fc7b3fc --- /dev/null +++ b/python/tools/src/com/jetbrains/python/tools/PythonUniversalStubsBuilder.kt @@ -0,0 +1,78 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.jetbrains.python.tools + +import com.intellij.openapi.util.io.FileUtil +import com.intellij.psi.stubs.PrebuiltStubsProviderBase +import com.intellij.util.io.ZipUtil +import com.jetbrains.python.psi.impl.stubs.PyPrebuiltStubsProvider +import org.jetbrains.intellij.build.pycharm.PyCharmBuildOptions +import java.io.File +import java.io.FileOutputStream +import java.util.zip.ZipOutputStream + + +/** + * @author Aleksey.Rostovskiy + */ +fun main(args: Array) { + if (args.size != 2) { + val zipsDirectory = System.getProperty("pycharm.zips.directory") + val prebuiltStubsArchive = PyCharmBuildOptions.getPrebuiltStubsArchive() + + if (zipsDirectory.isNullOrBlank() || prebuiltStubsArchive.isNullOrBlank()) { + throw IllegalArgumentException( + "Usage: PythonUniversalStubsBuilderKt " + ) + } + PythonUniversalStubsBuilder.generateArchive(zipsDirectory, prebuiltStubsArchive) + } + else { + PythonUniversalStubsBuilder.generateStubs(args[0], "${args[1]}/${PyPrebuiltStubsProvider.NAME}") + } +} + +object PythonUniversalStubsBuilder : PyGeneratorBase() { + fun generateStubs(root: String, outputPath: String) { + try { + app + + val files = rootFiles(root) + PyStubsGenerator("$outputPath/${PrebuiltStubsProviderBase.SDK_STUBS_STORAGE_NAME}") + .buildStubsForRoots(files) + } + catch (e: Throwable) { + e.printStackTrace() + } + finally { + tearDown() + } + } + + fun generateArchive(zipsDirectory: String, prebuiltStubsArchive: String) { + val tmpFolder = FileUtil.createTempDirectory("stubs", null) + tmpFolder.delete() + tmpFolder.mkdirs() + tmpFolder.deleteOnExit() + + try { + app + + val unzippedFiles = unzipArchivesToRoots(zipsDirectory) + PyStubsGenerator("${tmpFolder.absolutePath}/${PrebuiltStubsProviderBase.SDK_STUBS_STORAGE_NAME}") + .buildStubsForRoots(unzippedFiles) + } + catch (e: Throwable) { + e.printStackTrace() + } + finally { + tearDown() + } + + println("Generate archive $prebuiltStubsArchive") + val archive = File(prebuiltStubsArchive) + ZipOutputStream(FileOutputStream(archive)).use { + ZipUtil.addFileOrDirRecursively(it, archive, tmpFolder, PyPrebuiltStubsProvider.NAME, null, null) + } + } + +} \ No newline at end of file