From db05f6460896feead87839283b5ff1cb235b2216 Mon Sep 17 00:00:00 2001 From: Vladimir Krivosheev Date: Fri, 6 Dec 2019 08:14:24 +0100 Subject: [PATCH] simplify python stub generator GitOrigin-RevId: ee2ebf724a3f273d760af91dda97e9f89fb5c398 --- .../intellij/simpleApplicationBootstrapper.kt | 67 +++++++++++++++++++ .../testFramework/TestApplicationManager.kt | 65 ++++-------------- .../jetbrains/python/tools/IndicesBuilder.kt | 23 +++---- .../jetbrains/python/tools/PyGeneratorBase.kt | 14 ++-- .../tools/PythonUniversalStubsBuilder.kt | 32 +++------ 5 files changed, 104 insertions(+), 97 deletions(-) create mode 100644 platform/testFramework/src/com/intellij/simpleApplicationBootstrapper.kt diff --git a/platform/testFramework/src/com/intellij/simpleApplicationBootstrapper.kt b/platform/testFramework/src/com/intellij/simpleApplicationBootstrapper.kt new file mode 100644 index 000000000000..29ff9e7f7143 --- /dev/null +++ b/platform/testFramework/src/com/intellij/simpleApplicationBootstrapper.kt @@ -0,0 +1,67 @@ +// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij + +import com.intellij.concurrency.IdeaForkJoinWorkerThreadFactory +import com.intellij.diagnostic.ThreadDumper +import com.intellij.ide.IdeEventQueue +import com.intellij.ide.plugins.IdeaPluginDescriptorImpl +import com.intellij.ide.plugins.PluginManagerCore +import com.intellij.idea.* +import com.intellij.openapi.application.PathManager +import com.intellij.openapi.application.impl.ApplicationImpl +import com.intellij.ui.IconManager +import com.intellij.util.concurrency.AppExecutorUtil +import java.awt.EventQueue +import java.util.concurrent.CompletableFuture +import java.util.concurrent.ExecutionException +import java.util.concurrent.TimeUnit +import java.util.concurrent.TimeoutException +import java.util.function.Supplier + +fun loadHeadlessAppInUnitTestMode() { + doLoadApp { + EventQueue.invokeAndWait { + // replaces system event queue + IdeEventQueue.getInstance() + } + } +} + +internal fun doLoadApp(setupEventQueue: () -> Unit) { + Main.setFlags(arrayOf("inspect", "", "", "")) + assert(Main.isHeadless()) + assert(Main.isCommandLine()) + PluginManagerCore.isUnitTestMode = true + IdeaForkJoinWorkerThreadFactory.setupForkJoinCommonPool(true) + + val loadedPluginFuture = CompletableFuture.supplyAsync(Supplier { + PluginManagerCore.getLoadedPlugins(PathManager::class.java.classLoader) + }, AppExecutorUtil.getAppExecutorService()) + + setupEventQueue() + + val app = ApplicationImpl(true, true, true, true) + IconManager.activate() + val plugins: List + try { + plugins = registerRegistryAndInitStore(registerAppComponents(loadedPluginFuture, app), app) + .get(20, TimeUnit.SECONDS) + + val boundedExecutor = createExecutorToPreloadServices() + val preloadServiceFuture = preloadServices(plugins, app, boundedExecutor, "") + app.loadComponents(null) + + preloadServiceFuture + .thenCompose { callAppInitialized(app, boundedExecutor) } + .get(20, TimeUnit.SECONDS) + } + catch (e: TimeoutException) { + throw RuntimeException("Cannot preload services in 20 seconds: ${ThreadDumper.dumpThreadsToString()}", e) + } + catch (e: ExecutionException) { + throw e.cause ?: e + } + catch (e: InterruptedException) { + throw e.cause ?: e + } +} diff --git a/platform/testFramework/src/com/intellij/testFramework/TestApplicationManager.kt b/platform/testFramework/src/com/intellij/testFramework/TestApplicationManager.kt index 0d424a14e081..2c535761876a 100644 --- a/platform/testFramework/src/com/intellij/testFramework/TestApplicationManager.kt +++ b/platform/testFramework/src/com/intellij/testFramework/TestApplicationManager.kt @@ -8,20 +8,17 @@ import com.intellij.codeInsight.completion.CompletionProgressIndicator import com.intellij.codeInsight.hint.HintManager import com.intellij.codeInsight.hint.HintManagerImpl import com.intellij.codeInsight.lookup.LookupManager -import com.intellij.concurrency.IdeaForkJoinWorkerThreadFactory -import com.intellij.diagnostic.ThreadDumper +import com.intellij.doLoadApp import com.intellij.execution.process.ProcessIOExecutorService import com.intellij.ide.DataManager import com.intellij.ide.GeneratedSourceFileChangeTracker import com.intellij.ide.GeneratedSourceFileChangeTrackerImpl import com.intellij.ide.IdeEventQueue import com.intellij.ide.impl.HeadlessDataManager -import com.intellij.ide.plugins.IdeaPluginDescriptorImpl -import com.intellij.ide.plugins.PluginManagerCore import com.intellij.ide.startup.impl.StartupManagerImpl import com.intellij.ide.structureView.StructureViewFactory import com.intellij.ide.structureView.impl.StructureViewFactoryImpl -import com.intellij.idea.* +import com.intellij.idea.StartupUtil import com.intellij.openapi.Disposable import com.intellij.openapi.actionSystem.DataProvider import com.intellij.openapi.actionSystem.ex.ActionUtil @@ -56,7 +53,6 @@ import com.intellij.openapi.util.EmptyRunnable import com.intellij.psi.PsiManager import com.intellij.psi.impl.PsiManagerImpl import com.intellij.psi.templateLanguages.TemplateDataLanguageMappings -import com.intellij.ui.IconManager import com.intellij.ui.UiInterceptors import com.intellij.util.ReflectionUtil import com.intellij.util.concurrency.AppExecutorUtil @@ -76,9 +72,9 @@ import sun.awt.AWTAutoShutdown import java.awt.EventQueue import java.awt.Toolkit import java.util.* -import java.util.concurrent.* +import java.util.concurrent.DelayQueue +import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicBoolean -import java.util.function.Supplier import javax.swing.SwingUtilities import javax.swing.Timer @@ -128,7 +124,14 @@ class TestApplicationManager private constructor() { } HeavyPlatformTestCase.doAutodetectPlatformPrefix() - loadTestApp() + doLoadApp { + if (EventQueue.isDispatchThread()) { + StartupUtil.replaceSystemEventQueue(logger()) + } + else { + replaceIdeEventQueueSafely() + } + } isBootstrappingAppNow.set(false) result = TestApplicationManager() ourInstance = result @@ -156,50 +159,6 @@ class TestApplicationManager private constructor() { } } -private fun loadTestApp() { - Main.setFlags(arrayOf("inspect", "", "", "")) - assert(Main.isHeadless()) - assert(Main.isCommandLine()) - PluginManagerCore.isUnitTestMode = true - IdeaForkJoinWorkerThreadFactory.setupForkJoinCommonPool(true) - - val loadedPluginFuture = CompletableFuture.supplyAsync(Supplier { - PluginManagerCore.getLoadedPlugins(TestApplicationManager::class.java.classLoader) - }, AppExecutorUtil.getAppExecutorService()) - - if (EventQueue.isDispatchThread()) { - StartupUtil.replaceSystemEventQueue(logger()) - } - else { - replaceIdeEventQueueSafely() - } - - val app = ApplicationImpl(true, true, true, true) - IconManager.activate() - val plugins: List - try { - plugins = registerRegistryAndInitStore(registerAppComponents(loadedPluginFuture, app), app) - .get(20, TimeUnit.SECONDS) - - val boundedExecutor = createExecutorToPreloadServices() - val preloadServiceFuture = preloadServices(plugins, app, boundedExecutor, "") - app.loadComponents(null) - - preloadServiceFuture - .thenCompose { callAppInitialized(app, boundedExecutor) } - .get(20, TimeUnit.SECONDS) - } - catch (e: TimeoutException) { - throw RuntimeException("Cannot preload services in 20 seconds: ${ThreadDumper.dumpThreadsToString()}", e) - } - catch (e: ExecutionException) { - throw e.cause ?: e - } - catch (e: InterruptedException) { - throw e.cause ?: e - } -} - fun replaceIdeEventQueueSafely() { if (Toolkit.getDefaultToolkit().systemEventQueue is IdeEventQueue) { return diff --git a/python/tools/src/com/jetbrains/python/tools/IndicesBuilder.kt b/python/tools/src/com/jetbrains/python/tools/IndicesBuilder.kt index 8f6effdcaa8d..b1e8f64bf77d 100644 --- a/python/tools/src/com/jetbrains/python/tools/IndicesBuilder.kt +++ b/python/tools/src/com/jetbrains/python/tools/IndicesBuilder.kt @@ -11,11 +11,12 @@ import kotlin.system.exitProcess */ fun main(args: Array) { try { - require(args.size == 2) { - "Usage: IndicesBuilderKt " + if (args.size != 2) { + println("Usage: IndicesBuilderKt ") + exitProcess(1) } + IndicesBuilder.build(args[0], "${args[1]}/${PyPrebuiltStubsProvider.NAME}") - exitProcess(0) } catch (e: Throwable) { e.printStackTrace() @@ -23,20 +24,12 @@ fun main(args: Array) { } } -private object IndicesBuilder: PyGeneratorBase() { +private object IndicesBuilder : PyGeneratorBase() { fun build(root: String, outputPath: String) { - try { - app - + use { val files = rootFiles(root) - IdIndexGenerator("$outputPath/${PrebuiltIndexAwareIdIndexer.ID_INDEX_FILE_NAME}") - .buildIdIndexForRoots(files) - } - catch (e: Throwable) { - e.printStackTrace() - } - finally { - tearDown() + IdIndexGenerator("$outputPath/${PrebuiltIndexAwareIdIndexer.ID_INDEX_FILE_NAME}").buildIdIndexForRoots(files) + exitProcess(0) } } } diff --git a/python/tools/src/com/jetbrains/python/tools/PyGeneratorBase.kt b/python/tools/src/com/jetbrains/python/tools/PyGeneratorBase.kt index f8a1993f4d28..b1fd8dfaac3f 100644 --- a/python/tools/src/com/jetbrains/python/tools/PyGeneratorBase.kt +++ b/python/tools/src/com/jetbrains/python/tools/PyGeneratorBase.kt @@ -1,31 +1,33 @@ // Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. package com.jetbrains.python.tools +import com.intellij.loadHeadlessAppInUnitTestMode import com.intellij.openapi.application.PathManager import com.intellij.openapi.util.io.FileUtil import com.intellij.openapi.vfs.LocalFileSystem import com.intellij.openapi.vfs.VirtualFile -import com.intellij.testFramework.TestApplicationManager import com.intellij.util.PlatformUtils import com.intellij.util.io.Decompressor import java.nio.file.Files +import java.nio.file.Path import java.nio.file.Paths /** * @author Aleksey.Rostovskiy */ -open class PyGeneratorBase { +open class PyGeneratorBase : AutoCloseable { @Suppress("SpellCheckingInspection") - private val tempDir = Files.createTempDirectory("pystubs").toAbsolutePath() + protected val tempDir: Path = Files.createTempDirectory("pystubs").toAbsolutePath() - protected val app by lazy { + init { System.setProperty(PlatformUtils.PLATFORM_PREFIX_KEY, PlatformUtils.PYCHARM_CE_PREFIX) val dir = FileUtil.toSystemIndependentName(tempDir.toString()) System.setProperty(PathManager.PROPERTY_PLUGINS_PATH, "$dir/plugins") System.setProperty(PathManager.PROPERTY_SYSTEM_PATH, "$dir/system") System.setProperty(PathManager.PROPERTY_CONFIG_PATH, "$dir/config") - TestApplicationManager.getInstance() + + loadHeadlessAppInUnitTestMode() } protected fun rootFiles(root: String): List { @@ -54,7 +56,7 @@ open class PyGeneratorBase { } } - protected fun tearDown() { + final override fun close() { FileUtil.delete(tempDir) } } \ No newline at end of file diff --git a/python/tools/src/com/jetbrains/python/tools/PythonUniversalStubsBuilder.kt b/python/tools/src/com/jetbrains/python/tools/PythonUniversalStubsBuilder.kt index b88ca41aece3..14d2c7ecfc93 100644 --- a/python/tools/src/com/jetbrains/python/tools/PythonUniversalStubsBuilder.kt +++ b/python/tools/src/com/jetbrains/python/tools/PythonUniversalStubsBuilder.kt @@ -14,7 +14,10 @@ import kotlin.system.exitProcess */ fun main(args: Array) { try { - if (args.size != 2) { + if (args.size == 2) { + PythonUniversalStubsBuilder.generateStubs(args[0], "${args[1]}/${PyPrebuiltStubsProvider.NAME}") + } + else { val zipsDirectory = System.getProperty("intellij.build.pycharm.zips.directory") val prebuiltStubsArchive = PyCharmBuildOptions.getPrebuiltStubsArchive() if (zipsDirectory.isNullOrBlank() || prebuiltStubsArchive.isNullOrBlank()) { @@ -23,9 +26,6 @@ fun main(args: Array) { } PythonUniversalStubsBuilder.generateArchive(zipsDirectory, prebuiltStubsArchive) } - else { - PythonUniversalStubsBuilder.generateStubs(args[0], "${args[1]}/${PyPrebuiltStubsProvider.NAME}") - } exitProcess(0) } catch (e: Throwable) { @@ -36,37 +36,23 @@ fun main(args: Array) { private object PythonUniversalStubsBuilder : PyGeneratorBase() { fun generateStubs(root: String, outputPath: String) { - try { - app - + use { val files = rootFiles(root) PyStubsGenerator("$outputPath/${PrebuiltStubsProviderBase.SDK_STUBS_STORAGE_NAME}") .buildStubsForRoots(files) } - finally { - tearDown() - } } fun generateArchive(zipsDirectory: String, prebuiltStubsArchive: String) { - val tmpFolder = FileUtil.createTempDirectory("stubs", null) - tmpFolder.delete() - tmpFolder.mkdirs() - - try { - app - + val stubDir = tempDir.resolve("stubs") + use { val unzippedFiles = unzipArchivesToRoots(zipsDirectory) - PyStubsGenerator("${tmpFolder.absolutePath}/${PrebuiltStubsProviderBase.SDK_STUBS_STORAGE_NAME}") + PyStubsGenerator(FileUtil.toSystemIndependentName(stubDir.resolve(PrebuiltStubsProviderBase.SDK_STUBS_STORAGE_NAME).toString())) .buildStubsForRoots(unzippedFiles) println("Generate archive $prebuiltStubsArchive") val archive = File(prebuiltStubsArchive) - Compressor.Zip(archive).use { it.addDirectory(PyPrebuiltStubsProvider.NAME, tmpFolder) } - } - finally { - FileUtil.delete(tmpFolder) - tearDown() + Compressor.Zip(archive).use { it.addDirectory(PyPrebuiltStubsProvider.NAME, stubDir.toFile()) } } } } \ No newline at end of file