simplify python stub generator

GitOrigin-RevId: ee2ebf724a3f273d760af91dda97e9f89fb5c398

platform/testFramework/src/com/intellij/simpleApplicationBootstrapper.kt

@@ -0,0 +1,67 @@
+// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij
+
+import com.intellij.concurrency.IdeaForkJoinWorkerThreadFactory
+import com.intellij.diagnostic.ThreadDumper
+import com.intellij.ide.IdeEventQueue
+import com.intellij.ide.plugins.IdeaPluginDescriptorImpl
+import com.intellij.ide.plugins.PluginManagerCore
+import com.intellij.idea.*
+import com.intellij.openapi.application.PathManager
+import com.intellij.openapi.application.impl.ApplicationImpl
+import com.intellij.ui.IconManager
+import com.intellij.util.concurrency.AppExecutorUtil
+import java.awt.EventQueue
+import java.util.concurrent.CompletableFuture
+import java.util.concurrent.ExecutionException
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.TimeoutException
+import java.util.function.Supplier
+
+fun loadHeadlessAppInUnitTestMode() {
+  doLoadApp {
+    EventQueue.invokeAndWait {
+      // replaces system event queue
+      IdeEventQueue.getInstance()
+    }
+  }
+}
+
+internal fun doLoadApp(setupEventQueue: () -> Unit) {
+  Main.setFlags(arrayOf("inspect", "", "", ""))
+  assert(Main.isHeadless())
+  assert(Main.isCommandLine())
+  PluginManagerCore.isUnitTestMode = true
+  IdeaForkJoinWorkerThreadFactory.setupForkJoinCommonPool(true)
+
+  val loadedPluginFuture = CompletableFuture.supplyAsync(Supplier {
+    PluginManagerCore.getLoadedPlugins(PathManager::class.java.classLoader)
+  }, AppExecutorUtil.getAppExecutorService())
+
+  setupEventQueue()
+
+  val app = ApplicationImpl(true, true, true, true)
+  IconManager.activate()
+  val plugins: List<IdeaPluginDescriptorImpl>
+  try {
+    plugins = registerRegistryAndInitStore(registerAppComponents(loadedPluginFuture, app), app)
+      .get(20, TimeUnit.SECONDS)
+
+    val boundedExecutor = createExecutorToPreloadServices()
+    val preloadServiceFuture = preloadServices(plugins, app, boundedExecutor, "")
+    app.loadComponents(null)
+
+    preloadServiceFuture
+      .thenCompose { callAppInitialized(app, boundedExecutor) }
+      .get(20, TimeUnit.SECONDS)
+  }
+  catch (e: TimeoutException) {
+    throw RuntimeException("Cannot preload services in 20 seconds: ${ThreadDumper.dumpThreadsToString()}", e)
+  }
+  catch (e: ExecutionException) {
+    throw e.cause ?: e
+  }
+  catch (e: InterruptedException) {
+    throw e.cause ?: e
+  }
+}

platform/testFramework/src/com/intellij/testFramework/TestApplicationManager.kt

@@ -8,20 +8,17 @@ import com.intellij.codeInsight.completion.CompletionProgressIndicator
 import com.intellij.codeInsight.hint.HintManager
 import com.intellij.codeInsight.hint.HintManagerImpl
 import com.intellij.codeInsight.lookup.LookupManager
-import com.intellij.concurrency.IdeaForkJoinWorkerThreadFactory
-import com.intellij.diagnostic.ThreadDumper
+import com.intellij.doLoadApp
 import com.intellij.execution.process.ProcessIOExecutorService
 import com.intellij.ide.DataManager
 import com.intellij.ide.GeneratedSourceFileChangeTracker
 import com.intellij.ide.GeneratedSourceFileChangeTrackerImpl
 import com.intellij.ide.IdeEventQueue
 import com.intellij.ide.impl.HeadlessDataManager
-import com.intellij.ide.plugins.IdeaPluginDescriptorImpl
-import com.intellij.ide.plugins.PluginManagerCore
 import com.intellij.ide.startup.impl.StartupManagerImpl
 import com.intellij.ide.structureView.StructureViewFactory
 import com.intellij.ide.structureView.impl.StructureViewFactoryImpl
-import com.intellij.idea.*
+import com.intellij.idea.StartupUtil
 import com.intellij.openapi.Disposable
 import com.intellij.openapi.actionSystem.DataProvider
 import com.intellij.openapi.actionSystem.ex.ActionUtil
@@ -56,7 +53,6 @@ import com.intellij.openapi.util.EmptyRunnable
 import com.intellij.psi.PsiManager
 import com.intellij.psi.impl.PsiManagerImpl
 import com.intellij.psi.templateLanguages.TemplateDataLanguageMappings
-import com.intellij.ui.IconManager
 import com.intellij.ui.UiInterceptors
 import com.intellij.util.ReflectionUtil
 import com.intellij.util.concurrency.AppExecutorUtil
@@ -76,9 +72,9 @@ import sun.awt.AWTAutoShutdown
 import java.awt.EventQueue
 import java.awt.Toolkit
 import java.util.*
-import java.util.concurrent.*
+import java.util.concurrent.DelayQueue
+import java.util.concurrent.TimeUnit
 import java.util.concurrent.atomic.AtomicBoolean
-import java.util.function.Supplier
 import javax.swing.SwingUtilities
 import javax.swing.Timer
 
@@ -128,7 +124,14 @@ class TestApplicationManager private constructor() {
       }
 
       HeavyPlatformTestCase.doAutodetectPlatformPrefix()
-      loadTestApp()
+      doLoadApp {
+        if (EventQueue.isDispatchThread()) {
+          StartupUtil.replaceSystemEventQueue(logger<TestApplicationManager>())
+        }
+        else {
+          replaceIdeEventQueueSafely()
+        }
+      }
       isBootstrappingAppNow.set(false)
       result = TestApplicationManager()
       ourInstance = result
@@ -156,50 +159,6 @@ class TestApplicationManager private constructor() {
   }
 }
 
-private fun loadTestApp() {
-  Main.setFlags(arrayOf("inspect", "", "", ""))
-  assert(Main.isHeadless())
-  assert(Main.isCommandLine())
-  PluginManagerCore.isUnitTestMode = true
-  IdeaForkJoinWorkerThreadFactory.setupForkJoinCommonPool(true)
-
-  val loadedPluginFuture = CompletableFuture.supplyAsync(Supplier {
-    PluginManagerCore.getLoadedPlugins(TestApplicationManager::class.java.classLoader)
-  }, AppExecutorUtil.getAppExecutorService())
-
-  if (EventQueue.isDispatchThread()) {
-    StartupUtil.replaceSystemEventQueue(logger<TestApplicationManager>())
-  }
-  else {
-    replaceIdeEventQueueSafely()
-  }
-
-  val app = ApplicationImpl(true, true, true, true)
-  IconManager.activate()
-  val plugins: List<IdeaPluginDescriptorImpl>
-  try {
-    plugins = registerRegistryAndInitStore(registerAppComponents(loadedPluginFuture, app), app)
-      .get(20, TimeUnit.SECONDS)
-
-    val boundedExecutor = createExecutorToPreloadServices()
-    val preloadServiceFuture = preloadServices(plugins, app, boundedExecutor, "")
-    app.loadComponents(null)
-
-    preloadServiceFuture
-      .thenCompose { callAppInitialized(app, boundedExecutor) }
-      .get(20, TimeUnit.SECONDS)
-  }
-  catch (e: TimeoutException) {
-    throw RuntimeException("Cannot preload services in 20 seconds: ${ThreadDumper.dumpThreadsToString()}", e)
-  }
-  catch (e: ExecutionException) {
-    throw e.cause ?: e
-  }
-  catch (e: InterruptedException) {
-    throw e.cause ?: e
-  }
-}
-
 fun replaceIdeEventQueueSafely() {
   if (Toolkit.getDefaultToolkit().systemEventQueue is IdeEventQueue) {
     return

python/tools/src/com/jetbrains/python/tools/IndicesBuilder.kt

@@ -11,11 +11,12 @@ import kotlin.system.exitProcess
  */
 fun main(args: Array<String>) {
   try {
-    require(args.size == 2) {
-      "Usage: IndicesBuilderKt <input folder with files> <output folder to store indices>"
+    if (args.size != 2) {
+      println("Usage: IndicesBuilderKt <input folder with files> <output folder to store indices>")
+      exitProcess(1)
     }
+
     IndicesBuilder.build(args[0], "${args[1]}/${PyPrebuiltStubsProvider.NAME}")
-    exitProcess(0)
   }
   catch (e: Throwable) {
     e.printStackTrace()
@@ -23,20 +24,12 @@ fun main(args: Array<String>) {
   }
 }
 
-private object IndicesBuilder: PyGeneratorBase() {
+private object IndicesBuilder : PyGeneratorBase() {
   fun build(root: String, outputPath: String) {
-    try {
-      app
-
+    use {
       val files = rootFiles(root)
-      IdIndexGenerator("$outputPath/${PrebuiltIndexAwareIdIndexer.ID_INDEX_FILE_NAME}")
-        .buildIdIndexForRoots(files)
-    }
-    catch (e: Throwable) {
-      e.printStackTrace()
-    }
-    finally {
-      tearDown()
+      IdIndexGenerator("$outputPath/${PrebuiltIndexAwareIdIndexer.ID_INDEX_FILE_NAME}").buildIdIndexForRoots(files)
+      exitProcess(0)
     }
   }
 }

python/tools/src/com/jetbrains/python/tools/PyGeneratorBase.kt

@@ -1,31 +1,33 @@
 // Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
 package com.jetbrains.python.tools
 
+import com.intellij.loadHeadlessAppInUnitTestMode
 import com.intellij.openapi.application.PathManager
 import com.intellij.openapi.util.io.FileUtil
 import com.intellij.openapi.vfs.LocalFileSystem
 import com.intellij.openapi.vfs.VirtualFile
-import com.intellij.testFramework.TestApplicationManager
 import com.intellij.util.PlatformUtils
 import com.intellij.util.io.Decompressor
 import java.nio.file.Files
+import java.nio.file.Path
 import java.nio.file.Paths
 
 /**
  * @author Aleksey.Rostovskiy
  */
-open class PyGeneratorBase {
+open class PyGeneratorBase : AutoCloseable {
   @Suppress("SpellCheckingInspection")
-  private val tempDir = Files.createTempDirectory("pystubs").toAbsolutePath()
+  protected val tempDir: Path = Files.createTempDirectory("pystubs").toAbsolutePath()
 
-  protected val app by lazy {
+  init {
     System.setProperty(PlatformUtils.PLATFORM_PREFIX_KEY, PlatformUtils.PYCHARM_CE_PREFIX)
 
     val dir = FileUtil.toSystemIndependentName(tempDir.toString())
     System.setProperty(PathManager.PROPERTY_PLUGINS_PATH, "$dir/plugins")
     System.setProperty(PathManager.PROPERTY_SYSTEM_PATH, "$dir/system")
     System.setProperty(PathManager.PROPERTY_CONFIG_PATH, "$dir/config")
-    TestApplicationManager.getInstance()
+
+    loadHeadlessAppInUnitTestMode()
   }
 
   protected fun rootFiles(root: String): List<VirtualFile> {
@@ -54,7 +56,7 @@ open class PyGeneratorBase {
     }
   }
 
-  protected fun tearDown() {
+  final override fun close() {
     FileUtil.delete(tempDir)
   }
 }

python/tools/src/com/jetbrains/python/tools/PythonUniversalStubsBuilder.kt

@@ -14,7 +14,10 @@ import kotlin.system.exitProcess
  */
 fun main(args: Array<String>) {
   try {
-    if (args.size != 2) {
+    if (args.size == 2) {
+      PythonUniversalStubsBuilder.generateStubs(args[0], "${args[1]}/${PyPrebuiltStubsProvider.NAME}")
+    }
+    else {
       val zipsDirectory = System.getProperty("intellij.build.pycharm.zips.directory")
       val prebuiltStubsArchive = PyCharmBuildOptions.getPrebuiltStubsArchive()
       if (zipsDirectory.isNullOrBlank() || prebuiltStubsArchive.isNullOrBlank()) {
@@ -23,9 +26,6 @@ fun main(args: Array<String>) {
       }
       PythonUniversalStubsBuilder.generateArchive(zipsDirectory, prebuiltStubsArchive)
     }
-    else {
-      PythonUniversalStubsBuilder.generateStubs(args[0], "${args[1]}/${PyPrebuiltStubsProvider.NAME}")
-    }
     exitProcess(0)
   }
   catch (e: Throwable) {
@@ -36,37 +36,23 @@ fun main(args: Array<String>) {
 
 private object PythonUniversalStubsBuilder : PyGeneratorBase() {
   fun generateStubs(root: String, outputPath: String) {
-    try {
-      app
-
+    use {
       val files = rootFiles(root)
       PyStubsGenerator("$outputPath/${PrebuiltStubsProviderBase.SDK_STUBS_STORAGE_NAME}")
         .buildStubsForRoots(files)
     }
-    finally {
-      tearDown()
-    }
   }
 
   fun generateArchive(zipsDirectory: String, prebuiltStubsArchive: String) {
-    val tmpFolder = FileUtil.createTempDirectory("stubs", null)
-    tmpFolder.delete()
-    tmpFolder.mkdirs()
-
-    try {
-      app
-
+    val stubDir = tempDir.resolve("stubs")
+    use {
       val unzippedFiles = unzipArchivesToRoots(zipsDirectory)
-      PyStubsGenerator("${tmpFolder.absolutePath}/${PrebuiltStubsProviderBase.SDK_STUBS_STORAGE_NAME}")
+      PyStubsGenerator(FileUtil.toSystemIndependentName(stubDir.resolve(PrebuiltStubsProviderBase.SDK_STUBS_STORAGE_NAME).toString()))
         .buildStubsForRoots(unzippedFiles)
 
       println("Generate archive $prebuiltStubsArchive")
       val archive = File(prebuiltStubsArchive)
-      Compressor.Zip(archive).use { it.addDirectory(PyPrebuiltStubsProvider.NAME, tmpFolder) }
-    }
-    finally {
-      FileUtil.delete(tmpFolder)
-      tearDown()
+      Compressor.Zip(archive).use { it.addDirectory(PyPrebuiltStubsProvider.NAME, stubDir.toFile()) }
     }
   }
 }