move ktor to community build scripts, get rid of http jdk client in favour of ktor (attempt 2)

GitOrigin-RevId: 99cd075f101199b399a3092c8eb65990fac90f68

.idea/libraries/ktor_client_encoding.xml

@@ -0,0 +1,12 @@
+<component name="libraryTable">
+  <library name="ktor-client-encoding" type="repository">
+    <properties include-transitive-deps="false" maven-id="io.ktor:ktor-client-encoding-jvm:2.1.2" />
+    <CLASSES>
+      <root url="jar://$MAVEN_REPOSITORY$/io/ktor/ktor-client-encoding-jvm/2.1.2/ktor-client-encoding-jvm-2.1.2.jar!/" />
+    </CLASSES>
+    <JAVADOC />
+    <SOURCES>
+      <root url="jar://$MAVEN_REPOSITORY$/io/ktor/ktor-client-encoding-jvm/2.1.2/ktor-client-encoding-jvm-2.1.2-sources.jar!/" />
+    </SOURCES>
+  </library>
+</component>

build/tasks/src/org/jetbrains/intellij/build/io/http.kt

@@ -1,56 +0,0 @@
-// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
-package org.jetbrains.intellij.build.io
-
-import com.intellij.diagnostic.telemetry.use
-import org.jetbrains.intellij.build.tracer
-import java.io.ByteArrayOutputStream
-import java.lang.Thread.sleep
-import java.net.URI
-import java.net.http.HttpClient
-import java.net.http.HttpRequest
-import java.net.http.HttpResponse
-import java.time.Duration
-import java.util.zip.GZIPInputStream
-
-private val httpClient by lazy {
-  HttpClient.newBuilder()
-    .followRedirects(HttpClient.Redirect.NORMAL)
-    .connectTimeout(Duration.ofSeconds(5))
-    .build()
-}
-
-fun download(url: String): ByteArray {
-  tracer.spanBuilder("download").setAttribute("url", url).use {
-    var attemptNumber = 0
-    while (true) {
-      val request = HttpRequest.newBuilder(URI(url))
-        .header("Accept", "application/json")
-        .header("Accept-Encoding", "gzip")
-        .build()
-
-      val response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream())
-      val statusCode = response.statusCode()
-      val encoding = response.headers().firstValue("Content-Encoding").orElse("")
-      // readAllBytes doesn't work due to incorrect assert in HttpResponseInputStream
-      val byteOut = ByteArrayOutputStream()
-      (if (encoding == "gzip") GZIPInputStream(response.body()) else response.body()).use {
-        it.transferTo(byteOut)
-      }
-      val content = byteOut.toByteArray()
-
-      if (statusCode == 200) {
-        return content
-      }
-      else if (attemptNumber > 3 || statusCode < 500) {
-        // do not attempt again if client error
-        throw RuntimeException("Cannot download $url (status=${response.statusCode()}, content=${content.toString(Charsets.UTF_8)})")
-      }
-      else {
-        attemptNumber++
-        sleep(attemptNumber * 1_000L)
-      }
-    }
-  }
-
-  throw IllegalStateException("must be unreachable")
-}

build/tasks/test/org/jetbrains/intellij/build/tasks/TaskTest.kt

@@ -1,53 +0,0 @@
-// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
-package org.jetbrains.intellij.build.tasks
-
-import com.intellij.testFramework.rules.InMemoryFsExtension
-import org.assertj.core.api.Assertions.assertThat
-import org.junit.jupiter.api.Test
-import org.junit.jupiter.api.extension.RegisterExtension
-import java.io.BufferedInputStream
-import java.io.DataInputStream
-import java.nio.file.Files
-import java.util.*
-
-class TaskTest {
-  @RegisterExtension
-  @JvmField
-  val fs = InMemoryFsExtension()
-
-  companion object {
-    @JvmStatic
-    val logger = object : System.Logger {
-      override fun getName() = ""
-
-      override fun isLoggable(level: System.Logger.Level) = true
-
-      override fun log(level: System.Logger.Level, bundle: ResourceBundle?, message: String, thrown: Throwable?) {
-        if (level == System.Logger.Level.ERROR) {
-          throw RuntimeException(message, thrown)
-        }
-        else {
-          println(message)
-        }
-      }
-
-      override fun log(level: System.Logger.Level, bundle: ResourceBundle?, message: String, vararg params: Any?) {
-        log(level, bundle = null, message = message, thrown = null)
-      }
-
-      override fun log(level: System.Logger.Level, message: String) {
-        log(level = level, bundle = null, message = message, thrown = null)
-      }
-    }
-  }
-
-  @Test
-  fun `broken plugins`() {
-    val targetFile = fs.root.resolve("result")
-    buildBrokenPlugins(targetFile, "2020.3", isInDevelopmentMode = false)
-    DataInputStream(BufferedInputStream(Files.newInputStream(targetFile), 32_000)).use { stream ->
-      assertThat(stream.readByte()).isEqualTo(2)
-      assertThat(stream.readUTF()).isEqualTo("2020.3")
-    }
-  }
-}

platform/bootstrap/dev/src/org/jetbrains/intellij/build/devServer/DevMain.kt

@@ -12,6 +12,7 @@ fun main(rawArgs: Array<String>) {
       platformPrefix = System.getProperty("idea.platform.prefix") ?: "idea",
       additionalModules = emptyList(),
       homePath = Path.of(PathManager.getHomePath()),
+      keepHttpClient = false,
     ))
   }
 

platform/build-scripts/dev-server/intellij.platform.devBuildServer.iml

@@ -47,7 +47,6 @@
     <orderEntry type="module" module-name="intellij.platform.util.classLoader" />
     <orderEntry type="module" module-name="intellij.platform.util.rt.java8" />
     <orderEntry type="library" scope="RUNTIME" name="xz" level="project" />
-    <orderEntry type="library" scope="RUNTIME" name="ktor-client" level="project" />
     <orderEntry type="library" scope="RUNTIME" name="ktor-client-auth" level="project" />
     <orderEntry type="library" scope="RUNTIME" name="ktor-client-cio" level="project" />
   </component>

platform/build-scripts/dev-server/src/DevIdeaBuilder.kt

@@ -7,6 +7,7 @@ import kotlinx.coroutines.runBlocking
 import org.jetbrains.intellij.build.ConsoleSpanExporter
 import org.jetbrains.intellij.build.TraceManager.spanBuilder
 import org.jetbrains.intellij.build.TracerProviderManager
+import org.jetbrains.intellij.build.closeKtorClient
 import java.util.function.Supplier
 
 object DevIdeaBuilder {
@@ -34,5 +35,9 @@ suspend fun buildProductInProcess(request: BuildRequest) {
   spanBuilder("build ide").setAttribute("request", request.toString()).useWithScope2 {
     BuildServer(homePath = request.homePath, productionClassOutput = request.productionClassOutput)
       .buildProductInProcess(isServerMode = false, request = request)
+    // otherwise, thread leak in tests
+    if (!request.keepHttpClient) {
+      closeKtorClient()
+    }
   }
 }

platform/build-scripts/dev-server/src/IdeBuilder.kt

@@ -38,6 +38,7 @@ data class BuildRequest(
   @JvmField val homePath: Path,
   @JvmField val productionClassOutput: Path = Path.of(System.getenv("CLASSES_DIR")
                                                       ?: homePath.resolve("out/classes/production").toString()).toAbsolutePath(),
+  @JvmField val keepHttpClient: Boolean = true,
 )
 
 private suspend fun computeLibClassPath(targetFile: Path, homePath: Path, context: BuildContext) {

platform/build-scripts/intellij.platform.buildScripts.iml

@@ -77,5 +77,9 @@
     <orderEntry type="library" name="jsch-agent-proxy" level="project" />
     <orderEntry type="library" name="jsch-agent-proxy-sshj" level="project" />
     <orderEntry type="library" name="commons-io" level="project" />
+    <orderEntry type="library" name="ktor-client" level="project" />
+    <orderEntry type="library" name="ktor-client-auth" level="project" />
+    <orderEntry type="library" name="ktor-client-cio" level="project" />
+    <orderEntry type="library" name="ktor-client-encoding" level="project" />
   </component>
 </module>

platform/build-scripts/src/org/jetbrains/intellij/build/ProprietaryBuildTools.kt

@@ -62,7 +62,7 @@ class ProprietaryBuildTools(
           error("Must be not called if usePresignedNativeFiles is false")
         }
 
-        override fun commandLineClient(context: BuildContext, os: OsFamily, arch: JvmArchitecture): Path? {
+        override suspend fun commandLineClient(context: BuildContext, os: OsFamily, arch: JvmArchitecture): Path? {
           return null
         }
       },

platform/build-scripts/src/org/jetbrains/intellij/build/SignTool.kt

@@ -21,5 +21,5 @@ interface SignTool {
    */
   suspend fun getPresignedLibraryFile(path: String, libName: String, libVersion: String, context: BuildContext): Path?
 
-  fun commandLineClient(context: BuildContext, os: OsFamily, arch: JvmArchitecture): Path?
+  suspend fun commandLineClient(context: BuildContext, os: OsFamily, arch: JvmArchitecture): Path?
 }

platform/build-scripts/src/org/jetbrains/intellij/build/fus/StatisticsRecorderBundledMetadataProvider.kt

@@ -11,6 +11,7 @@ import kotlinx.coroutines.Job
 import org.jetbrains.intellij.build.BuildContext
 import org.jetbrains.intellij.build.BuildOptions
 import org.jetbrains.intellij.build.TraceManager.spanBuilder
+import org.jetbrains.intellij.build.downloadAsBytes
 import org.jetbrains.intellij.build.impl.ModuleOutputPatcher
 import org.jetbrains.intellij.build.impl.createSkippableJob
 import java.util.concurrent.CancellationException
@@ -23,8 +24,8 @@ internal fun CoroutineScope.createStatisticsRecorderBundledMetadataProviderTask(
   val featureUsageStatisticsProperties = context.proprietaryBuildTools.featureUsageStatisticsProperties ?: return null
   return createSkippableJob(
     spanBuilder("bundle a default version of feature usage statistics"),
-    BuildOptions.FUS_METADATA_BUNDLE_STEP,
-    context
+    taskId = BuildOptions.FUS_METADATA_BUNDLE_STEP,
+    context = context
   ) {
     try {
       val recorderId = featureUsageStatisticsProperties.recorderId
@@ -52,12 +53,12 @@ private fun appendProductCode(uri: String, context: BuildContext): String {
   return if (uri.endsWith('/')) "$uri$name" else "$uri/$name"
 }
 
-private fun download(url: String): ByteArray {
+private suspend fun download(url: String): ByteArray {
   Span.current().addEvent("download", Attributes.of(AttributeKey.stringKey("url"), url))
-  return org.jetbrains.intellij.build.io.download(url)
+  return downloadAsBytes(url)
 }
 
-private fun metadataServiceUri(featureUsageStatisticsProperties: FeatureUsageStatisticsProperties, context: BuildContext): String {
+private suspend fun metadataServiceUri(featureUsageStatisticsProperties: FeatureUsageStatisticsProperties, context: BuildContext): String {
   val providerUri = appendProductCode(featureUsageStatisticsProperties.metadataProviderUri, context)
   Span.current().addEvent("parsing", Attributes.of(AttributeKey.stringKey("url"), providerUri))
   val appInfo = context.applicationInfo

platform/build-scripts/src/org/jetbrains/intellij/build/impl/BundledRuntime.kt

@@ -6,14 +6,14 @@ import org.jetbrains.intellij.build.OsFamily
 import java.nio.file.Path
 
 interface BundledRuntime {
-  fun getHomeForCurrentOsAndArch(): Path
+  suspend fun getHomeForCurrentOsAndArch(): Path
 
   /**
    * contract: returns a directory, where only one subdirectory is available: 'jbr', which contains specified JBR
    */
-  fun extract(prefix: String, os: OsFamily, arch: JvmArchitecture): Path
+  suspend fun extract(prefix: String, os: OsFamily, arch: JvmArchitecture): Path
 
-  fun extractTo(prefix: String, os: OsFamily, destinationDir: Path, arch: JvmArchitecture)
+  suspend fun extractTo(prefix: String, os: OsFamily, destinationDir: Path, arch: JvmArchitecture)
 
   fun archiveName(prefix: String, arch: JvmArchitecture, os: OsFamily, forceVersionWithUnderscores: Boolean = false): String
 

platform/build-scripts/src/org/jetbrains/intellij/build/impl/BundledRuntimeImpl.kt

@@ -7,10 +7,10 @@ import com.intellij.openapi.util.io.NioFiles
 import kotlinx.collections.immutable.persistentListOf
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
 import org.jetbrains.intellij.build.*
+import org.jetbrains.intellij.build.TraceManager.spanBuilder
 import org.jetbrains.intellij.build.dependencies.BuildDependenciesDownloader
 import org.jetbrains.intellij.build.dependencies.BuildDependenciesExtractOptions
 import org.jetbrains.intellij.build.dependencies.DependenciesProperties
-import java.net.URI
 import java.nio.file.*
 import java.nio.file.attribute.BasicFileAttributes
 import java.nio.file.attribute.DosFileAttributeView
@@ -34,7 +34,7 @@ class BundledRuntimeImpl(
     options.bundledRuntimeBuild ?: dependenciesProperties.property("runtimeBuild")
   }
 
-  override fun getHomeForCurrentOsAndArch(): Path {
+  override suspend fun getHomeForCurrentOsAndArch(): Path {
     var prefix = "jbr_jcef-"
     val os = OsFamily.currentOs
     val arch = JvmArchitecture.currentJvmArch
@@ -59,7 +59,7 @@ class BundledRuntimeImpl(
   }
 
   // contract: returns a directory, where only one subdirectory is available: 'jbr', which contains specified JBR
-  override fun extract(prefix: String, os: OsFamily, arch: JvmArchitecture): Path {
+  override suspend fun extract(prefix: String, os: OsFamily, arch: JvmArchitecture): Path {
     val targetDir = paths.communityHomeDir.resolve("build/download/${prefix}${build}-${os.jbrArchiveSuffix}-$arch")
     val jbrDir = targetDir.resolve("jbr")
 
@@ -80,14 +80,14 @@ class BundledRuntimeImpl(
     return targetDir
   }
 
-  override fun extractTo(prefix: String, os: OsFamily, destinationDir: Path, arch: JvmArchitecture) {
+  override suspend fun extractTo(prefix: String, os: OsFamily, destinationDir: Path, arch: JvmArchitecture) {
     doExtract(findArchive(prefix, os, arch), destinationDir, os)
   }
 
-  private fun findArchive(prefix: String, os: OsFamily, arch: JvmArchitecture): Path {
-    val archiveName = archiveName(prefix, arch, os)
-    val url = URI("https://cache-redirector.jetbrains.com/intellij-jbr/$archiveName")
-    return BuildDependenciesDownloader.downloadFileToCacheLocation(paths.communityHomeDirRoot, url)
+  private suspend fun findArchive(prefix: String, os: OsFamily, arch: JvmArchitecture): Path {
+    val archiveName = archiveName(prefix = prefix, arch = arch, os = os)
+    val url = "https://cache-redirector.jetbrains.com/intellij-jbr/$archiveName"
+    return downloadFileToCacheLocation(url = url, communityRoot = paths.communityHomeDirRoot)
   }
 
   /**
@@ -145,7 +145,7 @@ private fun getArchSuffix(arch: JvmArchitecture): String {
 }
 
 private fun doExtract(archive: Path, destinationDir: Path, os: OsFamily) {
-  TraceManager.spanBuilder("extract JBR")
+  spanBuilder("extract JBR")
     .setAttribute("archive", archive.toString())
     .setAttribute("os", os.osName)
     .setAttribute("destination", destinationDir.toString())

platform/build-scripts/src/org/jetbrains/intellij/build/impl/CompilationContextImpl.kt

@@ -271,9 +271,13 @@ class CompilationContextImpl private constructor(model: JpsModel,
     this.nameToModule = modules.associateByTo(HashMap(modules.size)) { it.name }
     val buildOut = options.outputRootPath ?: buildOutputRootEvaluator(project)
     val logDir = options.logPath?.let { Path.of(it).toAbsolutePath().normalize() } ?: buildOut.resolve("log")
-    paths = BuildPathsImpl(communityHome, projectHome, buildOut, logDir)
+    paths = BuildPathsImpl(communityHome = communityHome, projectHome = projectHome, buildOut = buildOut, logDir = logDir)
     dependenciesProperties = DependenciesProperties(paths.communityHomeDirRoot)
-    bundledRuntime = BundledRuntimeImpl(options, paths, dependenciesProperties, messages::error, messages::info)
+    bundledRuntime = BundledRuntimeImpl(options = options,
+                                        paths = paths,
+                                        dependenciesProperties = dependenciesProperties,
+                                        error = messages::error,
+                                        info = messages::info)
   }
 }
 

platform/build-scripts/src/org/jetbrains/intellij/build/impl/MacDmgBuilder.kt

@@ -113,11 +113,11 @@ private suspend fun generateIntegrityManifest(sitFile: Path, sitRoot: String, co
   }
 }
 
-private fun buildAndSignWithMacBuilderHost(sitFile: Path,
-                                           macHostProperties: MacHostProperties,
-                                           notarize: Boolean,
-                                           customizer: MacDistributionCustomizer,
-                                           context: BuildContext) {
+private suspend fun buildAndSignWithMacBuilderHost(sitFile: Path,
+                                                   macHostProperties: MacHostProperties,
+                                                   notarize: Boolean,
+                                                   customizer: MacDistributionCustomizer,
+                                                   context: BuildContext) {
   val dmgImage = if (context.options.buildStepsToSkip.contains(BuildOptions.MAC_DMG_STEP)) {
     null
   }

platform/build-scripts/src/org/jetbrains/intellij/build/impl/TestingTasksImpl.kt

@@ -15,6 +15,7 @@ import com.intellij.openapi.util.io.NioFiles
 import com.intellij.openapi.util.text.StringUtilRt
 import com.intellij.util.lang.UrlClassLoader
 import io.opentelemetry.api.common.AttributeKey
+import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.runBlocking
 import org.jetbrains.intellij.build.*
 import org.jetbrains.intellij.build.CompilationTasks.Companion.create
@@ -336,9 +337,9 @@ internal class TestingTasksImpl(private val context: CompilationContext, private
       messages.info("Tests from bucket ${allSystemProperties[TestCaseLoader.TEST_RUNNER_INDEX_FLAG]} of ${numberOfBuckets} will be executed")
     }
     messages.block("Test classpath and runtime info") {
-      val runtime = runtimeExecutablePath().toString()
-      messages.info("Runtime: ${runtime}")
-      runBlocking {
+      runBlocking(Dispatchers.IO) {
+        val runtime = getRuntimeExecutablePath().toString()
+        messages.info("Runtime: ${runtime}")
         runProcess(args = listOf(runtime, "-version"), inheritOut = true)
       }
       messages.info("Runtime options: ${allJvmArgs}")
@@ -346,6 +347,7 @@ internal class TestingTasksImpl(private val context: CompilationContext, private
       messages.info("Bootstrap classpath: ${bootstrapClasspath}")
       messages.info("Tests classpath: ${testClasspath}")
       modulePath?.let { mp ->
+        @Suppress("SpellCheckingInspection")
         messages.info("Tests modulepath: $mp")
       }
       if (!envVariables.isEmpty()) {
@@ -363,7 +365,7 @@ internal class TestingTasksImpl(private val context: CompilationContext, private
     notifySnapshotBuilt(allJvmArgs)
   }
 
-  private fun runtimeExecutablePath(): Path {
+  private suspend fun getRuntimeExecutablePath(): Path {
     val runtimeDir: Path
     if (options.customRuntimePath != null) {
       runtimeDir = Path.of(options.customRuntimePath)
@@ -376,7 +378,7 @@ internal class TestingTasksImpl(private val context: CompilationContext, private
     }
 
     var java = runtimeDir.resolve(if (SystemInfoRt.isWindows) "bin/java.exe" else "bin/java")
-    if (SystemInfoRt.isMac && !Files.exists(java)) {
+    if (SystemInfoRt.isMac && Files.notExists(java)) {
       java = runtimeDir.resolve("Contents/Home/bin/java")
     }
     check(Files.exists(java)) { "java executable is missing: ${java}" }
@@ -422,6 +424,7 @@ internal class TestingTasksImpl(private val context: CompilationContext, private
     jvmArgs += options.jvmMemoryOptions?.split(Regex("\\s+")) ?: listOf("-Xms750m", "-Xmx750m")
 
     val tempDir = System.getProperty("teamcity.build.tempDir", System.getProperty("java.io.tmpdir"))
+    @Suppress("SpellCheckingInspection")
     mapOf(
       "idea.platform.prefix" to options.platformPrefix,
       "idea.home.path" to context.paths.projectHome.toString(),
@@ -441,11 +444,11 @@ internal class TestingTasksImpl(private val context: CompilationContext, private
       "io.netty.leakDetectionLevel" to "PARANOID",
       "kotlinx.coroutines.debug" to "on",
       "sun.io.useCanonCaches" to "false",
-    ).forEach(BiConsumer { k, v ->
+    ).forEach { (k, v) ->
       if (v != null) {
         systemProperties.putIfAbsent(k, v)
       }
-    })
+    }
 
     System.getProperties().forEach(BiConsumer { key, value ->
       if ((key as String).startsWith("pass.")) {
@@ -673,7 +676,7 @@ internal class TestingTasksImpl(private val context: CompilationContext, private
     }
 
     val argFile = CommandLineWrapperUtil.createArgumentFile(args, Charset.defaultCharset())
-    val runtime = runtimeExecutablePath().toString()
+    val runtime = runBlocking(Dispatchers.IO) { getRuntimeExecutablePath ().toString() }
     context.messages.info("Starting tests on runtime ${runtime}")
     val builder = ProcessBuilder(runtime, "@" + argFile.absolutePath)
     builder.environment().putAll(envVariables)
@@ -746,6 +749,7 @@ private fun publishTestDiscovery(messages: BuildMessages, file: String?) {
       val map = LinkedHashMap<String, String>(7)
       map["teamcity-build-number"] = System.getProperty("build.number")
       map["teamcity-build-type-id"] = System.getProperty("teamcity.buildType.id")
+      @Suppress("SpellCheckingInspection")
       map["teamcity-build-configuration-name"] = System.getenv("TEAMCITY_BUILDCONF_NAME")
       map["teamcity-build-project-name"] = System.getenv("TEAMCITY_PROJECT_NAME")
       map["branch"] = System.getProperty("teamcity.build.branch")?.takeIf(String::isNotEmpty) ?: "master"

platform/build-scripts/src/org/jetbrains/intellij/build/impl/brokenPlugins.kt

@@ -1,11 +1,11 @@
-// Copyright 2000-2021 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
-package org.jetbrains.intellij.build.tasks
+// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package org.jetbrains.intellij.build.impl
 
 import io.opentelemetry.api.trace.Span
 import kotlinx.serialization.Serializable
 import kotlinx.serialization.builtins.ListSerializer
 import kotlinx.serialization.json.Json
-import org.jetbrains.intellij.build.io.download
+import org.jetbrains.intellij.build.downloadAsText
 import java.io.BufferedOutputStream
 import java.io.DataOutputStream
 import java.nio.file.Files
@@ -17,11 +17,13 @@ private const val MARKETPLACE_BROKEN_PLUGINS_URL = "https://plugins.jetbrains.co
 /**
  * Generate brokenPlugins.txt file using JetBrains Marketplace.
  */
-fun buildBrokenPlugins(targetFile: Path, currentBuildString: String, isInDevelopmentMode: Boolean) {
+suspend fun buildBrokenPlugins(targetFile: Path, currentBuildString: String, isInDevelopmentMode: Boolean) {
   val span = Span.current()
 
   val allBrokenPlugins = try {
-    downloadFileFromMarketplace(span)
+    val jsonFormat = Json { ignoreUnknownKeys = true }
+    val content = downloadAsText(MARKETPLACE_BROKEN_PLUGINS_URL)
+    jsonFormat.decodeFromString(ListSerializer(MarketplaceBrokenPlugin.serializer()), content)
   }
   catch (e: Exception) {
     if (isInDevelopmentMode) {
@@ -49,12 +51,6 @@ fun buildBrokenPlugins(targetFile: Path, currentBuildString: String, isInDevelop
   span.setAttribute("pluginCount", result.size.toLong())
 }
 
-private fun downloadFileFromMarketplace(span: Span): List<MarketplaceBrokenPlugin> {
-  val jsonFormat = Json { ignoreUnknownKeys = true }
-  val content = download(MARKETPLACE_BROKEN_PLUGINS_URL).toString(Charsets.UTF_8)
-  return jsonFormat.decodeFromString(ListSerializer(MarketplaceBrokenPlugin.serializer()), content)
-}
-
 private fun storeBrokenPlugin(brokenPlugin: Map<String, Set<String>>, build: String, targetFile: Path) {
   Files.createDirectories(targetFile.parent)
   DataOutputStream(BufferedOutputStream(Files.newOutputStream(targetFile), 32_000)).use { out ->

platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/CompilationPartsUtil.kt

@@ -19,7 +19,6 @@ import org.jetbrains.annotations.VisibleForTesting
 import org.jetbrains.intellij.build.BuildMessages
 import org.jetbrains.intellij.build.CompilationContext
 import org.jetbrains.intellij.build.TraceManager.spanBuilder
-import org.jetbrains.intellij.build.httpClient
 import org.jetbrains.intellij.build.io.AddDirEntriesMode
 import org.jetbrains.intellij.build.io.deleteDir
 import org.jetbrains.intellij.build.io.zip

platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/http.kt

@@ -1,5 +1,5 @@
 // Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
-package org.jetbrains.intellij.build
+package org.jetbrains.intellij.build.impl.compilation
 
 import okhttp3.MediaType.Companion.toMediaType
 import okhttp3.OkHttpClient
@@ -29,7 +29,7 @@ internal inline fun <T> Response.useSuccessful(task: (Response) -> T): T {
   }
 }
 
-internal val httpClient by lazy {
+internal val httpClient: OkHttpClient by lazy {
   val timeout = 1L
   val unit = TimeUnit.MINUTES
   OkHttpClient.Builder()
@@ -57,26 +57,14 @@ internal val httpClient by lazy {
           if (error == null) {
             error = IOException("$maxTryCount attempts to ${request.method} ${request.url} failed")
           }
-          error?.addSuppressed(e)
+          error.addSuppressed(e)
           null
         }
         tryCount++
       }
-      while ((response == null || response!!.code >= 500) && tryCount < maxTryCount)
+      while ((response == null || response.code >= 500) && tryCount < maxTryCount)
       response ?: throw error ?: IllegalStateException()
     }
     .followRedirects(true)
     .build()
-}
-
-@Suppress("HttpUrlsUsage")
-internal fun toUrlWithTrailingSlash(serverUrl: String): String {
-  var result = serverUrl
-  if (!result.startsWith("http://") && !result.startsWith("https://")) {
-    result = "http://$result"
-  }
-  if (!result.endsWith('/')) {
-    result += '/'
-  }
-  return result
 }

platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/upload.kt

@@ -17,10 +17,7 @@ import okhttp3.*
 import okhttp3.MediaType.Companion.toMediaType
 import okhttp3.RequestBody.Companion.toRequestBody
 import okio.*
-import org.jetbrains.intellij.build.MEDIA_TYPE_BINARY
 import org.jetbrains.intellij.build.TraceManager.spanBuilder
-import org.jetbrains.intellij.build.httpClient
-import org.jetbrains.intellij.build.useSuccessful
 import java.nio.ByteBuffer
 import java.nio.channels.FileChannel
 import java.nio.file.Files

platform/build-scripts/src/org/jetbrains/intellij/build/ktor.kt

@@ -0,0 +1,255 @@
+@file:Suppress("BlockingMethodInNonBlockingContext", "ReplaceGetOrSet")
+package org.jetbrains.intellij.build
+
+import com.intellij.diagnostic.telemetry.useWithScope2
+import com.intellij.util.concurrency.SynchronizedClearableLazy
+import io.ktor.client.HttpClient
+import io.ktor.client.call.body
+import io.ktor.client.engine.cio.CIO
+import io.ktor.client.plugins.HttpRequestRetry
+import io.ktor.client.plugins.UserAgent
+import io.ktor.client.plugins.auth.Auth
+import io.ktor.client.plugins.auth.providers.BasicAuthCredentials
+import io.ktor.client.plugins.auth.providers.BearerTokens
+import io.ktor.client.plugins.auth.providers.basic
+import io.ktor.client.plugins.auth.providers.bearer
+import io.ktor.client.plugins.compression.ContentEncoding
+import io.ktor.client.request.get
+import io.ktor.client.statement.bodyAsChannel
+import io.ktor.client.statement.bodyAsText
+import io.ktor.http.HttpHeaders
+import io.ktor.util.read
+import io.ktor.utils.io.*
+import io.ktor.utils.io.core.use
+import io.ktor.utils.io.jvm.nio.copyTo
+import io.opentelemetry.api.common.AttributeKey
+import io.opentelemetry.api.common.Attributes
+import io.opentelemetry.api.trace.Span
+import kotlinx.coroutines.*
+import org.jetbrains.intellij.build.TraceManager.spanBuilder
+import org.jetbrains.intellij.build.dependencies.BuildDependenciesCommunityRoot
+import org.jetbrains.intellij.build.dependencies.BuildDependenciesDownloader
+import org.jetbrains.xxh3.Xx3UnencodedString
+import java.nio.channels.FileChannel
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.StandardCopyOption
+import java.nio.file.StandardOpenOption
+import java.nio.file.attribute.FileTime
+import java.time.Instant
+import java.util.*
+import java.util.concurrent.locks.*
+import kotlin.time.Duration.Companion.hours
+
+const val SPACE_REPO_HOST = "packages.jetbrains.team"
+
+private val httpClient = SynchronizedClearableLazy {
+  // HttpTimeout is not used - CIO engine handles that
+  HttpClient(CIO) {
+    // we have custom error handler
+    expectSuccess = true
+
+    engine {
+      requestTimeout = 2.hours.inWholeMilliseconds
+    }
+
+    install(ContentEncoding) {
+      deflate(1.0F)
+      gzip(0.9F)
+    }
+
+    install(HttpRequestRetry) {
+      retryOnExceptionOrServerErrors(maxRetries = 3)
+      exponentialDelay()
+    }
+
+    install(UserAgent) {
+      agent = "Build Script Downloader"
+    }
+  }
+}
+
+private val httpSpaceClient = SynchronizedClearableLazy {
+  httpClient.value.config {
+    // we have custom error handler
+    expectSuccess = false
+
+    val token = System.getenv("SPACE_PACKAGE_TOKEN")
+    if (!token.isNullOrEmpty()) {
+      install(Auth) {
+        bearer {
+          sendWithoutRequest { request ->
+            request.url.host == SPACE_REPO_HOST
+          }
+
+          loadTokens {
+            BearerTokens(token, "")
+          }
+        }
+      }
+    }
+    else {
+      val userName = System.getProperty("jps.auth.spaceUsername")
+      val password = System.getProperty("jps.auth.spacePassword")
+      if (userName != null && password != null) {
+        install(Auth) {
+          basic {
+            sendWithoutRequest { request ->
+              request.url.host == SPACE_REPO_HOST
+            }
+
+            credentials {
+              BasicAuthCredentials(username = userName, password = password)
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+fun closeKtorClient() {
+  httpClient.drop()?.close()
+  httpSpaceClient.drop()?.close()
+}
+
+private val fileLocks = StripedMutex()
+
+suspend fun downloadAsBytes(url: String): ByteArray {
+  return spanBuilder("download").setAttribute("url", url).useWithScope2 {
+    withContext(Dispatchers.IO) {
+      httpClient.value.get(url).body()
+    }
+  }
+}
+
+suspend fun downloadAsText(url: String): String {
+  return spanBuilder("download").setAttribute("url", url).useWithScope2 {
+    withContext(Dispatchers.IO) {
+      httpClient.value.get(url).bodyAsText()
+    }
+  }
+}
+
+suspend fun downloadFileToCacheLocation(url: String, context: BuildContext): Path {
+  return downloadFileToCacheLocation(url, context.paths.communityHomeDirRoot)
+}
+
+suspend fun downloadFileToCacheLocation(url: String, communityRoot: BuildDependenciesCommunityRoot): Path {
+  BuildDependenciesDownloader.cleanUpIfRequired(communityRoot)
+
+  val target = BuildDependenciesDownloader.getTargetFile(communityRoot, url)
+  val targetPath = target.toString()
+  val lock = fileLocks.getLock(Xx3UnencodedString.hashUnencodedString(targetPath).toInt())
+  lock.lock()
+  try {
+    val now = Instant.now()
+    if (Files.exists(target)) {
+      Span.current().addEvent("use asset from cache", Attributes.of(
+        AttributeKey.stringKey("url"), url,
+        AttributeKey.stringKey("target"), targetPath,
+      ))
+
+      // update file modification time to maintain FIFO caches i.e. in persistent cache folder on TeamCity agent
+      Files.setLastModifiedTime(target, FileTime.from(now))
+      return target
+    }
+
+    return spanBuilder("download").setAttribute("url", url).setAttribute("target", targetPath).useWithScope2 {
+      // save to the same disk to ensure that move will be atomic and not as a copy
+      val tempFile = target.parent
+        .resolve("${target.fileName}-${(now.epochSecond - 1634886185).toString(36)}-${now.nano.toString(36)}".take(255))
+      try {
+        val response = httpSpaceClient.value.get(url)
+        coroutineScope {
+          response.bodyAsChannel().copyAndClose(writeChannel(tempFile))
+        }
+        val statusCode = response.status.value
+        if (statusCode != 200) {
+          val builder = StringBuilder("Cannot download\n")
+          val headers = response.headers
+          headers.names()
+            .asSequence()
+            .sorted()
+            .flatMap { headerName -> headers.getAll(headerName)!!.map { value -> "Header: $headerName: $value\n" } }
+            .forEach(builder::append)
+          builder.append('\n')
+          if (Files.exists(tempFile)) {
+            Files.newInputStream(tempFile).use { inputStream ->
+              // yes, not trying to guess encoding
+              // string constructor should be exception free,
+              // so at worse we'll get some random characters
+              builder.append(inputStream.readNBytes(1024).toString(StandardCharsets.UTF_8))
+            }
+          }
+          throw BuildDependenciesDownloader.HttpStatusException(builder.toString(), statusCode, url)
+        }
+
+        val contentLength = response.headers.get(HttpHeaders.ContentLength)?.toLongOrNull() ?: -1
+        check(contentLength > 0) { "Header '${HttpHeaders.ContentLength}' is missing or zero for $url" }
+        val fileSize = Files.size(tempFile)
+        check(fileSize == contentLength) {
+          "Wrong file length after downloading uri '$url' to '$tempFile': expected length $contentLength " +
+          "from Content-Length header, but got $fileSize on disk"
+        }
+        Files.move(tempFile, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING)
+      }
+      finally {
+        Files.deleteIfExists(tempFile)
+      }
+
+      target
+    }
+  }
+  finally {
+    lock.unlock()
+  }
+}
+
+fun CoroutineScope.readChannel(file: Path): ByteReadChannel {
+  return writer(CoroutineName("file-reader") + Dispatchers.IO, autoFlush = false) {
+    @Suppress("BlockingMethodInNonBlockingContext")
+    FileChannel.open(file, StandardOpenOption.READ).use { fileChannel ->
+      @Suppress("DEPRECATION")
+      channel.writeSuspendSession {
+        while (true) {
+          val buffer = request(1)
+          if (buffer == null) {
+            channel.flush()
+            tryAwait(1)
+            continue
+          }
+
+          val rc = fileChannel.read(buffer)
+          if (rc == -1) {
+            break
+          }
+          written(rc)
+        }
+      }
+    }
+  }.channel
+}
+
+private val WRITE_NEW_OPERATION = EnumSet.of(StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW)
+
+internal fun CoroutineScope.writeChannel(file: Path): ByteWriteChannel {
+  return reader(CoroutineName("file-writer") + Dispatchers.IO, autoFlush = true) {
+    FileChannel.open(file, WRITE_NEW_OPERATION).use { fileChannel ->
+      channel.copyTo(fileChannel)
+    }
+  }.channel
+}
+
+@Suppress("HttpUrlsUsage")
+internal fun toUrlWithTrailingSlash(serverUrl: String): String {
+  var result = serverUrl
+  if (!result.startsWith("http://") && !result.startsWith("https://")) {
+    result = "http://$result"
+  }
+  if (!result.endsWith('/')) {
+    result += '/'
+  }
+  return result
+}

platform/build-scripts/src/org/jetbrains/intellij/build/striped.kt

@@ -0,0 +1,16 @@
+// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package org.jetbrains.intellij.build
+
+import kotlinx.coroutines.sync.Mutex
+
+private const val MAX_POWER_OF_TWO = 1 shl Integer.SIZE - 2
+private const val ALL_SET = 0.inv()
+
+internal class StripedMutex(stripeCount: Int = 64) {
+  private val mask = if (stripeCount > MAX_POWER_OF_TWO) ALL_SET else (1 shl (Integer.SIZE - Integer.numberOfLeadingZeros(stripeCount - 1))) - 1
+  private val locks = Array(mask + 1) { Mutex() }
+
+  fun getLock(hash: Int): Mutex {
+    return locks[hash and mask]
+  }
+}

platform/build-scripts/testFramework/src/org/jetbrains/intellij/build/testFramework/buildScriptTestUtils.kt

@@ -192,6 +192,8 @@ suspend fun runTestBuild(context: BuildContext, traceSpanName: String? = null, o
       }
   }
   finally {
+    closeKtorClient()
+
     // close debug logging to prevent locking of output directory on Windows
     (context.messages as BuildMessagesImpl).close()
 

platform/build-scripts/tests/testSrc/org/jetbrains/intellij/build/impl/BundledRuntimeTest.kt

@@ -3,6 +3,8 @@ package org.jetbrains.intellij.build.impl
 
 import com.intellij.openapi.util.io.FileUtil
 import com.intellij.openapi.util.io.NioFiles
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.runBlocking
 import org.jetbrains.intellij.build.*
 import org.jetbrains.intellij.build.dependencies.JdkDownloader
 import org.junit.Test
@@ -10,7 +12,7 @@ import java.nio.file.Files
 
 class BundledRuntimeTest {
   @Test
-  fun download() {
+  fun download(): Unit = runBlocking(Dispatchers.IO) {
     withCompilationContext { context ->
       val bundledRuntime = BundledRuntimeImpl(context.options, context.paths, context.dependenciesProperties, context.messages::error, context.messages::info)
       val currentJbr = bundledRuntime.getHomeForCurrentOsAndArch()
@@ -53,8 +55,16 @@ class BundledRuntimeTest {
   @Test
   fun currentArchDownload() {
     withCompilationContext { context ->
-      val currentJbrHome = BundledRuntimeImpl(context.options, context.paths, context.dependenciesProperties, context.messages::error, context.messages::info)
-        .getHomeForCurrentOsAndArch()
+      val currentJbrHome = runBlocking(Dispatchers.IO) {
+        BundledRuntimeImpl(
+          options = context.options,
+          paths = context.paths,
+          dependenciesProperties = context.dependenciesProperties,
+          error = context.messages::error,
+          info = context.messages::info
+        )
+          .getHomeForCurrentOsAndArch()
+      }
       val javaExe = JdkDownloader.getJavaExecutable(currentJbrHome)
       val process = ProcessBuilder(javaExe.toString(), "-version")
         .inheritIO()
@@ -66,7 +76,7 @@ class BundledRuntimeTest {
     }
   }
 
-  private fun withCompilationContext(block: (CompilationContext) -> Unit) {
+  private inline fun withCompilationContext(block: (CompilationContext) -> Unit) {
     val tempDir = Files.createTempDirectory("compilation-context-")
     try {
       val communityHome = IdeaProjectLoaderUtil.guessCommunityHome(javaClass)