fix cross-platform build (part 2)

GitOrigin-RevId: 8e228c45e883dbf17dbe4284367c62e3ba1e7c4a

build/tasks/src/org/jetbrains/intellij/build/io/byteBuffer.kt

@@ -42,7 +42,7 @@ private val unmap by lazy {
   lookup.findVirtual(unsafeClass, "invokeCleaner", MethodType.methodType(Void.TYPE, ByteBuffer::class.java)).bindTo(unsafe)
 }
 
-internal fun unmapBuffer(buffer: ByteBuffer) {
+fun unmapBuffer(buffer: ByteBuffer) {
   unmap.invokeExact(buffer)
 }
 

build/tasks/src/org/jetbrains/intellij/build/io/zip.kt

@@ -23,20 +23,12 @@ fun zipWithCompression(targetFile: Path,
   Files.createDirectories(targetFile.parent)
   ZipFileWriter(channel = FileChannel.open(targetFile, if (overwrite) W_OVERWRITE else W_CREATE_NEW),
                 deflater = Deflater(compressionLevel, true)).use { zipFileWriter ->
-    val fileAdded: ((String) -> Boolean)?
-    val dirNameSetToAdd: Set<String>
     if (addDirEntriesMode == AddDirEntriesMode.NONE) {
-      if (fileFilter == null) {
-        fileAdded = null
-      }
-      else {
-        fileAdded = fileFilter
-      }
-      dirNameSetToAdd = emptySet()
+      doArchive(zipFileWriter = zipFileWriter, fileAdded = fileFilter, dirs = dirs)
     }
     else {
-      dirNameSetToAdd = LinkedHashSet()
-      fileAdded = { name ->
+      val dirNameSetToAdd = LinkedHashSet<String>()
+      val fileAdded = { name: String ->
         if (fileFilter != null && !fileFilter(name)) {
           false
         }
@@ -49,11 +41,11 @@ fun zipWithCompression(targetFile: Path,
           true
         }
       }
-    }
 
-    doArchive(zipFileWriter = zipFileWriter, fileAdded = fileAdded, dirs = dirs)
-    for (dir in dirNameSetToAdd) {
-      zipFileWriter.dir(dir)
+      doArchive(zipFileWriter = zipFileWriter, fileAdded = fileAdded, dirs = dirs)
+      for (dir in dirNameSetToAdd) {
+        zipFileWriter.dir(dir)
+      }
     }
   }
 }
@@ -65,19 +57,13 @@ fun zip(targetFile: Path,
         overwrite: Boolean = false,
         fileFilter: ((name: String) -> Boolean)? = null) {
   Files.createDirectories(targetFile.parent)
-  val packageIndexBuilder = PackageIndexBuilder()
   ZipFileWriter(channel = FileChannel.open(targetFile, if (overwrite) W_OVERWRITE else W_CREATE_NEW)).use { zipFileWriter ->
-    val fileAdded: ((String) -> Boolean)?
     if (addDirEntriesMode == AddDirEntriesMode.NONE) {
-      if (fileFilter == null) {
-        fileAdded = null
-      }
-      else {
-        fileAdded = fileFilter
-      }
+      doArchive(zipFileWriter = zipFileWriter, fileAdded = fileFilter, dirs = dirs)
     }
     else {
-      fileAdded = { name ->
+      val packageIndexBuilder = PackageIndexBuilder()
+      val fileAdded = { name: String ->
         if (fileFilter != null && !fileFilter(name)) {
           false
         }
@@ -86,9 +72,9 @@ fun zip(targetFile: Path,
           true
         }
       }
+      doArchive(zipFileWriter = zipFileWriter, fileAdded = fileAdded, dirs = dirs)
+      packageIndexBuilder.writePackageIndex(zipFileWriter, addDirEntriesMode = addDirEntriesMode)
     }
-    doArchive(zipFileWriter = zipFileWriter, fileAdded = fileAdded, dirs = dirs)
-    packageIndexBuilder.writePackageIndex(zipFileWriter, addDirEntriesMode = addDirEntriesMode)
   }
 }
 

build/tasks/src/org/jetbrains/intellij/build/tasks/mergeJars.kt

@@ -204,6 +204,8 @@ private fun getIgnoredNames(): Set<String> {
   set.add("native-image")
   set.add("native")
   set.add("licenses")
+  set.add("META-INF/LGPL2.1")
+  set.add("META-INF/AL2.0")
   @Suppress("SpellCheckingInspection")
   set.add(".gitkeep")
   set.add(INDEX_FILENAME)

platform/build-scripts/dev-server/src/IdeBuilder.kt

@@ -10,6 +10,7 @@ import com.intellij.util.lang.PathClassLoader
 import com.intellij.util.lang.UrlClassLoader
 import io.opentelemetry.api.common.AttributeKey
 import io.opentelemetry.api.trace.Span
+import kotlinx.collections.immutable.persistentListOf
 import kotlinx.coroutines.*
 import org.jetbrains.intellij.build.*
 import org.jetbrains.intellij.build.TraceManager.spanBuilder
@@ -367,7 +368,7 @@ private fun createBuildOptions(runDir: Path): BuildOptions {
   val options = BuildOptions()
   options.printFreeSpace = false
   options.useCompiledClassesFromProjectOutput = true
-  options.targetOs = BuildOptions.OS_NONE
+  options.targetOs = persistentListOf()
   options.cleanOutputFolder = false
   options.skipDependencySetup = true
   options.outputRootPath = runDir

platform/build-scripts/src/org/jetbrains/intellij/build/BuildOptions.kt

@@ -1,8 +1,9 @@
 // Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package org.jetbrains.intellij.build
 
-import com.intellij.openapi.util.SystemInfoRt
 import com.intellij.util.SystemProperties
+import kotlinx.collections.immutable.PersistentList
+import kotlinx.collections.immutable.persistentListOf
 import org.jetbrains.annotations.ApiStatus
 import org.jetbrains.annotations.ApiStatus.Internal
 import java.nio.file.Path
@@ -181,7 +182,7 @@ class BuildOptions {
   /**
    * Specifies for which operating systems distributions should be built.
    */
-  var targetOs: String
+  var targetOs: PersistentList<OsFamily>
 
   /**
    * Specifies for which arch distributions should be built. null means all
@@ -333,20 +334,21 @@ class BuildOptions {
   @ApiStatus.Experimental
   var signNativeFiles = true
 
+  @ApiStatus.Experimental
+  @ApiStatus.Internal
+  var compressZipFiles = true
+
   init {
-    var targetOs = System.getProperty(TARGET_OS_PROPERTY, OS_ALL)
-    if (targetOs == OS_CURRENT) {
-      targetOs = when {
-        SystemInfoRt.isWindows -> OS_WINDOWS
-        SystemInfoRt.isMac -> OS_MAC
-        SystemInfoRt.isLinux -> OS_LINUX
-        else -> throw RuntimeException("Unknown OS")
-      }
+    val targetOsId = System.getProperty(TARGET_OS_PROPERTY, OS_ALL).lowercase()
+    targetOs = when {
+      targetOsId == OS_CURRENT -> persistentListOf(OsFamily.currentOs)
+      targetOsId.isEmpty() || targetOsId == OS_ALL -> OsFamily.ALL
+      targetOsId == OS_NONE -> persistentListOf()
+      targetOsId == OsFamily.MACOS.osId -> persistentListOf(OsFamily.MACOS)
+      targetOsId == OsFamily.WINDOWS.osId -> persistentListOf(OsFamily.WINDOWS)
+      targetOsId == OsFamily.LINUX.osId -> persistentListOf(OsFamily.LINUX)
+      else -> throw IllegalStateException("Unknown target OS $targetOsId")
     }
-    else if (targetOs.isEmpty()) {
-      targetOs = OS_ALL
-    }
-    this.targetOs = targetOs
 
     val sourceDateEpoch = System.getenv("SOURCE_DATE_EPOCH")
     buildDateInSeconds = sourceDateEpoch?.toLong() ?: (System.currentTimeMillis() / 1000)

platform/build-scripts/src/org/jetbrains/intellij/build/BuildTasks.kt

@@ -72,13 +72,6 @@ interface BuildTasks {
    */
   fun buildFullUpdaterJar()
 
-  /**
-   * Performs a fast dry run to check that the build scripts a properly configured. It'll run compilation, build platform and plugin JAR files,
-   * build searchable options index and scramble the main JAR, but won't produce the product archives and installation files which occupy a lot
-   * of disk space and take a long time to build.
-   */
-  suspend fun runTestBuild()
-
   suspend fun buildUnpackedDistribution(targetDirectory: Path, includeBinAndRuntime: Boolean)
 
   suspend fun buildDmg(macZipDir: Path)

platform/build-scripts/src/org/jetbrains/intellij/build/impl/BuildContextImpl.kt

@@ -243,14 +243,10 @@ class BuildContextImpl private constructor(
     return true
   }
 
-  override fun shouldBuildDistributions(): Boolean {
-    return options.targetOs.lowercase() != BuildOptions.OS_NONE
-  }
+  override fun shouldBuildDistributions(): Boolean = !options.targetOs.isEmpty()
 
   override fun shouldBuildDistributionForOS(os: OsFamily, arch: JvmArchitecture): Boolean {
-    return shouldBuildDistributions()
-           && listOf(BuildOptions.OS_ALL, os.osId).contains(options.targetOs.lowercase())
-           && (options.targetArch == null || options.targetArch == arch)
+    return shouldBuildDistributions() && options.targetOs.contains(os) && (options.targetArch == null || options.targetArch == arch)
   }
 
   override fun createCopyForProduct(productProperties: ProductProperties, projectHomeForCustomizers: Path): BuildContext {

platform/build-scripts/src/org/jetbrains/intellij/build/impl/BuildTasksImpl.kt

@@ -53,7 +53,6 @@ import java.nio.file.StandardCopyOption
 import java.nio.file.attribute.DosFileAttributeView
 import java.nio.file.attribute.PosixFilePermission
 import java.util.*
-import java.util.function.BiConsumer
 import java.util.function.Predicate
 import java.util.stream.Collectors
 
@@ -139,27 +138,10 @@ class BuildTasksImpl(context: BuildContext) : BuildTasks {
     doBuildUpdaterJar(context, "updater.jar")
   }
 
-  override suspend fun runTestBuild() {
-    checkProductProperties(context)
-    val builderState = compileModulesForDistribution(context)
-
-    coroutineScope {
-      createMavenArtifactJob(context, builderState)
-
-      val entries = DistributionJARsBuilder(builderState).buildJARs(context)
-      layoutShared(context)
-      checkClassFiles(context.paths.distAllDir, context)
-      if (context.productProperties.buildSourcesArchive) {
-        buildSourcesArchive(entries, context)
-      }
-      buildOsSpecificDistributions(context)
-    }
-  }
-
   override suspend fun buildUnpackedDistribution(targetDirectory: Path, includeBinAndRuntime: Boolean) {
     val currentOs = OsFamily.currentOs
     context.paths.distAllDir = targetDirectory
-    context.options.targetOs = currentOs.osId
+    context.options.targetOs = persistentListOf(currentOs)
     context.options.buildStepsToSkip.add(BuildOptions.GENERATE_JAR_ORDER_STEP)
     BundledMavenDownloader.downloadMavenCommonLibs(context.paths.communityHomeDirRoot)
     BundledMavenDownloader.downloadMavenDistribution(context.paths.communityHomeDirRoot)
@@ -189,6 +171,23 @@ class BuildTasksImpl(context: BuildContext) : BuildTasks {
   }
 }
 
+suspend fun doRunTestBuild(context: BuildContext) {
+  checkProductProperties(context as BuildContextImpl)
+  val builderState = compileModulesForDistribution(context)
+
+  coroutineScope {
+    createMavenArtifactJob(context, builderState)
+
+    val entries = DistributionJARsBuilder(builderState).buildJARs(context)
+    layoutShared(context)
+    checkClassFiles(context.paths.distAllDir, context)
+    if (context.productProperties.buildSourcesArchive) {
+      buildSourcesArchive(entries, context)
+    }
+    buildOsSpecificDistributions(context)
+  }
+}
+
 internal data class SupportedDistribution(@JvmField val os: OsFamily, @JvmField val arch: JvmArchitecture)
 
 @JvmField
@@ -653,7 +652,7 @@ suspend fun buildDistributions(context: BuildContext) {
             if (context.buildNumber == null) {
               Span.current().addEvent("Toolbox LiteGen is not executed - it does not support SNAPSHOT build numbers")
             }
-            else if (context.options.targetOs != BuildOptions.OS_ALL) {
+            else if (context.options.targetOs != OsFamily.ALL) {
               Span.current().addEvent(
                 "Toolbox LiteGen is not executed - it doesn't support installers are being built only for specific OS")
             }
@@ -1004,6 +1003,7 @@ private fun buildCrossPlatformZip(distResults: List<DistributionForOsTaskResult>
     distFiles = context.getDistFiles(os = null, arch = null),
     extraFiles = mapOf("dependencies.txt" to dependenciesFile),
     distAllDir = context.paths.distAllDir,
+    compress = context.options.compressZipFiles,
   )
 
   checkInArchive(targetFile, "", context)
@@ -1067,9 +1067,10 @@ private fun crossPlatformZip(macX64DistDir: Path,
                              executablePatterns: List<String>,
                              distFiles: Collection<DistFile>,
                              extraFiles: Map<String, Path>,
-                             distAllDir: Path) {
+                             distAllDir: Path,
+                             compress: Boolean) {
   writeNewFile(targetFile) { outFileChannel ->
-    NoDuplicateZipArchiveOutputStream(outFileChannel).use { out ->
+    NoDuplicateZipArchiveOutputStream(outFileChannel, compress = compress).use { out ->
       out.setUseZip64(Zip64Mode.Never)
 
       out.entryToDir(winX64DistDir.resolve("bin/idea.properties"), "bin/win")
@@ -1079,9 +1080,9 @@ private fun crossPlatformZip(macX64DistDir: Path,
       out.entryToDir(macX64DistDir.resolve("bin/${executableName}.vmoptions"), "bin/mac")
       out.entry("bin/mac/${executableName}64.vmoptions", macX64DistDir.resolve("bin/${executableName}.vmoptions"))
 
-      extraFiles.forEach(BiConsumer { p, f ->
+      for ((p, f) in extraFiles) {
         out.entry(p, f)
-      })
+      }
 
       out.entry("product-info.json", productJson)
 
@@ -1145,7 +1146,7 @@ private fun crossPlatformZip(macX64DistDir: Path,
         }
       }
 
-      val commonFilter: (String) -> Boolean = { relPath: String ->
+      val commonFilter: (String) -> Boolean = { relPath ->
         !relPath.startsWith("bin/fsnotifier") &&
         !relPath.startsWith("bin/repair") &&
         !relPath.startsWith("bin/restart") &&
@@ -1155,32 +1156,39 @@ private fun crossPlatformZip(macX64DistDir: Path,
         !relPath.startsWith("help/")
       }
 
-      val zipFiles = LinkedHashMap<String, Path>()
+      val zipFileUniqueGuard = HashMap<String, Path>()
 
       out.dir(distAllDir, "", fileFilter = { _, relPath -> relPath != "bin/idea.properties" }, entryCustomizer = entryCustomizer)
 
-      out.dir(macX64DistDir, "", fileFilter = { _, relPath ->
-        commonFilter.invoke(relPath) &&
-        filterFileIfAlreadyInZip(relPath, macX64DistDir.resolve(relPath), zipFiles)
+      out.dir(macX64DistDir, "", fileFilter = { _, relativePath ->
+        commonFilter.invoke(relativePath) &&
+        filterFileIfAlreadyInZip(relativePath, macX64DistDir.resolve(relativePath), zipFileUniqueGuard)
       }, entryCustomizer = entryCustomizer)
 
       out.dir(macArm64DistDir, "", fileFilter = { _, relPath ->
         commonFilter.invoke(relPath) &&
-        filterFileIfAlreadyInZip(relPath, macArm64DistDir.resolve(relPath), zipFiles)
+        filterFileIfAlreadyInZip(relPath, macArm64DistDir.resolve(relPath), zipFileUniqueGuard)
       }, entryCustomizer = entryCustomizer)
 
       out.dir(linuxX64DistDir, "", fileFilter = { _, relPath ->
         commonFilter.invoke(relPath) &&
-        filterFileIfAlreadyInZip(relPath, linuxX64DistDir.resolve(relPath), zipFiles)
+        filterFileIfAlreadyInZip(relPath, linuxX64DistDir.resolve(relPath), zipFileUniqueGuard)
       }, entryCustomizer = entryCustomizer)
 
-      val winExcludes = distFiles.mapTo(HashSet(distFiles.size)) { "${it.relativeDir}/${it.file.fileName}" }
-      out.dir(winX64DistDir, "", fileFilter = { _, relPath ->
-        commonFilter.invoke(relPath) &&
-        !(relPath.startsWith("bin/${executableName}") && relPath.endsWith(".exe")) &&
-        !winExcludes.contains(relPath) &&
-        filterFileIfAlreadyInZip(relPath, winX64DistDir.resolve(relPath), zipFiles)
+      out.dir(startDir = winX64DistDir, prefix = "", fileFilter = { _, relativePath ->
+        commonFilter.invoke(relativePath) &&
+        !(relativePath.startsWith("bin/${executableName}") && relativePath.endsWith(".exe")) &&
+        filterFileIfAlreadyInZip(relativePath, winX64DistDir.resolve(relativePath), zipFileUniqueGuard)
       }, entryCustomizer = entryCustomizer)
+
+      for (distFile in distFiles) {
+        // linux and windows: we don't add win and linux specific dist dirs for ARM, so, copy distFiles explicitly
+        // macOS: we don't copy dist files for macOS distribution to avoid extra copy operation
+        val relativePath = "${distFile.relativeDir}/${distFile.file.fileName}"
+        if (zipFileUniqueGuard.putIfAbsent(relativePath, distFile.file) == null) {
+          out.entry(relativePath, distFile.file)
+        }
+      }
     }
   }
 }

platform/build-scripts/src/org/jetbrains/intellij/build/impl/DistributionJARsBuilder.kt

@@ -1125,7 +1125,7 @@ private suspend fun archivePlugins(items: Collection<NonBundledPlugin>, compress
               }
               else {
                 writeNewFile(target) { outFileChannel ->
-                  NoDuplicateZipArchiveOutputStream(outFileChannel).use { out ->
+                  NoDuplicateZipArchiveOutputStream(outFileChannel, compress = context.options.compressZipFiles).use { out ->
                     out.setUseZip64(Zip64Mode.Never)
                     out.dir(source, "${source.fileName}/", entryCustomizer = { entry, file, _ ->
                       if (Files.isExecutable(file)) {

platform/build-scripts/src/org/jetbrains/intellij/build/impl/MacDistributionBuilder.kt

@@ -107,6 +107,7 @@ class MacDistributionBuilder(override val context: BuildContext,
     unpackPty4jNative(context, macDistDir, "darwin")
 
     generateBuildTxt(context, macDistDir.resolve("Resources"))
+    // if copyDistFiles false, it means that we will copy dist files directly without stage dir
     if (copyDistFiles) {
       copyDistFiles(context = context, newDir = macDistDir, os = OsFamily.MACOS, arch = arch)
     }
@@ -487,7 +488,7 @@ private fun MacDistributionBuilder.buildMacZip(targetFile: Path,
       }
 
       writeNewFile(targetFile) { targetFileChannel ->
-        NoDuplicateZipArchiveOutputStream(targetFileChannel).use { zipOutStream ->
+        NoDuplicateZipArchiveOutputStream(targetFileChannel, compress = context.options.compressZipFiles).use { zipOutStream ->
           zipOutStream.setLevel(compressionLevel)
 
           zipOutStream.entry("$zipRoot/Resources/product-info.json", productJson.encodeToByteArray())

platform/build-scripts/src/org/jetbrains/intellij/build/impl/MacDmgBuilder.kt

@@ -59,7 +59,7 @@ internal suspend fun signAndBuildDmg(builder: MacDistributionBuilder,
   val targetName = context.productProperties.getBaseArtifactName(context.applicationInfo, context.buildNumber) + suffix
   val sitFile = (if (context.publishSitArchive) context.paths.artifactDir else context.paths.tempDir).resolve("$targetName.sit")
 
-  prepareMacZip(macZip, sitFile, productJson, zipRoot)
+  prepareMacZip(macZip, sitFile, productJson, zipRoot, context.options.compressZipFiles)
 
   val sign = !context.options.buildStepsToSkip.contains(BuildOptions.MAC_SIGN_STEP)
   if (!sign) {
@@ -224,14 +224,11 @@ private fun buildDmgLocally(tempDir: Path, targetFileName: String, customizer: M
 }
 
 // our zip for JARs, but here we need to support file permissions - that's why apache compress is used
-private fun prepareMacZip(macZip: Path,
-                          sitFile: Path,
-                          productJson: String,
-                          zipRoot: String) {
+private fun prepareMacZip(macZip: Path, sitFile: Path, productJson: String, zipRoot: String, compress: Boolean) {
   Files.newByteChannel(macZip, StandardOpenOption.READ).use { sourceFileChannel ->
     ZipFile(sourceFileChannel).use { zipFile ->
       writeNewFile(sitFile) { targetFileChannel ->
-        NoDuplicateZipArchiveOutputStream(targetFileChannel).use { out ->
+        NoDuplicateZipArchiveOutputStream(targetFileChannel, compress = compress).use { out ->
           // file is used only for transfer to mac builder
           out.setLevel(Deflater.BEST_SPEED)
           out.setUseZip64(Zip64Mode.Never)

platform/build-scripts/src/org/jetbrains/intellij/build/impl/WinExeInstallerBuilder.kt

@@ -6,6 +6,7 @@ import com.intellij.openapi.util.SystemInfoRt
 import com.intellij.openapi.util.io.FileUtilRt
 import com.intellij.openapi.util.io.NioFiles
 import com.intellij.util.io.Decompressor
+import io.opentelemetry.api.trace.Span
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.withContext
 import org.jetbrains.intellij.build.BuildContext
@@ -68,13 +69,13 @@ internal suspend fun buildNsisInstaller(winDistPath: Path,
                                         jreDir: Path,
                                         context: BuildContext): Path? {
   if (!SystemInfoRt.isWindows && !SystemInfoRt.isLinux) {
-    context.messages.warning("Windows installer can be built only under Windows or Linux")
+    Span.current().addEvent("Windows installer can be built only under Windows or Linux")
     return null
   }
 
   val communityHome = context.paths.communityHomeDir
   val outFileName = context.productProperties.getBaseArtifactName(context.applicationInfo, context.buildNumber) + suffix
-  context.messages.progress("Building Windows installer $outFileName")
+  Span.current().setAttribute(outFileName, outFileName)
 
   val box = context.paths.tempDir.resolve("winInstaller${suffix}")
   //noinspection SpellCheckingInspection

platform/build-scripts/src/org/jetbrains/intellij/build/impl/WindowsDistributionBuilder.kt

@@ -138,7 +138,7 @@ internal class WindowsDistributionBuilder(
                               context = context)
       }
 
-      context.executeStep(spanBuilder("build Windows Exe Installer"), BuildOptions.WINDOWS_EXE_INSTALLER_STEP) {
+      context.executeStep(spanBuilder("build Windows installer").setAttribute("arch", arch.dirName), BuildOptions.WINDOWS_EXE_INSTALLER_STEP) {
         val productJsonDir = context.paths.tempDir.resolve("win.dist.product-info.json.exe")
         validateProductJson(jsonText = generateProductJson(targetDir = productJsonDir, isJreIncluded = true, context = context),
                             relativePathToProductJson = "",
@@ -388,7 +388,13 @@ private fun CoroutineScope.createBuildWinZipTask(jreDirectoryPaths: List<Path>,
       val zipPrefix = customizer.getRootDirectoryName(context.applicationInfo, context.buildNumber)
       val dirs = listOf(context.paths.distAllDir, winDistPath, productJsonDir) + jreDirectoryPaths
 
-      zipWithCompression(targetFile = targetFile, dirs = dirs.associateWithTo(LinkedHashMap(dirs.size)) { zipPrefix })
+      val dirMap = dirs.associateWithTo(LinkedHashMap(dirs.size)) { zipPrefix }
+      if (context.options.compressZipFiles) {
+        zipWithCompression(targetFile = targetFile, dirs = dirMap)
+      }
+      else {
+        zip(targetFile = targetFile, dirs = dirMap, addDirEntriesMode = AddDirEntriesMode.NONE)
+      }
       checkInArchive(archiveFile = targetFile, pathInArchive = zipPrefix, context = context)
       context.notifyArtifactWasBuilt(targetFile)
       targetFile

platform/build-scripts/src/org/jetbrains/intellij/build/impl/archive.kt

@@ -1,4 +1,6 @@
 // Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+@file:Suppress("ConstPropertyName")
+
 package org.jetbrains.intellij.build.impl
 
 import com.intellij.openapi.util.io.FileUtilRt
@@ -7,19 +9,16 @@ import org.apache.commons.compress.archivers.ArchiveEntry
 import org.apache.commons.compress.archivers.zip.ZipArchiveEntry
 import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream
 import org.apache.commons.io.FileExistsException
-import org.apache.commons.io.IOUtils
 import org.jetbrains.intellij.build.io.readZipFile
+import org.jetbrains.intellij.build.io.unmapBuffer
+import java.nio.channels.FileChannel
 import java.nio.channels.SeekableByteChannel
-import java.nio.file.Files
-import java.nio.file.LinkOption
-import java.nio.file.NoSuchFileException
-import java.nio.file.Path
+import java.nio.file.*
 import java.nio.file.attribute.BasicFileAttributes
 import java.nio.file.attribute.FileTime
 import java.util.*
 import java.util.function.BiConsumer
 import java.util.zip.ZipEntry
-import kotlin.io.path.inputStream
 import kotlin.io.path.readText
 
 private const val fileFlag = 32768  // 0100000
@@ -27,16 +26,15 @@ private const val fileFlag = 32768  // 0100000
 const val executableFileUnixMode = fileFlag or 493  // 0755
 
 fun filterFileIfAlreadyInZip(relativePath: String, file: Path, zipFiles: MutableMap<String, Path>): Boolean {
-  val found = zipFiles.put(relativePath, file) ?: return true
-
-  if (IOUtils.contentEquals(file.inputStream(), found.inputStream())) {
+  val old = zipFiles.putIfAbsent(relativePath, file) ?: return true
+  if (compareByMemoryMappedFiles(file, old)) {
     return false
   }
 
   val file1Text = file.readText()
-  val file2Text = found.readText()
+  val file2Text = old.readText()
   val isAsciiText: (Char) -> Boolean = { it == '\t' || it == '\n' || it == '\r' || it.code in 32..126 }
-  val message = "Two files '${found}' and '${file}' with the same target path '${relativePath}' have different content"
+  val message = "Two files '${old}' and '${file}' with the same target path '${relativePath}' have different content"
   if (file1Text.take(1024).all(isAsciiText) && file2Text.take(1024).all(isAsciiText)) {
     throw RuntimeException("$message\n\nFile 1: ${"-".repeat(80)}\n$file1Text\n\nFile 2 ${"-".repeat(80)}\n$file2Text")
   }
@@ -45,6 +43,31 @@ fun filterFileIfAlreadyInZip(relativePath: String, file: Path, zipFiles: Mutable
   }
 }
 
+private fun compareByMemoryMappedFiles(path1: Path, path2: Path): Boolean {
+  FileChannel.open(path1, StandardOpenOption.READ).use { channel1 ->
+    FileChannel.open(path2, StandardOpenOption.READ).use { channel2 ->
+      val size = channel1.size()
+      if (size != channel2.size()) {
+        return false
+      }
+
+      val m1 = channel1.map(FileChannel.MapMode.READ_ONLY, 0, size)
+      try {
+        val m2 = channel2.map(FileChannel.MapMode.READ_ONLY, 0, size)
+        try {
+          return m1 == m2
+        }
+        finally {
+          unmapBuffer(m2)
+        }
+      }
+      finally {
+        unmapBuffer(m1)
+      }
+    }
+  }
+}
+
 fun consumeDataByPrefix(file: Path, prefixWithEndingSlash: String, consumer: BiConsumer<String, ByteArray>) {
   readZipFile(file) { name, entry ->
     if (name.startsWith(prefixWithEndingSlash)) {
@@ -179,9 +202,15 @@ private class ZipArchiveEntryAssertName(name: String): ZipArchiveEntry(name) {
   }
 }
 
-internal class NoDuplicateZipArchiveOutputStream(channel: SeekableByteChannel) : ZipArchiveOutputStream(channel) {
+internal class NoDuplicateZipArchiveOutputStream(channel: SeekableByteChannel, compress: Boolean) : ZipArchiveOutputStream(channel) {
   private val entries = HashSet<String>()
 
+  init {
+    if (!compress) {
+      setMethod(ZipEntry.STORED)
+    }
+  }
+
   override fun putArchiveEntry(archiveEntry: ArchiveEntry) {
     val entryName = archiveEntry.name
     assertRelativePathIsCorrectForPackaging(entryName)

platform/build-scripts/src/org/jetbrains/intellij/build/impl/distUtil.kt

@@ -52,7 +52,6 @@ internal fun generateBuildTxt(context: BuildContext, targetDirectory: Path) {
 }
 
 internal fun copyDistFiles(context: BuildContext, newDir: Path, os: OsFamily, arch: JvmArchitecture) {
-  Files.createDirectories(newDir)
   for (item in context.getDistFiles(os, arch)) {
     val dir = newDir.resolve(item.relativeDir)
     Files.createDirectories(dir)

platform/build-scripts/src/org/jetbrains/intellij/build/impl/productInfo/ProductInfoValidator.kt

@@ -3,18 +3,20 @@ package org.jetbrains.intellij.build.impl.productInfo
 
 import com.fasterxml.jackson.databind.ObjectMapper
 import com.intellij.openapi.util.io.FileUtilRt
-import com.intellij.util.lang.ImmutableZipFile
 import com.networknt.schema.JsonSchemaFactory
 import com.networknt.schema.SpecVersion
 import kotlinx.serialization.decodeFromString
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
+import org.apache.commons.compress.archivers.zip.ZipFile
 import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream
 import org.jetbrains.intellij.build.BuildContext
 import org.jetbrains.intellij.build.BuildMessages
 import org.jetbrains.intellij.build.CompilationContext
 import org.jetbrains.intellij.build.OsFamily
+import java.nio.channels.FileChannel
 import java.nio.file.Files
 import java.nio.file.Path
+import java.nio.file.StandardOpenOption
 
 /**
  * Checks that product-info.json file located in `archivePath` archive in `pathInArchive` subdirectory is correct
@@ -99,8 +101,11 @@ private fun joinPaths(parent: String, child: String): String {
 private fun archiveContainsEntry(archiveFile: Path, entryPath: String): Boolean {
   val fileName = archiveFile.fileName.toString()
   if (fileName.endsWith(".zip") || fileName.endsWith(".jar")) {
-    ImmutableZipFile.load(archiveFile).use {
-      return it.getResource(entryPath) != null
+    // don't use ImmutableZipFile - archive maybe more than 2GB
+    FileChannel.open(archiveFile, StandardOpenOption.READ).use { channel ->
+      ZipFile(channel).use {
+        return it.getEntry(entryPath) != null
+      }
     }
   }
   else if (fileName.endsWith(".tar.gz")) {
@@ -122,8 +127,11 @@ private fun archiveContainsEntry(archiveFile: Path, entryPath: String): Boolean
 private fun loadEntry(archiveFile: Path, entryPath: String): ByteArray? {
   val fileName = archiveFile.fileName.toString()
   if (fileName.endsWith(".zip") || fileName.endsWith(".jar")) {
-    ImmutableZipFile.load(archiveFile).use {
-      return it.getResource(entryPath)?.data
+    // don't use ImmutableZipFile - archive maybe more than 2GB
+    FileChannel.open(archiveFile, StandardOpenOption.READ).use { channel ->
+      ZipFile(channel).use {
+        return it.getInputStream(it.getEntry(entryPath)).readAllBytes()
+      }
     }
   }
   else if (fileName.endsWith(".tar.gz")) {
@@ -136,6 +144,9 @@ private fun loadEntry(archiveFile: Path, entryPath: String): ByteArray? {
         }
       }
     }
+    return null
+  }
+  else {
+    return null
   }
-  return null
 }

platform/build-scripts/testFramework/src/org/jetbrains/intellij/build/testFramework/buildScriptTestUtils.kt

@@ -13,6 +13,8 @@ import kotlinx.coroutines.runBlocking
 import org.jetbrains.intellij.build.*
 import org.jetbrains.intellij.build.dependencies.BuildDependenciesCommunityRoot
 import org.jetbrains.intellij.build.impl.BuildContextImpl
+import org.jetbrains.intellij.build.impl.buildDistributions
+import org.jetbrains.intellij.build.impl.doRunTestBuild
 import org.jetbrains.intellij.build.impl.logging.BuildMessagesImpl
 import org.jetbrains.intellij.build.testFramework.binaryReproducibility.BuildArtifactsReproducibilityTest
 import org.opentest4j.TestAbortedException
@@ -60,6 +62,7 @@ suspend fun createBuildContext(
 ): BuildContext {
   val options = BuildOptions()
   options.signNativeFiles = false
+  options.compressZipFiles = false
   customizeBuildOptionsForTest(options, productProperties, skipDependencySetup)
   buildOptionsCustomizer(options)
   return BuildContextImpl.createContext(communityHome = communityHomePath,
@@ -127,7 +130,7 @@ private fun testBuild(
   }
 
   runTestBuild(
-    buildContext = context,
+    context = context,
     traceSpanName = traceSpanName,
     onFinish = onFinish,
   )
@@ -135,13 +138,13 @@ private fun testBuild(
 
 // FIXME: test reproducibility
 fun runTestBuild(
-  buildContext: BuildContext,
+  context: BuildContext,
   traceSpanName: String? = null,
   onFinish: suspend (context: BuildContext) -> Unit = {},
 ) {
   initializeTracer
 
-  val productProperties = buildContext.productProperties
+  val productProperties = context.productProperties
 
   // to see in Jaeger as a one trace
   val traceFileName = "${productProperties.baseFileName}-trace.json"
@@ -150,13 +153,18 @@ fun runTestBuild(
   val spanScope = span.makeCurrent()
 
   try {
-    val outDir = buildContext.paths.buildOutputDir
+    val outDir = context.paths.buildOutputDir
     span.setAttribute("outDir", outDir.toString())
-    val messages = buildContext.messages as BuildMessagesImpl
+    val messages = context.messages as BuildMessagesImpl
     try {
       runBlocking(Dispatchers.Default) {
-        BuildTasks.create(buildContext).runTestBuild()
-        onFinish(buildContext)
+        doRunTestBuild(context)
+        if (context.options.targetOs == OsFamily.ALL) {
+          buildDistributions(context)
+        }
+        else {
+          onFinish(context)
+        }
       }
     }
     catch (e: Throwable) {