move intellij.platform.util.diff and intellij.platform.util.text.matching from util.jar to app.jar (it is not a low-level util - we can raise the language level)

GitOrigin-RevId: d5aeb967779a489ac57e16fdbb023604a3a1a825
This commit is contained in:
Vladimir Krivosheev
2023-02-11 18:12:52 +01:00
committed by intellij-monorepo-bot
parent 1ca22d5109
commit 06d24b7493
44 changed files with 1494 additions and 1603 deletions

View File

@@ -4,10 +4,7 @@ package org.jetbrains.intellij.build
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.NonCancellable
import kotlinx.coroutines.withContext
import org.jetbrains.intellij.build.impl.BaseLayout
import org.jetbrains.intellij.build.impl.JarPackager
import org.jetbrains.intellij.build.impl.LibraryPackMode
import org.jetbrains.intellij.build.impl.buildJar
import org.jetbrains.intellij.build.impl.*
import org.jetbrains.intellij.build.io.deleteDir
import org.jetbrains.intellij.build.io.zipWithCompression
import java.nio.file.Files
@@ -20,19 +17,18 @@ suspend fun buildCommunityStandaloneJpsBuilder(targetDir: Path,
context: BuildContext,
dryRun: Boolean = false,
layoutCustomizer: ((BaseLayout) -> Unit) = {}) {
val layout = BaseLayout()
val layout = PlatformLayout()
layout.withModules(listOf(
"intellij.platform.util",
"intellij.platform.util.classLoader",
"intellij.platform.util.text.matching",
"intellij.platform.util.base",
"intellij.platform.util.xmlDom",
"intellij.platform.util.jdom",
"intellij.platform.tracing.rt",
"intellij.platform.util.diff",
"intellij.platform.util.rt.java8",
), "util.jar")
).map { ModuleItem(moduleName = it, relativeOutputFile = "util.jar", reason = null) })
layout.withModule("intellij.platform.util.rt", "util_rt.jar")
layout.withModule("intellij.platform.jps.build.launcher", "jps-launcher.jar")
@@ -41,7 +37,7 @@ suspend fun buildCommunityStandaloneJpsBuilder(targetDir: Path,
"intellij.platform.jps.model",
"intellij.platform.jps.model.impl",
"intellij.platform.jps.model.serialization",
), "jps-model.jar")
).map { ModuleItem(moduleName = it, relativeOutputFile = "jps-model.jar", reason = null) })
layout.withModules(listOf(
"intellij.java.guiForms.rt",
@@ -50,7 +46,7 @@ suspend fun buildCommunityStandaloneJpsBuilder(targetDir: Path,
"intellij.java.compiler.instrumentationUtil.java8",
"intellij.platform.jps.build",
"intellij.tools.jps.build.standalone",
), "jps-builders.jar")
).map { ModuleItem(moduleName = it, relativeOutputFile = "jps-builders.jar", reason = null) })
layout.withModule("intellij.java.rt", "idea_rt.jar")
layout.withModule("intellij.platform.jps.build.javac.rt", "jps-builders-6.jar")
@@ -99,7 +95,7 @@ suspend fun buildCommunityStandaloneJpsBuilder(targetDir: Path,
Files.createTempDirectory(targetDir, "jps-standalone-community-")
}
try {
JarPackager.pack(jarToModules = layout.jarToModules, outputDir = tempDir, context = context, layout = layout, dryRun = dryRun)
JarPackager.pack(includedModules = layout.includedModules, outputDir = tempDir, context = context, layout = layout, dryRun = dryRun)
val targetFile = targetDir.resolve("standalone-jps-$buildNumber.zip")
withContext(Dispatchers.IO) {

View File

@@ -3,7 +3,6 @@ package org.jetbrains.intellij.build
import kotlinx.collections.immutable.persistentListOf
import org.jetbrains.intellij.build.dependencies.BuildDependenciesCommunityRoot
import org.jetbrains.intellij.build.impl.BaseLayout
import org.jetbrains.intellij.build.impl.BuildContextImpl
import org.jetbrains.intellij.build.kotlin.KotlinBinaries
@@ -44,8 +43,7 @@ open class IdeaCommunityProperties(private val communityHomeDir: Path) : BaseIde
useSplash = true
buildCrossPlatformDistribution = true
productLayout.productImplementationModules = listOf("intellij.platform.main")
productLayout.withAdditionalPlatformJar(BaseLayout.APP_JAR, "intellij.idea.community.resources")
productLayout.productImplementationModules = listOf("intellij.platform.main", "intellij.idea.community.resources")
productLayout.bundledPluginModules = IDEA_BUNDLED_PLUGINS
.add("intellij.javaFX.community")
.toMutableList()

View File

@@ -1,5 +1,5 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("ReplacePutWithAssignment", "ReplaceGetOrSet")
@file:Suppress("ReplacePutWithAssignment", "ReplaceGetOrSet", "RAW_RUN_BLOCKING")
package org.jetbrains.intellij.build.io
@@ -68,11 +68,12 @@ suspend fun runJava(mainClass: String,
fun javaRunFailed(reason: String) {
span.setAttribute("classPath", classPathStringBuilder.substring("-classpath".length))
span.setAttribute("processArgs", processArgs.joinToString(separator = " "))
span.setAttribute("output", runCatching { Files.readString(outputFile) }.getOrNull() ?: "output file doesn't exist")
span.setAttribute("errorOutput",
runCatching { Files.readString(errorOutputFile) }.getOrNull() ?: "error output file doesn't exist")
onError?.invoke()
throw RuntimeException("Cannot execute $mainClass: $reason")
throw RuntimeException("Cannot execute $mainClass: $reason\n${processArgs.joinToString(separator = " ")}")
}
try {
@@ -147,7 +148,6 @@ private fun createProcessArgs(javaExe: Path,
mainClass: String,
args: List<String>): MutableList<String> {
val processArgs = mutableListOf<String>()
// FIXME: enforce JBR
processArgs.add(javaExe.toString())
processArgs.add("-Djava.awt.headless=true")
processArgs.add("-Dapple.awt.UIElement=true")
@@ -281,8 +281,6 @@ private fun appendArg(value: String, builder: StringBuilder) {
}
}
class ProcessRunTimedOut(message: String) : RuntimeException(message)
internal suspend fun dumpThreads(pid: Long) {
val jstack = System.getenv("JAVA_HOME")
?.removeSuffix("/")

View File

@@ -22,7 +22,7 @@ fun zipWithCompression(targetFile: Path,
fileFilter: ((name: String) -> Boolean)? = null) {
Files.createDirectories(targetFile.parent)
ZipFileWriter(channel = FileChannel.open(targetFile, if (overwrite) W_OVERWRITE else W_CREATE_NEW),
deflater = Deflater(compressionLevel, true)).use { zipFileWriter ->
deflater = if (compressionLevel == Deflater.NO_COMPRESSION) null else Deflater(compressionLevel, true)).use { zipFileWriter ->
if (addDirEntriesMode == AddDirEntriesMode.NONE) {
doArchive(zipFileWriter = zipFileWriter, fileAdded = fileFilter, dirs = dirs)
}
@@ -50,7 +50,7 @@ fun zipWithCompression(targetFile: Path,
}
}
// symlinks not supported but can be easily implemented - see CollectingVisitor.visitFile
// symlinks are not supported but can be easily implemented - see CollectingVisitor.visitFile
fun zip(targetFile: Path,
dirs: Map<Path, String>,
addDirEntriesMode: AddDirEntriesMode = AddDirEntriesMode.RESOURCE_ONLY,

View File

@@ -22,21 +22,17 @@ sealed interface Source {
private val USER_HOME = Path.of(System.getProperty("user.home"))
private val MAVEN_REPO = USER_HOME.resolve(".m2/repository")
data class ZipSource(val file: Path,
val excludes: List<Regex> = emptyList(),
data class ZipSource(@JvmField val file: Path,
@JvmField val excludes: List<Regex> = emptyList(),
override val filter: ((String) -> Boolean)? = null,
override val sizeConsumer: IntConsumer? = null) : Source, Comparable<ZipSource> {
override fun compareTo(other: ZipSource) = file.compareTo(other.file)
override fun toString(): String {
val shortPath = if (file.startsWith(MAVEN_REPO)) {
MAVEN_REPO.relativize(file).toString()
}
else if (file.startsWith(USER_HOME)) {
"~/" + USER_HOME.relativize(file)
}
else {
file.toString()
val shortPath = when {
file.startsWith(MAVEN_REPO) -> MAVEN_REPO.relativize(file).toString()
file.startsWith(USER_HOME) -> "~/" + USER_HOME.relativize(file)
else -> file.toString()
}
return "zip(file=$shortPath)"
}
@@ -47,12 +43,7 @@ data class DirSource(@JvmField val dir: Path,
override val sizeConsumer: IntConsumer? = null,
@JvmField val prefix: String = "") : Source {
override fun toString(): String {
val shortPath = if (dir.startsWith(USER_HOME)) {
"~/" + USER_HOME.relativize(dir)
}
else {
dir.toString()
}
val shortPath = if (dir.startsWith(USER_HOME)) "~/${USER_HOME.relativize(dir)}" else dir.toString()
return "dir(dir=$shortPath, excludes=${excludes.size})"
}
}
@@ -67,9 +58,7 @@ data class InMemoryContentSource(@JvmField val relativePath: String,
if (relativePath != other.relativePath) return false
if (!data.contentEquals(other.data)) return false
if (sizeConsumer != other.sizeConsumer) return false
return true
return sizeConsumer == other.sizeConsumer
}
override fun hashCode(): Int {
@@ -240,6 +229,7 @@ private fun checkName(name: String,
excludes: List<Regex>,
includeManifest: Boolean,
requiresMavenFiles: Boolean): Boolean {
@Suppress("SpellCheckingInspection")
return !ignoredNames.contains(name) &&
excludes.none { it.matches(name) } &&
!name.endsWith(".kotlin_metadata") &&
@@ -248,6 +238,35 @@ private fun checkName(name: String,
!name.startsWith("META-INF/license/") &&
!name.startsWith("META-INF/LICENSE-") &&
!name.startsWith("native-image/") &&
// Class 'jakarta.json.JsonValue' not found while looking for field 'jakarta.json.JsonValue NULL'
//!name.startsWith("com/jayway/jsonpath/spi/json/JakartaJsonProvider") &&
//!name.startsWith("com/jayway/jsonpath/spi/json/JsonOrgJsonProvider") &&
//!name.startsWith("com/jayway/jsonpath/spi/json/TapestryJsonProvider") &&
//
//!name.startsWith("com/jayway/jsonpath/spi/mapper/JakartaMappingProvider") &&
//!name.startsWith("com/jayway/jsonpath/spi/mapper/JsonOrgMappingProvider") &&
//!name.startsWith("com/jayway/jsonpath/spi/mapper/TapestryMappingProvider") &&
//!name.startsWith("io/opentelemetry/exporter/internal/grpc/") &&
//!name.startsWith("io/opentelemetry/exporter/internal/okhttp/") &&
//// com.thaiopensource.datatype.xsd.regex.xerces2 is used instead
//!name.startsWith("com/thaiopensource/datatype/xsd/regex/xerces/RegexEngineImpl") &&
//!name.startsWith("com/thaiopensource/relaxng/util/JingTask") &&
//!name.startsWith("com/thaiopensource/validate/schematron") &&
//!name.startsWith("com/thoughtworks/xstream/core/util/ISO8601JodaTimeConverter") &&
//!name.startsWith("com/thoughtworks/xstream/io/xml/BEAStaxDriver") &&
//!name.startsWith("com/thoughtworks/xstream/io/xml/AbstractXppDomDriver") &&
//!name.startsWith("com/thoughtworks/xstream/io/xml/Xom") &&
//!name.startsWith("com/thoughtworks/xstream/io/xml/Dom4") &&
//!name.startsWith("com/thoughtworks/xstream/io/xml/JDom2") &&
//!name.startsWith("com/thoughtworks/xstream/io/xml/KXml2") &&
//!name.startsWith("com/thoughtworks/xstream/io/xml/Wstx") &&
//!name.startsWith("com/thoughtworks/xstream/io/xml/Xpp3") &&
//!name.startsWith("com/thoughtworks/xstream/io/xml/xppdom") &&
//!name.startsWith("com/thoughtworks/xstream/io/xml/XppDom") &&
//!name.startsWith("com/michaelbaranov/microba/jgrpah/birdview/Birdview") &&
!name.startsWith("native/") &&
!name.startsWith("licenses/") &&
(requiresMavenFiles || (name != "META-INF/maven" && !name.startsWith("META-INF/maven/"))) &&

View File

@@ -34,11 +34,9 @@ private val sourceToNames: Map<String, MutableList<String>> by lazy {
val sourceToNames = LinkedHashMap<String, MutableList<String>>()
getClassLoadingLog().bufferedReader().forEachLine {
val data = it.split(':', limit = 2)
val sourcePath = data[1]
// main jar is scrambled - doesn't make sense to reorder it
if (sourcePath != "lib/idea.jar") {
sourceToNames.computeIfAbsent(sourcePath) { mutableListOf() }.add(data[0])
}
val sourcePath = data.get(1)
// the main jar is scrambled - doesn't make sense to reorder it
sourceToNames.computeIfAbsent(sourcePath) { mutableListOf() }.add(data.get(0))
}
sourceToNames
}
@@ -53,13 +51,12 @@ fun reorderJar(relativePath: String, file: Path) {
}
}
fun generateClasspath(homeDir: Path, mainJarName: String, antTargetFile: Path?): PersistentList<String> {
fun generateClasspath(homeDir: Path, antTargetFile: Path?): PersistentList<String> {
val libDir = homeDir.resolve("lib")
val appFile = libDir.resolve("app.jar")
tracer.spanBuilder("generate app.jar")
.setAttribute("dir", homeDir.toString())
.setAttribute("mainJarName", mainJarName)
.use {
transformFile(appFile) { target ->
writeNewZip(target) { zipCreator ->
@@ -68,15 +65,6 @@ fun generateClasspath(homeDir: Path, mainJarName: String, antTargetFile: Path?):
entryName != "module-info.class"
}
val mainJar = libDir.resolve(mainJarName)
if (Files.exists(mainJar)) {
// no such file in community (no closed sources)
copyZipRaw(mainJar, packageIndexBuilder, zipCreator) { entryName ->
entryName != "module-info.class"
}
Files.delete(mainJar)
}
// packing to product.jar maybe disabled
val productJar = libDir.resolve("product.jar")
if (Files.exists(productJar)) {
@@ -105,7 +93,6 @@ fun generateClasspath(homeDir: Path, mainJarName: String, antTargetFile: Path?):
val sourceToNames = readClassLoadingLog(
classLoadingLog = PackageIndexBuilder::class.java.classLoader.getResourceAsStream("$classifier/class-report.txt")!!,
rootDir = homeDir,
mainJarName = mainJarName
)
val files = computeAppClassPath(sourceToNames, libDir)
if (antTargetFile != null) {
@@ -137,14 +124,11 @@ private inline fun addJarsFromDir(dir: Path, consumer: (Sequence<Path>) -> Unit)
}
}
internal fun readClassLoadingLog(classLoadingLog: InputStream, rootDir: Path, mainJarName: String): Map<Path, List<String>> {
internal fun readClassLoadingLog(classLoadingLog: InputStream, rootDir: Path): Map<Path, List<String>> {
val sourceToNames = LinkedHashMap<Path, MutableList<String>>()
classLoadingLog.bufferedReader().forEachLine {
val data = it.split(':', limit = 2)
var sourcePath = data[1]
if (sourcePath == "lib/idea.jar") {
sourcePath = "lib/$mainJarName"
}
val sourcePath = data[1]
sourceToNames.computeIfAbsent(rootDir.resolve(sourcePath)) { mutableListOf() }.add(data[0])
}
return sourceToNames
@@ -171,7 +155,7 @@ fun reorderJar(jarFile: Path, orderedNames: List<String>, resultJarFile: Path):
readZipEntries(sourceBuffer, fileSize) { name, entry ->
entries.add(EntryData(name, entry))
}
// ignore existing package index on reorder - a new one will be computed even if it is the same, do not optimize for simplicity
// ignore the existing package index on reorder - a new one will be computed even if it is the same, do not optimize for simplicity
entries.sortWith(Comparator { o1, o2 ->
val o2p = o2.name
if ("META-INF/plugin.xml" == o2p) {

View File

@@ -77,7 +77,7 @@ class ReorderJarsTest {
Files.createDirectories(tempDir)
runBlocking {
doReorderJars(readClassLoadingLog(path.resolve("order.txt").inputStream(), path, "idea.jar"), path, tempDir)
doReorderJars(readClassLoadingLog(path.resolve("order.txt").inputStream(), path), path, tempDir)
}
val files = tempDir.toFile().listFiles()!!
assertThat(files).isNotNull()
@@ -102,7 +102,7 @@ class ReorderJarsTest {
val path = testDataPath
runBlocking {
doReorderJars(readClassLoadingLog(path.resolve("zkmOrder.txt").inputStream(), path, "idea.jar"), path, tempDir)
doReorderJars(readClassLoadingLog(path.resolve("zkmOrder.txt").inputStream(), path), path, tempDir)
}
val files = tempDir.toFile().listFiles()!!
assertThat(files).isNotNull()

View File

@@ -1,15 +1,16 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("ReplaceGetOrSet", "ReplacePutWithAssignment")
@file:Suppress("ReplaceGetOrSet", "ReplacePutWithAssignment", "LiftReturnOrAssignment")
package org.jetbrains.intellij.build.devServer
import kotlinx.coroutines.CompletableDeferred
import kotlinx.coroutines.Deferred
import kotlinx.coroutines.ExperimentalCoroutinesApi
import com.intellij.diagnostic.telemetry.useWithScope2
import kotlinx.coroutines.*
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
import kotlinx.serialization.json.Json
import org.jetbrains.intellij.build.BuildOptions
import org.jetbrains.intellij.build.TraceManager
import org.jetbrains.intellij.build.closeKtorClient
import java.nio.file.Files
import java.nio.file.Path
import java.util.concurrent.ConcurrentHashMap
@@ -22,24 +23,40 @@ internal data class ProductConfiguration(@JvmField val modules: List<String>, @J
private const val PRODUCTS_PROPERTIES_PATH = "build/dev-build.json"
internal class BuildServer(homePath: Path, productionClassOutput: Path) {
private val configuration: Configuration
private val platformPrefixToPluginBuilder = ConcurrentHashMap<String, Deferred<IdeBuilder>>()
init {
// for compatibility with local runs and runs on CI
System.setProperty(BuildOptions.PROJECT_CLASSES_OUTPUT_DIRECTORY_PROPERTY, productionClassOutput.parent.toString())
val jsonFormat = Json { isLenient = true }
configuration = jsonFormat.decodeFromString(Configuration.serializer(), Files.readString(homePath.resolve(PRODUCTS_PROPERTIES_PATH)))
}
// not synchronized version
suspend fun buildProductInProcess(isServerMode: Boolean, request: BuildRequest): IdeBuilder {
suspend fun buildProductInProcess(request: BuildRequest) {
TraceManager.spanBuilder("build ide").setAttribute("request", request.toString()).useWithScope2 {
val platformPrefix = request.platformPrefix
return buildProduct(productConfiguration = getProductConfiguration(platformPrefix), request = request, isServerMode = isServerMode)
val configuration = createConfiguration(homePath = request.homePath, productionClassOutput = request.productionClassOutput)
val productConfiguration = getProductConfiguration(configuration, platformPrefix)
try {
buildProduct(productConfiguration = productConfiguration, request = request, isServerMode = false)
}
finally {
// otherwise, a thread leak in tests
if (!request.keepHttpClient) {
withContext(NonCancellable) {
closeKtorClient()
}
}
}
}
}
private fun createConfiguration(productionClassOutput: Path, homePath: Path): Configuration {
// for compatibility with local runs and runs on CI
System.setProperty(BuildOptions.PROJECT_CLASSES_OUTPUT_DIRECTORY_PROPERTY, productionClassOutput.parent.toString())
return Json.decodeFromString(Configuration.serializer(), Files.readString(homePath.resolve(PRODUCTS_PROPERTIES_PATH)))
}
private fun getProductConfiguration(configuration: Configuration, platformPrefix: String): ProductConfiguration {
return configuration.products.get(platformPrefix) ?: throw ConfigurationException(
"No production configuration for platform prefix `$platformPrefix` please add to `$PRODUCTS_PROPERTIES_PATH` if needed"
)
}
internal class BuildServer(homePath: Path, productionClassOutput: Path) {
private val configuration = createConfiguration(productionClassOutput, homePath)
private val platformPrefixToPluginBuilder = ConcurrentHashMap<String, Deferred<IdeBuilder>>()
suspend fun checkOrCreateIdeBuilder(request: BuildRequest): IdeBuilder {
platformPrefixToPluginBuilder.get(request.platformPrefix)?.let {
@@ -53,30 +70,27 @@ internal class BuildServer(homePath: Path, productionClassOutput: Path) {
}
try {
return buildProductInProcess(isServerMode = true, request = request)
val platformPrefix = request.platformPrefix
return buildProduct(productConfiguration = getProductConfiguration(configuration, platformPrefix),
request = request,
isServerMode = true)
}
catch (e: Throwable) {
ideBuilderDeferred.completeExceptionally(e)
throw e
}
}
}
private fun getProductConfiguration(platformPrefix: String): ProductConfiguration {
return configuration.products.get(platformPrefix) ?: throw ConfigurationException(
"No production configuration for platform prefix `$platformPrefix` please add to `$PRODUCTS_PROPERTIES_PATH` if needed"
)
@OptIn(ExperimentalCoroutinesApi::class)
private suspend fun checkChangesIfNeeded(ideBuilderDeferred: Deferred<IdeBuilder>): IdeBuilder {
if (ideBuilderDeferred.isActive) {
return ideBuilderDeferred.await()
}
@OptIn(ExperimentalCoroutinesApi::class)
private suspend fun checkChangesIfNeeded(ideBuilderDeferred: Deferred<IdeBuilder>): IdeBuilder {
if (ideBuilderDeferred.isActive) {
return ideBuilderDeferred.await()
}
else {
val ideBuilder = ideBuilderDeferred.getCompleted()
ideBuilder.checkChanged()
return ideBuilder
}
else {
val ideBuilder = ideBuilderDeferred.getCompleted()
ideBuilder.checkChanged()
return ideBuilder
}
}

View File

@@ -1,13 +1,12 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("RAW_RUN_BLOCKING")
package org.jetbrains.intellij.build.devServer
import com.intellij.diagnostic.telemetry.useWithScope2
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.runBlocking
import org.jetbrains.intellij.build.ConsoleSpanExporter
import org.jetbrains.intellij.build.TraceManager.spanBuilder
import org.jetbrains.intellij.build.TracerProviderManager
import org.jetbrains.intellij.build.closeKtorClient
object DevIdeaBuilder {
@JvmStatic
@@ -28,15 +27,4 @@ object DevIdeaBuilder {
}
}
}
}
suspend fun buildProductInProcess(request: BuildRequest) {
spanBuilder("build ide").setAttribute("request", request.toString()).useWithScope2 {
BuildServer(homePath = request.homePath, productionClassOutput = request.productionClassOutput)
.buildProductInProcess(isServerMode = false, request = request)
// otherwise, thread leak in tests
if (!request.keepHttpClient) {
closeKtorClient()
}
}
}

View File

@@ -19,8 +19,6 @@ import org.jetbrains.intellij.build.impl.*
import org.jetbrains.intellij.build.impl.projectStructureMapping.LibraryFileEntry
import org.jetbrains.intellij.build.impl.projectStructureMapping.ModuleOutputEntry
import org.jetbrains.jps.model.artifact.JpsArtifactService
import org.jetbrains.jps.model.library.JpsOrderRootType
import org.jetbrains.jps.util.JpsPathUtil
import org.jetbrains.xxh3.Xx3UnencodedString
import java.lang.invoke.MethodHandles
import java.lang.invoke.MethodType
@@ -77,7 +75,7 @@ internal class IdeBuilder(internal val pluginBuilder: PluginBuilder,
internal suspend fun buildProduct(productConfiguration: ProductConfiguration, request: BuildRequest, isServerMode: Boolean): IdeBuilder {
val runDir = withContext(Dispatchers.IO) {
var rootDir = request.homePath.resolve("out/dev-run")
// if symlinked to ram disk, use real path for performance reasons and avoid any issues in ant/other code
// if symlinked to ram disk, use a real path for performance reasons and avoid any issues in ant/other code
if (Files.exists(rootDir)) {
// toRealPath must be called only on existing file
rootDir = rootDir.toRealPath()
@@ -85,7 +83,7 @@ internal suspend fun buildProduct(productConfiguration: ProductConfiguration, re
val classifier = if (request.isIdeProfileAware) computeAdditionalModulesFingerprint(request.additionalModules) else ""
val runDir = rootDir.resolve((if (request.platformPrefix == "Idea") "idea-community" else request.platformPrefix) + classifier)
// on start delete everything to avoid stale data
// on start, delete everything to avoid stale data
if (Files.isDirectory(runDir)) {
val usePluginCache = spanBuilder("check plugin cache applicability").useWithScope2 {
checkBuildModulesModificationAndMark(productConfiguration, request.productionClassOutput)
@@ -114,7 +112,9 @@ internal suspend fun buildProduct(productConfiguration: ProductConfiguration, re
}
// remove all modules without content root
val modules = plugin.includedModuleNames
val modules = plugin.includedModules.asSequence()
.map { it.moduleName }
.distinct()
.filter { it == plugin.mainModule || !context.findRequiredModule(it).contentRootsList.urls.isEmpty() }
.toList()
val pluginBuildDescriptor = PluginBuildDescriptor(dir = pluginRootDir.resolve(plugin.directoryName),
@@ -170,10 +170,12 @@ private suspend fun createBuildContext(productConfiguration: ProductConfiguratio
}
}
BuildContextImpl.createContext(compilationContext = compilationContext.await(),
projectHome = request.homePath,
productProperties = productProperties.await()
)
BuildContextImpl(compilationContext = compilationContext.await(),
productProperties = productProperties.await(),
windowsDistributionCustomizer = null,
linuxDistributionCustomizer = null,
macDistributionCustomizer = null,
proprietaryBuildTools = ProprietaryBuildTools.DUMMY)
}
}
@@ -264,7 +266,7 @@ private suspend fun createLibClassPath(homePath: Path, context: BuildContext): S
platform = platformLayout,
context = context,
copyFiles = isPackagedLib)
// for some reasons maybe duplicated paths - use set
// for some reasons, maybe duplicated paths - use set
val classPath = LinkedHashSet<String>()
if (isPackagedLib) {
projectStructureMapping.mapTo(classPath) { it.path.toString() }
@@ -291,11 +293,6 @@ private suspend fun createLibClassPath(homePath: Path, context: BuildContext): S
else -> throw UnsupportedOperationException("Entry $entry is not supported")
}
}
for (libName in platformLayout.projectLibrariesToUnpack.values()) {
val library = context.project.libraryCollection.findLibrary(libName) ?: throw IllegalStateException("Cannot find library $libName")
library.getRootUrls(JpsOrderRootType.COMPILED).mapTo(classPath, JpsPathUtil::urlToPath)
}
}
val projectLibDir = homePath.resolve("lib")
@@ -374,6 +371,7 @@ private fun getCommunityHomePath(homePath: Path): BuildDependenciesCommunityRoot
private fun createBuildOptions(runDir: Path): BuildOptions {
val options = BuildOptions()
options.printFreeSpace = false
options.validateImplicitPlatformModule = false
options.useCompiledClassesFromProjectOutput = true
options.targetOs = persistentListOf()
options.cleanOutputFolder = false

View File

@@ -123,7 +123,7 @@ internal class PluginBuilder(private val outDir: Path,
if (mainModule != "intellij.platform.builtInHelp") {
checkOutputOfPluginModules(mainPluginModule = mainModule,
jarToModules = plugin.layout.jarToModules,
includedModules = plugin.layout.includedModules,
moduleExcludes = plugin.layout.moduleExcludes,
context = context)
}
@@ -139,7 +139,7 @@ internal class PluginBuilder(private val outDir: Path,
layoutDistribution(layout = plugin.layout,
targetDirectory = plugin.dir,
moduleOutputPatcher = moduleOutputPatcher,
jarToModule = plugin.layout.jarToModules,
includedModules = plugin.layout.includedModules,
context = context)
withContext(Dispatchers.IO) {
plugin.markAsBuilt(outDir)

View File

@@ -4,23 +4,14 @@
package org.jetbrains.intellij.build
import kotlinx.collections.immutable.*
import org.jetbrains.intellij.build.impl.BaseLayout
import org.jetbrains.intellij.build.impl.LibraryPackMode
import org.jetbrains.intellij.build.impl.PlatformLayout
import org.jetbrains.intellij.build.impl.TEST_FRAMEWORK_JAR
import org.jetbrains.intellij.build.kotlin.KotlinPluginBuilder
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardCopyOption
private val JAVA_IDE_API_MODULES = java.util.List.of(
"intellij.xml.dom",
"intellij.jsp.base"
)
private val JAVA_IDE_IMPLEMENTATION_MODULES = java.util.List.of(
"intellij.xml.dom.impl",
"intellij.tools.testsBootstrap"
)
private val BASE_CLASS_VERSIONS = persistentHashMapOf(
"" to "17",
"lib/idea_rt.jar" to "1.7",
@@ -124,6 +115,19 @@ val CE_CLASS_VERSIONS: PersistentMap<String, String> = BASE_CLASS_VERSIONS.putAl
"plugins/Groovy/lib/groovy-constants-rt.jar" to "1.7",
))
val TEST_FRAMEWORK_WITH_JAVA_RT: (PlatformLayout, BuildContext) -> Unit = { layout, _ ->
for (name in listOf("intellij.platform.testFramework.common",
"intellij.platform.testFramework.junit5",
"intellij.platform.testFramework",
"intellij.platform.testFramework.core",
"intellij.platform.testFramework.impl",
"intellij.tools.testsBootstrap",
"intellij.java.rt")) {
layout.withModule(name, "testFramework.jar")
}
}
/**
* Base class for all editions of IntelliJ IDEA
*/
@@ -132,15 +136,11 @@ abstract class BaseIdeaProperties : ProductProperties() {
@Suppress("LeakingThis")
configureJetBrainsProduct(this)
productLayout.mainJarName = "idea.jar"
productLayout.withAdditionalPlatformJar(BaseLayout.APP_JAR, "intellij.java.ide.resources")
productLayout.addPlatformSpec { layout, _ ->
for (name in JAVA_IDE_API_MODULES) {
if (!productLayout.productApiModules.contains(name)) {
layout.withModule(name)
}
layout.withModule("intellij.java.ide.resources")
if (!productLayout.productApiModules.contains("intellij.jsp.base")) {
layout.withModule("intellij.jsp.base")
}
for (moduleName in arrayOf(
"intellij.java.testFramework",
@@ -150,14 +150,10 @@ abstract class BaseIdeaProperties : ProductProperties() {
"intellij.platform.testFramework.junit5",
"intellij.platform.testFramework",
"intellij.platform.uast.tests",
"intellij.tools.testsBootstrap",
)) {
if (!productLayout.productApiModules.contains(moduleName)) {
layout.withModule(moduleName, "testFramework.jar")
}
}
for (name in JAVA_IDE_IMPLEMENTATION_MODULES) {
if (!productLayout.productImplementationModules.contains(name)) {
layout.withModule(name)
if (!productLayout.productApiModules.contains(moduleName) && !productLayout.productImplementationModules.contains(moduleName)) {
layout.withModule(moduleName, TEST_FRAMEWORK_JAR)
}
}
//todo currently intellij.platform.testFramework included into idea.jar depends on this jar so it cannot be moved to java plugin

View File

@@ -82,6 +82,8 @@ interface BuildContext : CompilationContext {
fun findFileInModuleSources(moduleName: String, relativePath: String): Path?
fun findFileInModuleSources(module: JpsModule, relativePath: String): Path?
suspend fun signFiles(files: List<Path>, options: PersistentMap<String, String> = persistentMapOf()) {
proprietaryBuildTools.signTool.signFiles(files = files, context = this, options = options)
}

View File

@@ -288,7 +288,11 @@ class BuildOptions {
var printEnvironmentInfo = SystemProperties.getBooleanProperty("intellij.print.environment", false)
@Internal
var printFreeSpace = true
@JvmField
var printFreeSpace: Boolean = true
@Internal
@JvmField
var validateImplicitPlatformModule: Boolean = true
/**
* Specifies list of names of directories of bundled plugins which shouldn't be included into the product distribution. This option can be

View File

@@ -1,4 +1,6 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("LiftReturnOrAssignment")
package org.jetbrains.intellij.build
import kotlinx.collections.immutable.PersistentList
@@ -313,15 +315,16 @@ object CommunityRepositoryModules {
// modules:
// design-tools.jar
spec.withModule("intellij.android.compose-designer", "design-tools.jar")
spec.withModule("intellij.android.design-plugin", "design-tools.jar")
spec.withModule("intellij.android.compose-designer")
if (mainModuleName != "intellij.android.design-plugin") {
spec.withModule("intellij.android.design-plugin")
}
@Suppress("SpellCheckingInspection")
spec.withModule("intellij.android.designer.customview", "design-tools.jar")
spec.withModule("intellij.android.designer", "design-tools.jar")
spec.withModule("intellij.android.glance-designer", "design-tools.jar")
spec.withModule("intellij.android.layoutlib", "design-tools.jar")
spec.withModule("intellij.android.nav.editor", "design-tools.jar")
spec.withModule("intellij.android.designer.customview")
spec.withModule("intellij.android.designer")
spec.withModule("intellij.android.glance-designer")
spec.withModule("intellij.android.layoutlib")
spec.withModule("intellij.android.nav.editor")
// libs:
spec.withProjectLibrary("layoutlib")
@@ -435,7 +438,9 @@ object CommunityRepositoryModules {
spec.withModule("intellij.android.newProjectWizard", "android.jar")
spec.withModule("intellij.android.observable.ui", "android.jar")
spec.withModule("intellij.android.observable", "android.jar")
spec.withModule("intellij.android.plugin", "android.jar")
if (mainModuleName != "intellij.android.plugin") {
spec.withModule("intellij.android.plugin", "android.jar")
}
spec.withModule("intellij.android.profilersAndroid", "android.jar")
spec.withModule("intellij.android.projectSystem.gradle.models", "android.jar")
spec.withModule("intellij.android.projectSystem.gradle.psd", "android.jar")

View File

@@ -20,18 +20,13 @@ val DEFAULT_BUNDLED_PLUGINS: PersistentList<String> = persistentListOf(
)
class ProductModulesLayout {
/**
* Name of the main product JAR file. Outputs of {@link #productImplementationModules} will be packed into it.
*/
lateinit var mainJarName: String
/**
* Names of the additional product-specific modules which need to be packed into openapi.jar in the product's 'lib' directory.
*/
var productApiModules: List<String> = emptyList()
/**
* Names of the additional product-specific modules which need to be included into {@link #mainJarName} in the product's 'lib' directory
* Names of the additional product-specific modules which need to be included in the product's 'lib' directory
*/
var productImplementationModules: List<String> = emptyList()
@@ -74,16 +69,6 @@ class ProductModulesLayout {
field = value
}
/**
* Names of the project libraries which JARs' contents should be extracted into {@link #mainJarName} JAR.
*/
var projectLibrariesToUnpackIntoMainJar: PersistentList<String> = persistentListOf()
/**
* Maps names of JARs to names of the modules; these modules will be packed into these JARs and copied to the product's 'lib' directory.
*/
val additionalPlatformJars: MultiMap<String, String> = MultiMap.createLinkedSet()
/**
* Module name to list of Ant-like patterns describing entries which should be excluded from its output.
* <strong>This is a temporary property added to keep layout of some products. If some directory from a module shouldn't be included into the
@@ -94,16 +79,16 @@ class ProductModulesLayout {
/**
* Additional customizations of platform JARs. **This is a temporary property added to keep layout of some products.**
*/
internal var platformLayoutSpec = persistentListOf<(PlatformLayout.Spec, BuildContext) -> Unit>()
internal var platformLayoutSpec = persistentListOf<(PlatformLayout, BuildContext) -> Unit>()
@Deprecated("PlatformLayout should be immutable", replaceWith = ReplaceWith("addPlatformSpec"))
fun addPlatformCustomizer(customizer: BiConsumer<PlatformLayout, BuildContext>) {
platformLayoutSpec = platformLayoutSpec.add { spec, context ->
customizer.accept(spec.layout, context)
platformLayoutSpec = platformLayoutSpec.add { layout, context ->
customizer.accept(layout, context)
}
}
fun addPlatformSpec(customizer: (PlatformLayout.Spec, BuildContext) -> Unit) {
fun addPlatformSpec(customizer: (PlatformLayout, BuildContext) -> Unit) {
platformLayoutSpec = platformLayoutSpec.add(customizer)
}
@@ -159,20 +144,9 @@ class ProductModulesLayout {
result.addAll(enabledPluginModules)
pluginLayouts.asSequence()
.filter { enabledPluginModules.contains(it.mainModule) }
.flatMapTo(result) { it.includedModuleNames }
.flatMapTo(result) { it.includedModules.map { it.moduleName }.distinct() }
return result
}
/**
* Map name of JAR to names of the modules; these modules will be packed into these JARs and copied to the product's 'lib' directory.
*/
fun withAdditionalPlatformJar(jarName: String, vararg moduleNames: String) {
additionalPlatformJars.putValues(jarName, moduleNames.asList())
}
fun withoutAdditionalPlatformJar(jarName: String, moduleName: String) {
additionalPlatformJars.remove(jarName, moduleName)
}
}
internal fun createPluginLayoutSet(expectedSize: Int): MutableSet<PluginLayout> {

View File

@@ -6,6 +6,7 @@ import kotlinx.collections.immutable.PersistentMap
import kotlinx.collections.immutable.persistentListOf
import kotlinx.collections.immutable.persistentMapOf
import org.jetbrains.annotations.ApiStatus
import org.jetbrains.annotations.ApiStatus.Internal
import org.jetbrains.intellij.build.impl.productInfo.CustomProperty
import org.jetbrains.jps.model.module.JpsModule
import java.nio.file.Path
@@ -46,6 +47,9 @@ abstract class ProductProperties {
*/
lateinit var applicationInfoModule: String
@Internal
var productPluginSourceModuleName: String? = null
/**
* Enables fast activation of a running IDE instance from the launcher
* (at the moment, it is only implemented in the native Windows one).
@@ -129,16 +133,7 @@ abstract class ProductProperties {
/**
* If `true`, the product's main JAR file will be scrambled using [ProprietaryBuildTools.scrambleTool].
*/
var scrambleMainJar = false
@ApiStatus.Experimental
var useProductJar = true
/**
* If `false`, names of private fields won't be scrambled (to avoid problems with serialization).
* This field is ignored if [scrambleMainJar] is `false`.
*/
var scramblePrivateFields = true
var scrambleMainJar: Boolean = false
/**
* Path to an alternative scramble script which will should be used for a product.
@@ -148,7 +143,7 @@ abstract class ProductProperties {
/**
* Describes which modules should be included in the product's platform and which plugins should be bundled with the product.
*/
val productLayout = ProductModulesLayout()
val productLayout: ProductModulesLayout = ProductModulesLayout()
/**
* If `true`, a cross-platform ZIP archive containing binaries for all OSes will be built.
@@ -156,7 +151,7 @@ abstract class ProductProperties {
* (override [getCrossPlatformZipFileName] to change the file name).
* Cross-platform distribution is required for [plugins development](https://github.com/JetBrains/gradle-intellij-plugin).
*/
var buildCrossPlatformDistribution = false
var buildCrossPlatformDistribution: Boolean = false
/**
* Specifies name of cross-platform ZIP archive if `[buildCrossPlatformDistribution]` is set to `true`.
@@ -212,7 +207,7 @@ abstract class ProductProperties {
* If `true`, a .zip archive containing sources of modules included in the product will be produced.
* See also [includeIntoSourcesArchiveFilter].
*/
var buildSourcesArchive = false
var buildSourcesArchive: Boolean = false
/**
* Determines sources of which modules should be included in the source archive when [buildSourcesArchive] is `true`.
@@ -222,7 +217,7 @@ abstract class ProductProperties {
/**
* Specifies how Maven artifacts for IDE modules should be generated; by default, no artifacts are generated.
*/
val mavenArtifacts = MavenArtifactsProperties()
val mavenArtifacts: MavenArtifactsProperties = MavenArtifactsProperties()
/**
* Specified additional modules (not included into the product layout) which need to be compiled when product is built.
@@ -273,8 +268,7 @@ abstract class ProductProperties {
* If `true`, a distribution contains libraries and launcher script for running IDE in Remote Development mode.
*/
@ApiStatus.Internal
open fun addRemoteDevelopmentLibraries(): Boolean =
productLayout.bundledPluginModules.contains("intellij.remoteDevServer")
open fun addRemoteDevelopmentLibraries(): Boolean = productLayout.bundledPluginModules.contains("intellij.remoteDevServer")
/**
* Build steps which are always skipped for this product.

View File

@@ -1,6 +1,8 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build
import org.jetbrains.intellij.build.impl.ModuleItem
import org.jetbrains.intellij.build.impl.PlatformLayout
import org.jetbrains.intellij.build.impl.PluginLayout
import java.nio.file.Path
@@ -10,24 +12,21 @@ import java.nio.file.Path
*/
interface ScrambleTool {
/**
* @return list of modules used by the tool which need to be compiled before {@link #scramble} method is invoked
* @return list of modules used by the tool which needs to be compiled before {@link #scramble} method is invoked
*/
val additionalModulesToCompile: List<String>
/**
* Scramble [mainJarName] in "[BuildPaths.distAllDir]/lib" directory
*/
suspend fun scramble(mainJarName: String, context: BuildContext)
suspend fun scramble(platform: PlatformLayout, context: BuildContext)
suspend fun scramblePlugin(context: BuildContext, pluginLayout: PluginLayout, targetDir: Path, additionalPluginsDir: Path)
/**
* @return list of names of JAR files which cannot be included into the product 'lib' directory in plain form
*/
val namesOfJarsRequiredToBeScrambled: List<String>
/**
* Returns list of module names which cannot be included into the product without scrambling.
* Returns list of module names which cannot be included in the product without scrambling.
*/
val namesOfModulesRequiredToBeScrambled: List<String>
suspend fun validatePlatformLayout(modules: Collection<ModuleItem>, context: BuildContext)
}

View File

@@ -1,30 +1,37 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("ReplaceGetOrSet")
package org.jetbrains.intellij.build.impl
import com.intellij.util.containers.MultiMap
import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet
import kotlinx.collections.immutable.*
import java.util.*
import org.jetbrains.annotations.TestOnly
import java.lang.IllegalStateException
import java.lang.StackWalker.Option
import kotlin.collections.LinkedHashSet
import kotlin.streams.asSequence
const val APP_JAR: String = "app.jar"
const val PRODUCT_JAR: String = "product.jar"
internal const val TEST_FRAMEWORK_JAR: String = "testFramework.jar"
/**
* Describes layout of a plugin or the platform JARs in the product distribution
*/
open class BaseLayout {
companion object {
const val APP_JAR: String = "app.jar"
}
sealed class BaseLayout {
// one module can be packed into several JARs; that's why we have map "jar to modules" and not "module to jar"
private val _jarToModules = TreeMap<String, MutableList<String>>()
private val _includedModules = LinkedHashSet<ModuleItem>()
/** JAR name (or path relative to 'lib' directory) to names of modules */
val jarToModules: Map<String, List<String>>
get() = Collections.unmodifiableMap(_jarToModules)
val includedModules: Collection<ModuleItem>
get() = _includedModules
/** artifact name to a relative output path */
@JvmField
internal var includedArtifacts: PersistentMap<String, String> = persistentMapOf()
/** list of additional resources which should be included in the distribution */
@JvmField
internal var resourcePaths: PersistentList<ModuleResourceData> = persistentListOf()
/** module name to entries which should be excluded from its output */
@@ -32,25 +39,57 @@ open class BaseLayout {
private set
@Suppress("SSBasedInspection")
internal val includedProjectLibraries = ObjectOpenHashSet<ProjectLibraryData>()
@JvmField
@PublishedApi
internal val includedProjectLibraries: ObjectOpenHashSet<ProjectLibraryData> = ObjectOpenHashSet()
val includedModuleLibraries: MutableSet<ModuleLibraryData> = LinkedHashSet()
/** module name to name of the module library */
val excludedModuleLibraries: MultiMap<String, String> = MultiMap.createLinked()
/** JAR name -> name of a project library which content should be unpacked */
val projectLibrariesToUnpack: MultiMap<String, String> = MultiMap.createLinked()
val modulesWithExcludedModuleLibraries: MutableList<String> = mutableListOf()
// only as guard for checkAndAssociateModuleNameWithJarPath - do not use it, because strictly speaking for one module, maybe several JARs
private val _includedModuleNamesToJarPath = mutableMapOf<String, String>()
val includedModuleNames: Set<String>
get() = Collections.unmodifiableSet(_includedModuleNamesToJarPath.keys)
fun hasLibrary(name: String): Boolean = includedProjectLibraries.any { it.libraryName == name }
fun withModules(moduleNames: List<String>, relativeJarPath: String) {
for (moduleName in moduleNames) {
withModule(moduleName = moduleName, relativeJarPath = relativeJarPath)
@TestOnly
fun includedProjectLibraryNames(): Sequence<String> = includedProjectLibraries.asSequence().map { it.libraryName }
fun filteredIncludedModuleNames(excludedRelativeJarPath: String): Sequence<String> {
return _includedModules.asSequence().filter { it.relativeOutputFile != excludedRelativeJarPath }.map { it.moduleName }
}
fun withModules(items: Collection<ModuleItem>) {
for (item in items) {
checkNotExists(item)
}
_includedModules.addAll(items)
}
private fun checkNotExists(item: ModuleItem) {
val existing = _includedModules.firstOrNull { it.moduleName == item.moduleName } ?: return
// allow putting module to several JARs if JAR located in another dir
// (e.g. intellij.spring.customNs packed into main JAR and customNs/customNs.jar)
if (existing.relativeOutputFile != item.relativeOutputFile &&
(existing.relativeOutputFile.contains('/') || item.relativeOutputFile.contains('/'))) {
return
}
if (item.moduleName.startsWith("intellij.maven.artifactResolver.")) {
return
}
throw IllegalStateException(
"Module ${item.moduleName} is already configured to be included in the layout. " +
"Please make sure the module name is not duplicated." +
"\n The existing: $existing" +
"\n The new: $item"
)
}
abstract fun withModule(moduleName: String)
fun withModules(names: Iterable<String>) {
names.forEach(::withModule)
}
fun withModule(moduleName: String, relativeJarPath: String) {
@@ -58,29 +97,40 @@ open class BaseLayout {
"Module name must be not empty"
}
val previousJarPath = _includedModuleNamesToJarPath.put(moduleName, relativeJarPath)
if (previousJarPath != null && moduleName != "intellij.maven.artifactResolver.common") {
if (previousJarPath == relativeJarPath) {
// already added
return
}
// allow putting module to several JARs if JAR located in another dir
// (e.g. intellij.spring.customNs packed into main JAR and customNs/customNs.jar)
check(previousJarPath.contains('/') || relativeJarPath.contains('/')) {
"Module '$moduleName' cannot be packed into $relativeJarPath because it is already configured to be packed into $previousJarPath"
val stackTrace = StackWalker.getInstance(Option.RETAIN_CLASS_REFERENCE).walk { stream ->
stream.use {
stream.asSequence()
.dropWhile {
val declaringClass = it.declaringClass.name
// startsWith - spec class
declaringClass.startsWith("org.jetbrains.intellij.build.impl.BaseLayout") ||
// startsWith - `Companion` object
declaringClass.startsWith("org.jetbrains.intellij.build.impl.PluginLayout")
}
.take(3)
.joinToString(separator = "\n ")
}
}
_jarToModules.computeIfAbsent(relativeJarPath) { mutableListOf() }.add(moduleName)
val item = ModuleItem(moduleName = moduleName, relativeOutputFile = relativeJarPath, reason = "withModule at \n $stackTrace")
checkNotExists(item)
_includedModules.add(item)
}
open fun withModule(moduleName: String) {
withModule(moduleName, "${convertModuleNameToFileName(moduleName)}.jar")
fun withProjectLibrary(libraryName: String, jarName: String, reason: String? = null) {
includedProjectLibraries.add(ProjectLibraryData(libraryName = libraryName,
packMode = LibraryPackMode.STANDALONE_MERGED,
outPath = jarName,
reason = reason))
}
fun withProjectLibraryUnpackedIntoJar(libraryName: String, jarName: String) {
projectLibrariesToUnpack.putValue(jarName, libraryName)
fun withProjectLibraries(libraryNames: Collection<String>, jarName: String, reason: String? = null) {
for (libraryName in libraryNames) {
includedProjectLibraries.add(ProjectLibraryData(libraryName = libraryName,
packMode = LibraryPackMode.STANDALONE_MERGED,
outPath = jarName,
reason = reason))
}
}
fun excludeFromModule(moduleName: String, excludedPattern: String) {
@@ -113,7 +163,8 @@ open class BaseLayout {
includedModuleLibraries.add(ModuleLibraryData(
moduleName = moduleName,
libraryName = libraryName,
relativeOutputPath = relativeOutputPath))
relativeOutputPath = relativeOutputPath,
))
}
/**
@@ -128,10 +179,43 @@ open class BaseLayout {
}
}
internal fun convertModuleNameToFileName(moduleName: String): String = moduleName.removePrefix("intellij.").replace('.', '-')
data class ModuleLibraryData(
val moduleName: String,
val libraryName: String,
val relativeOutputPath: String = "",
)
@JvmField val moduleName: String,
@JvmField val libraryName: String,
@JvmField val relativeOutputPath: String = "",
)
class ModuleItem(
@JvmField val moduleName: String,
// for one module, maybe several JARs - that's why `relativeOutputPath` is included into hash code
@JvmField val relativeOutputFile: String,
@JvmField val reason: String?,
) {
init {
require(!moduleName.isEmpty()) {
"Module name must be not empty"
}
}
override fun equals(other: Any?): Boolean {
if (this === other) {
return true
}
if (other !is ModuleItem) {
return false
}
if (moduleName != other.moduleName) {
return false
}
return relativeOutputFile == other.relativeOutputFile
}
override fun hashCode(): Int {
var result = moduleName.hashCode()
result = 31 * result + relativeOutputFile.hashCode()
return result
}
override fun toString(): String = "ModuleItem(moduleName=$moduleName, relativeOutputFile=$relativeOutputFile, reason=$reason)"
}

View File

@@ -13,7 +13,7 @@ sealed class BaseLayoutSpec(private val layout: BaseLayout) {
}
fun withModules(names: Iterable<String>) {
names.forEach(layout::withModule)
layout.withModules(names)
}
/**
@@ -86,6 +86,6 @@ sealed class BaseLayoutSpec(private val layout: BaseLayout) {
* Include contents of JARs of the project library {@code libraryName} into JAR {@code jarName}
*/
fun withProjectLibraryUnpackedIntoJar(libraryName: String, jarName: String) {
layout.withProjectLibraryUnpackedIntoJar(libraryName, jarName)
layout.withProjectLibrary(libraryName, jarName)
}
}

View File

@@ -25,7 +25,7 @@ import java.nio.file.Path
import java.util.concurrent.ConcurrentLinkedQueue
import java.util.concurrent.atomic.AtomicReference
class BuildContextImpl private constructor(
class BuildContextImpl(
private val compilationContext: CompilationContextImpl,
override val productProperties: ProductProperties,
override val windowsDistributionCustomizer: WindowsDistributionCustomizer?,
@@ -161,7 +161,11 @@ class BuildContextImpl private constructor(
}
override fun findFileInModuleSources(moduleName: String, relativePath: String): Path? {
for (info in getSourceRootsWithPrefixes(findRequiredModule(moduleName))) {
return findFileInModuleSources(findRequiredModule(moduleName), relativePath)
}
override fun findFileInModuleSources(module: JpsModule, relativePath: String): Path? {
for (info in getSourceRootsWithPrefixes(module)) {
if (relativePath.startsWith(info.second)) {
val result = info.first.resolve(Strings.trimStart(Strings.trimStart(relativePath, info.second), "/"))
if (Files.exists(result)) {

View File

@@ -57,6 +57,7 @@ import java.nio.file.attribute.PosixFilePermission
import java.util.*
import java.util.concurrent.TimeUnit
import java.util.function.Predicate
import java.util.zip.Deflater
import kotlin.io.path.setLastModifiedTime
class BuildTasksImpl(context: BuildContext) : BuildTasks {
@@ -80,12 +81,13 @@ class BuildTasksImpl(context: BuildContext) : BuildTasks {
checkPluginModules(mainPluginModules, "mainPluginModules", context)
copyDependenciesFile(context)
val pluginsToPublish = getPluginLayoutsByJpsModuleNames(mainPluginModules, context.productProperties.productLayout)
val distributionJARsBuilder = DistributionJARsBuilder(compilePlatformAndPluginModules(pluginsToPublish, context))
distributionJARsBuilder.buildSearchableOptions(context)
distributionJARsBuilder.buildNonBundledPlugins(pluginsToPublish = pluginsToPublish,
compressPluginArchive = context.options.compressZipFiles,
buildPlatformLibJob = null,
context = context)
val state = compilePlatformAndPluginModules(pluginsToPublish, context)
buildSearchableOptions(state, context)
buildNonBundledPlugins(pluginsToPublish = pluginsToPublish,
compressPluginArchive = context.options.compressZipFiles,
buildPlatformLibJob = null,
state = state,
context = context)
}
override fun compileProjectAndTests(includingTestsInModules: List<String>) {
@@ -111,7 +113,7 @@ class BuildTasksImpl(context: BuildContext) : BuildTasks {
context.options.buildStepsToSkip.add(BuildOptions.GENERATE_JAR_ORDER_STEP)
BundledMavenDownloader.downloadMavenCommonLibs(context.paths.communityHomeDirRoot)
BundledMavenDownloader.downloadMavenDistribution(context.paths.communityHomeDirRoot)
DistributionJARsBuilder(compileModulesForDistribution(context)).buildJARs(context = context, isUpdateFromSources = true)
buildDistribution(state = compileModulesForDistribution(context), context = context, isUpdateFromSources = true)
val arch = if (SystemInfo.isMac && CpuArch.isIntel64() && CpuArch.isEmulated()) {
JvmArchitecture.aarch64
}
@@ -142,7 +144,7 @@ class BuildTasksImpl(context: BuildContext) : BuildTasks {
suspend fun generateProjectStructureMapping(targetFile: Path, context: BuildContext) {
writeProjectStructureReport(
entries = generateProjectStructureMapping(context = context,
state = DistributionBuilderState(pluginsToPublish = emptySet(), context = context)),
state = createDistributionBuilderState(pluginsToPublish = emptySet(), context = context)),
file = targetFile,
buildPaths = context.paths
)
@@ -180,7 +182,7 @@ private suspend fun buildProvidedModuleList(targetFile: Path, state: Distributio
context.executeStep(spanBuilder("build provided module list"), BuildOptions.PROVIDED_MODULES_LIST_STEP) {
withContext(Dispatchers.IO) {
Files.deleteIfExists(targetFile)
val ideClasspath = DistributionJARsBuilder(state).createIdeClassPath(context)
val ideClasspath = createIdeClassPath(state, context)
// start the product in headless mode using com.intellij.ide.plugins.BundledPluginsLister
runApplicationStarter(context = context,
tempDir = context.paths.tempDir.resolve("builtinModules"),
@@ -422,8 +424,8 @@ suspend fun zipSourcesOfModules(modules: List<String>, targetFile: Path, include
val debugMapping = mutableListOf<String>()
for (moduleName in modules) {
val module = context.findRequiredModule(moduleName)
// We pack source of libraries which are included into compilation classpath for platform API modules,
// this way we'll get sources of all libraries useful for plugin developers and size of the archive will be reasonable
// We pack sources of libraries which are included in compilation classpath for platform API modules.
// This way we'll get sources of all libraries useful for plugin developers, and the size of the archive will be reasonable.
if (moduleName.startsWith("intellij.platform.") && context.findModule("$moduleName.impl") != null) {
val libraries = JpsJavaExtensionService.dependencies(module).productionOnly().compileOnly().recursivelyExportedOnly().libraries
includedLibraries.addAll(libraries)
@@ -436,7 +438,7 @@ suspend fun zipSourcesOfModules(modules: List<String>, targetFile: Path, include
.asSequence()
.map { it.asTyped(JpsRepositoryLibraryType.INSTANCE) }
.filterNotNull()
.filter { library -> library.getFiles(JpsOrderRootType.SOURCES).any { Files.notExists(it.toPath()) } }
.filter { library -> library.getPaths(JpsOrderRootType.SOURCES).any { Files.notExists(it) } }
.toList()
if (!librariesWithMissingSources.isEmpty()) {
withContext(Dispatchers.IO) {
@@ -508,8 +510,8 @@ private inline fun filterSourceFilesOnly(name: String, context: BuildContext, co
return sourceFiles
}
private fun compilePlatformAndPluginModules(pluginsToPublish: Set<PluginLayout>, context: BuildContext): DistributionBuilderState {
val distState = DistributionBuilderState(pluginsToPublish, context)
private suspend fun compilePlatformAndPluginModules(pluginsToPublish: Set<PluginLayout>, context: BuildContext): DistributionBuilderState {
val distState = createDistributionBuilderState(pluginsToPublish, context)
val compilationTasks = CompilationTasks.create(context)
compilationTasks.compileModules(
distState.getModulesForPluginsToPublish() +
@@ -576,13 +578,8 @@ suspend fun buildDistributions(context: BuildContext): Unit = spanBuilder("build
createMavenArtifactJob(context, distributionState)
spanBuilder("build platform and plugin JARs").useWithScope2<Unit> {
val distributionJARsBuilder = DistributionJARsBuilder(distributionState)
if (context.productProperties.buildDocAuthoringAssets) {
buildAdditionalAuthoringArtifacts(distributionJARsBuilder, context)
}
if (context.shouldBuildDistributions()) {
val entries = distributionJARsBuilder.buildJARs(context)
val entries = buildDistribution(state = distributionState, context)
if (context.productProperties.buildSourcesArchive) {
buildSourcesArchive(entries, context)
}
@@ -590,11 +587,12 @@ suspend fun buildDistributions(context: BuildContext): Unit = spanBuilder("build
else {
Span.current().addEvent("skip building product distributions because " +
"\"intellij.build.target.os\" property is set to \"${BuildOptions.OS_NONE}\"")
distributionJARsBuilder.buildSearchableOptions(context)
distributionJARsBuilder.buildNonBundledPlugins(pluginsToPublish = pluginsToPublish,
compressPluginArchive = context.options.compressZipFiles,
buildPlatformLibJob = null,
context = context)
buildSearchableOptions(distributionState, context)
buildNonBundledPlugins(pluginsToPublish = pluginsToPublish,
compressPluginArchive = context.options.compressZipFiles,
buildPlatformLibJob = null,
state = distributionState,
context = context)
}
}
@@ -715,10 +713,6 @@ private fun checkProductLayout(context: BuildContext) {
val layout = context.productProperties.productLayout
// todo mainJarName type specified as not-null - does it work?
val messages = context.messages
@Suppress("SENSELESS_COMPARISON")
check(layout.mainJarName != null) {
"productProperties.productLayout.mainJarName is not specified"
}
val pluginLayouts = layout.pluginLayouts
checkScrambleClasspathPlugins(pluginLayouts)
@@ -746,18 +740,15 @@ private fun checkProductLayout(context: BuildContext) {
}
checkModules(layout.productApiModules, "productProperties.productLayout.productApiModules", context)
checkModules(layout.productImplementationModules, "productProperties.productLayout.productImplementationModules", context)
checkModules(layout.additionalPlatformJars.values(), "productProperties.productLayout.additionalPlatformJars", context)
checkModules(layout.moduleExcludes.keys, "productProperties.productLayout.moduleExcludes", context)
checkModules(layout.mainModules, "productProperties.productLayout.mainModules", context)
checkProjectLibraries(layout.projectLibrariesToUnpackIntoMainJar,
"productProperties.productLayout.projectLibrariesToUnpackIntoMainJar", context)
for (plugin in pluginLayouts) {
checkBaseLayout(plugin, "\'${plugin.mainModule}\' plugin", context)
}
}
private fun checkBaseLayout(layout: BaseLayout, description: String, context: BuildContext) {
checkModules(layout.includedModuleNames.toList(), "moduleJars in $description", context)
checkModules(layout.includedModules.asSequence().map { it.moduleName }.distinct().toList(), "moduleJars in $description", context)
checkArtifacts(layout.includedArtifacts.keys, "includedArtifacts in $description", context)
checkModules(layout.resourcePaths.map { it.moduleName }, "resourcePaths in $description", context)
checkModules(layout.moduleExcludes.keys, "moduleExcludes in $description", context)
@@ -781,7 +772,6 @@ private fun checkBaseLayout(layout: BaseLayout, description: String, context: Bu
}
}
checkProjectLibraries(layout.projectLibrariesToUnpack.values(), "projectLibrariesToUnpack in $description", context)
checkModules(layout.modulesWithExcludedModuleLibraries, "modulesWithExcludedModuleLibraries in $description", context)
}
@@ -1144,10 +1134,9 @@ fun getModulesToCompile(buildContext: BuildContext): Set<String> {
val productLayout = buildContext.productProperties.productLayout
val result = LinkedHashSet<String>()
result.addAll(productLayout.getIncludedPluginModules(java.util.Set.copyOf(productLayout.bundledPluginModules)))
PlatformModules.collectPlatformModules(result)
collectPlatformModules(result)
result.addAll(productLayout.productApiModules)
result.addAll(productLayout.productImplementationModules)
result.addAll(productLayout.additionalPlatformJars.values())
result.addAll(getToolModules())
result.addAll(buildContext.productProperties.additionalModulesToCompile)
result.add("intellij.idea.community.build.tasks")
@@ -1156,29 +1145,29 @@ fun getModulesToCompile(buildContext: BuildContext): Set<String> {
return result
}
// Captures information about all available inspections in a JSON format as part of Inspectopedia project.
// This is later used by Qodana and other tools. Keymaps are extracted as XML file and also used in help authoring.
private suspend fun buildAdditionalAuthoringArtifacts(builder: DistributionJARsBuilder,
context: BuildContext) {
// Captures information about all available inspections in a JSON format as part of an Inspectopedia project.
// This is later used by Qodana and other tools. Keymaps are extracted as an XML file and also used in help authoring.
internal suspend fun buildAdditionalAuthoringArtifacts(ideClassPath: Set<String>, context: BuildContext) {
val commands = listOf(Pair("inspectopedia-generator", "inspections-${context.applicationInfo.productCode.lowercase()}"),
Pair("keymap", "keymap-${context.applicationInfo.productCode.lowercase()}"))
val ideClasspath = builder.createIdeClassPath(context)
val temporaryBuildDirectory = context.paths.tempDir
coroutineScope {
for (command in commands) {
launch {
val temporaryStepDirectory = temporaryBuildDirectory.resolve(command.first)
val targetPath = temporaryStepDirectory.resolve(command.second)
runApplicationStarter(context = context,
tempDir = temporaryStepDirectory,
ideClasspath = ideClassPath,
arguments = listOf(command.first, targetPath.toString()))
commands.forEach {
val temporaryStepDirectory = temporaryBuildDirectory.resolve(it.first)
val targetPath = temporaryStepDirectory.resolve(it.second)
runApplicationStarter(context = context,
tempDir = temporaryStepDirectory,
ideClasspath = ideClasspath,
arguments = listOf(it.first, targetPath.toAbsolutePath().toString()))
val targetFile = context.paths.artifactDir.resolve("${it.second}.zip")
zipWithCompression(targetFile = targetFile, dirs = mapOf(targetPath to ""))
val targetFile = context.paths.artifactDir.resolve("${command.second}.zip")
zipWithCompression(targetFile = targetFile,
dirs = mapOf(targetPath to ""),
compressionLevel = if (context.options.compressZipFiles) Deflater.DEFAULT_COMPRESSION else Deflater.NO_COMPRESSION)
}
}
}
}

View File

@@ -6,37 +6,35 @@ package org.jetbrains.intellij.build.impl
import it.unimi.dsi.fastutil.objects.ObjectLinkedOpenHashSet
import org.jetbrains.intellij.build.BuildContext
import org.jetbrains.intellij.build.ProductProperties
import org.jetbrains.intellij.build.impl.PlatformModules.hasPlatformCoverage
import org.jetbrains.jps.model.java.JpsJavaClasspathKind
import org.jetbrains.jps.model.java.JpsJavaExtensionService
import org.jetbrains.jps.model.library.JpsLibrary
import org.jetbrains.jps.model.module.JpsModuleReference
import java.util.*
class DistributionBuilderState(pluginsToPublish: Set<PluginLayout>, private val context: BuildContext) {
@JvmField
val pluginsToPublish: Set<PluginLayout>
@JvmField
val platform: PlatformLayout
suspend fun createDistributionBuilderState(pluginsToPublish: Set<PluginLayout>, context: BuildContext): DistributionBuilderState {
val pluginsToPublishFiltered = filterPluginsToPublish(pluginsToPublish, context)
val platform = createPlatformLayout(pluginsToPublishFiltered, context)
return DistributionBuilderState(platform = platform, pluginsToPublish = pluginsToPublishFiltered, context = context)
}
class DistributionBuilderState(@JvmField val platform: PlatformLayout,
@JvmField val pluginsToPublish: Set<PluginLayout>,
private val context: BuildContext) {
init {
this.pluginsToPublish = filterPluginsToPublish(pluginsToPublish, context)
platform = createPlatformLayout(this.pluginsToPublish, context)
val releaseDate = context.applicationInfo.majorReleaseDate
if (releaseDate.startsWith("__")) {
context.messages.error("Unresolved release-date: $releaseDate")
require(!releaseDate.startsWith("__")) {
"Unresolved release-date: $releaseDate"
}
}
val platformModules: Collection<String>
get() = (platform.includedModuleNames + getToolModules().asSequence()).toList()
get() = (platform.includedModules.asSequence().map { it.moduleName }.distinct() + getToolModules().asSequence()).toList()
fun getModulesForPluginsToPublish(): Set<String> {
val result = LinkedHashSet<String>()
result.addAll(platformModules)
pluginsToPublish.flatMapTo(result) { it.includedModuleNames }
pluginsToPublish.flatMapTo(result) { layout -> layout.includedModules.asSequence().map { it.moduleName } }
return result
}
@@ -81,14 +79,14 @@ private fun filterPluginsToPublish(plugins: Set<PluginLayout>, context: BuildCon
return result
}
fun createPlatformLayout(pluginsToPublish: Set<PluginLayout>, context: BuildContext): PlatformLayout {
suspend fun createPlatformLayout(pluginsToPublish: Set<PluginLayout>, context: BuildContext): PlatformLayout {
val productLayout = context.productProperties.productLayout
val enabledPluginModules = getEnabledPluginModules(pluginsToPublish, context.productProperties)
val projectLibrariesUsedByPlugins = computeProjectLibsUsedByPlugins(enabledPluginModules, context)
return PlatformModules.createPlatformLayout(productLayout,
hasPlatformCoverage(productLayout, enabledPluginModules, context),
projectLibrariesUsedByPlugins,
context)
return createPlatformLayout(productLayout = productLayout,
hasPlatformCoverage = hasPlatformCoverage(productLayout, enabledPluginModules, context),
additionalProjectLevelLibraries = projectLibrariesUsedByPlugins,
context = context)
}
private fun getEnabledPluginModules(pluginsToPublish: Set<PluginLayout>, productProperties: ProductProperties): Set<String> {
@@ -102,16 +100,15 @@ private fun computeProjectLibsUsedByPlugins(enabledPluginModules: Set<String>, c
val result = ObjectLinkedOpenHashSet<ProjectLibraryData>()
for (plugin in getPluginLayoutsByJpsModuleNames(modules = enabledPluginModules, productLayout = context.productProperties.productLayout)) {
val libsToUnpack = plugin.projectLibrariesToUnpack.values()
for (moduleName in plugin.includedModuleNames) {
for (moduleName in plugin.includedModules.asSequence().map { it.moduleName }.distinct()) {
val dependencies = JpsJavaExtensionService.dependencies(context.findRequiredModule(moduleName))
dependencies.includedIn(JpsJavaClasspathKind.PRODUCTION_RUNTIME).processLibraries(com.intellij.util.Consumer {library ->
if (!isProjectLibraryUsedByPlugin(library, plugin, libsToUnpack)) {
if (!isProjectLibraryUsedByPlugin(library = library, plugin = plugin)) {
return@Consumer
}
val name = library.name
val packMode = PlatformModules.CUSTOM_PACK_MODE.getOrDefault(name, LibraryPackMode.MERGED)
val packMode = PLATFORM_CUSTOM_PACK_MODE.getOrDefault(name, LibraryPackMode.MERGED)
result.addOrGet(ProjectLibraryData(name, packMode))
.dependentModules
.computeIfAbsent(plugin.directoryName) { ArrayList<String>() }
@@ -130,8 +127,6 @@ internal fun getToolModules(): List<String> {
/*required to build searchable options index*/ "intellij.platform.updater")
}
internal fun isProjectLibraryUsedByPlugin(library: JpsLibrary, plugin: BaseLayout, libsToUnpack: Collection<String>): Boolean {
return library.createReference().parentReference !is JpsModuleReference &&
!plugin.includedProjectLibraries.any {it.libraryName == library.name} &&
!libsToUnpack.contains(library.name)
internal fun isProjectLibraryUsedByPlugin(library: JpsLibrary, plugin: BaseLayout): Boolean {
return library.createReference().parentReference !is JpsModuleReference && !plugin.hasLibrary(library.name)
}

View File

@@ -1,5 +1,6 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("ReplaceGetOrSet", "ReplacePutWithAssignment", "ReplaceJavaStaticMethodWithKotlinAnalog")
package org.jetbrains.intellij.build.impl
import com.intellij.diagnostic.telemetry.use
@@ -8,16 +9,10 @@ import com.intellij.util.io.URLUtil
import com.intellij.util.io.sanitizeFileName
import com.intellij.util.lang.HashMapZipFile
import io.opentelemetry.api.common.AttributeKey
import io.opentelemetry.api.common.Attributes
import io.opentelemetry.api.trace.Span
import kotlinx.collections.immutable.PersistentMap
import kotlinx.collections.immutable.mutate
import kotlinx.collections.immutable.persistentMapOf
import kotlinx.collections.immutable.plus
import kotlinx.collections.immutable.*
import kotlinx.coroutines.*
import org.jetbrains.intellij.build.*
import org.jetbrains.intellij.build.TraceManager.spanBuilder
import org.jetbrains.intellij.build.impl.BaseLayout.Companion.APP_JAR
import org.jetbrains.intellij.build.impl.projectStructureMapping.*
import org.jetbrains.intellij.build.io.W_CREATE_NEW
import org.jetbrains.intellij.build.tasks.*
@@ -33,6 +28,7 @@ import java.nio.channels.FileChannel
import java.nio.file.FileSystems
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.PathMatcher
import java.util.*
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.ConcurrentLinkedQueue
@@ -80,42 +76,56 @@ private val extraMergeRules: PersistentMap<String, (String) -> Boolean> = persis
map.put("jsch-agent.jar") { it.startsWith("jsch-agent") }
map.put("rd.jar") { it.startsWith("rd-") }
// see ClassPathUtil.getUtilClassPath
map.put("3rd-party-rt.jar") { libsThatUsedInJps.contains(it) || it.startsWith("kotlinx-") || it == "kotlin-reflect" }
map.put("3rd-party-rt.jar") {
libsThatUsedInJps.contains(it) || it.startsWith("kotlinx-") || it == "kotlin-reflect" ||
// used by intellij.database.jdbcConsole
it == "jbr-api"
}
}
internal fun getLibraryFileName(lib: JpsLibrary): String {
val name = lib.name
internal fun getLibraryFileName(library: JpsLibrary): String {
val name = library.name
if (!name.startsWith("#")) {
return name
}
val roots = lib.getRoots(JpsOrderRootType.COMPILED)
val roots = library.getRoots(JpsOrderRootType.COMPILED)
check(roots.size == 1) {
"Non-single entry module library $name: ${roots.joinToString { it.url }}"
}
return PathUtilRt.getFileName(roots.first().url.removeSuffix(URLUtil.JAR_SEPARATOR))
}
class JarPackager private constructor(private val context: BuildContext) {
class JarPackager private constructor(private val collectNativeFiles: Boolean,
private val outputDir: Path,
private val context: BuildContext) {
private val jarDescriptors = LinkedHashMap<Path, JarDescriptor>()
private val libraryEntries = ConcurrentLinkedQueue<LibraryFileEntry>()
private val libToMetadata = HashMap<JpsLibrary, ProjectLibraryData>()
private val copiedFiles = HashMap<Path, CopiedFor>()
companion object {
suspend fun pack(jarToModules: Map<String, List<String>>,
suspend fun pack(includedModules: Collection<ModuleItem>,
outputDir: Path,
layout: BaseLayout = BaseLayout(),
layout: BaseLayout?,
moduleOutputPatcher: ModuleOutputPatcher = ModuleOutputPatcher(),
dryRun: Boolean = false,
context: BuildContext): Collection<DistributionFileEntry> {
val copiedFiles = HashMap<Path, CopiedFor>()
val packager = JarPackager(context)
for (data in layout.includedModuleLibraries) {
val library = context.findRequiredModule(data.moduleName).libraryCollection.libraries
.find { getLibraryFileName(it) == data.libraryName }
?: throw IllegalArgumentException("Cannot find library ${data.libraryName} in \'${data.moduleName}\' module")
var fileName = nameToJarFileName(data.libraryName)
var relativePath = data.relativeOutputPath
val packager = JarPackager(collectNativeFiles = layout !is PluginLayout, outputDir = outputDir, context = context)
// must be concurrent - buildJars executed in parallel
val moduleNameToSize = ConcurrentHashMap<String, Int>()
packager.packModules(includedModules = includedModules,
moduleNameToSize = moduleNameToSize,
moduleOutputPatcher = moduleOutputPatcher,
layout = layout)
for (item in (layout?.includedModuleLibraries ?: emptyList())) {
val library = context.findRequiredModule(item.moduleName).libraryCollection.libraries
.find { getLibraryFileName(it) == item.libraryName }
?: throw IllegalArgumentException("Cannot find library ${item.libraryName} in \'${item.moduleName}\' module")
var fileName = nameToJarFileName(item.libraryName)
var relativePath = item.relativeOutputPath
var targetFile: Path? = null
if (relativePath.endsWith(".jar")) {
val index = relativePath.lastIndexOf('/')
@@ -137,83 +147,31 @@ class JarPackager private constructor(private val context: BuildContext) {
packager.addLibrary(
library = library,
targetFile = targetFile!!,
files = getLibraryFiles(library = library, copiedFiles = copiedFiles, isModuleLevel = true, targetFile = targetFile)
files = getLibraryFiles(library = library, copiedFiles = packager.copiedFiles, isModuleLevel = true, targetFile = targetFile)
)
}
val extraLibSources = HashMap<String, MutableList<Source>>()
val libraryToMerge = packager.packProjectLibraries(jarToModuleNames = jarToModules,
outputDir = outputDir,
layout = layout,
copiedFiles = copiedFiles,
extraLibSources = extraLibSources)
val isRootDir = context.paths.distAllDir == outputDir.parent
if (isRootDir) {
for ((key, value) in extraMergeRules) {
packager.mergeLibsByPredicate(key, libraryToMerge, outputDir, value)
}
if (!libraryToMerge.isEmpty()) {
packager.filesToSourceWithMappings(outputDir.resolve(APP_JAR), libraryToMerge)
}
}
else if (!libraryToMerge.isEmpty()) {
val mainJarName = (layout as PluginLayout).getMainJarName()
assert(jarToModules.containsKey(mainJarName))
packager.filesToSourceWithMappings(outputDir.resolve(mainJarName), libraryToMerge)
}
// must be concurrent - buildJars executed in parallel
val moduleNameToSize = ConcurrentHashMap<String, Int>()
for ((jarPath, modules) in jarToModules) {
val jarFile = outputDir.resolve(jarPath)
val descriptor = packager.jarDescriptors.computeIfAbsent(jarFile) { JarDescriptor(jarFile = it) }
val includedModules = descriptor.includedModules
if (includedModules == null) {
descriptor.includedModules = modules.toMutableList()
}
else {
includedModules.addAll(modules)
}
val sourceList = descriptor.sources
extraLibSources.get(jarPath)?.let(sourceList::addAll)
packager.packModuleOutputAndUnpackedProjectLibraries(modules = modules,
jarPath = jarPath,
jarFile = jarFile,
moduleOutputPatcher = moduleOutputPatcher,
layout = layout,
moduleNameToSize = moduleNameToSize,
sourceList = sourceList)
}
val descriptors = ArrayList<BuildJarDescriptor>(packager.jarDescriptors.size)
val isReorderingEnabled = !context.options.buildStepsToSkip.contains(BuildOptions.GENERATE_JAR_ORDER_STEP)
val collectNativeFiles = layout !is PluginLayout
for (descriptor in packager.jarDescriptors.values) {
var pathInClassLog = ""
if (isReorderingEnabled) {
if (isRootDir) {
pathInClassLog = outputDir.parent.relativize(descriptor.jarFile).toString().replace(File.separatorChar, '/')
if (layout != null) {
val libraryToMerge = packager.packProjectLibraries(outputDir = outputDir,
layout = layout,
copiedFiles = packager.copiedFiles)
if (isRootDir) {
for ((key, value) in extraMergeRules) {
packager.mergeLibsByPredicate(key, libraryToMerge, outputDir, value)
}
else if (outputDir.startsWith(context.paths.distAllDir)) {
pathInClassLog = context.paths.distAllDir.relativize(descriptor.jarFile).toString().replace(File.separatorChar, '/')
}
else {
val parent = outputDir.parent
if (parent?.fileName.toString() == "plugins") {
pathInClassLog = outputDir.parent.parent.relativize(descriptor.jarFile).toString().replace(File.separatorChar, '/')
}
if (!libraryToMerge.isEmpty()) {
packager.filesToSourceWithMappings(outputDir.resolve(APP_JAR), libraryToMerge)
}
}
val fileName = descriptor.jarFile.fileName.toString()
descriptors.add(BuildJarDescriptor(
descriptor = descriptor,
relativePath = pathInClassLog,
collectNativeFiles = collectNativeFiles && (fileName == APP_JAR || fileName.startsWith("3rd-party-")),
))
else if (!libraryToMerge.isEmpty()) {
val mainJarName = (layout as PluginLayout).getMainJarName()
check(includedModules.any { it.relativeOutputFile == mainJarName })
packager.filesToSourceWithMappings(outputDir.resolve(mainJarName), libraryToMerge)
}
}
val nativeFiles = buildJars(descriptors = descriptors, dryRun = dryRun)
val nativeFiles = buildJars(descriptors = packager.jarDescriptors.values, dryRun = dryRun)
val list = mutableListOf<DistributionFileEntry>()
if (nativeFiles.isNotEmpty()) {
@@ -221,10 +179,11 @@ class JarPackager private constructor(private val context: BuildContext) {
}
for (item in packager.jarDescriptors.values) {
for (moduleName in (item.includedModules ?: emptyList())) {
for (module in item.includedModules) {
val moduleName = module.moduleName
val size = moduleNameToSize.get(moduleName)
?: throw IllegalStateException("Size is not set for $moduleName (moduleNameToSize=$moduleNameToSize)")
list.add(ModuleOutputEntry(path = item.jarFile, moduleName = moduleName, size = size))
list.add(ModuleOutputEntry(path = item.file, moduleName = moduleName, size = size, reason = module.reason))
}
}
@@ -276,6 +235,107 @@ class JarPackager private constructor(private val context: BuildContext) {
}
}
private fun packModules(includedModules: Collection<ModuleItem>,
moduleNameToSize: ConcurrentHashMap<String, Int>,
moduleOutputPatcher: ModuleOutputPatcher,
layout: BaseLayout?) {
for (item in includedModules) {
val descriptor = jarDescriptors.computeIfAbsent(outputDir.resolve(item.relativeOutputFile)) { jarFile ->
createJarDescriptor(outputDir = outputDir,
targetFile = jarFile,
collectNativeFiles = collectNativeFiles,
context = context)
}
descriptor.includedModules = descriptor.includedModules.add(item)
val sourceList = descriptor.sources
val moduleName = item.moduleName
val extraExcludes = layout?.moduleExcludes?.get(moduleName) ?: emptyList()
val sizeConsumer = IntConsumer {
moduleNameToSize.merge(moduleName, it) { oldValue, value -> oldValue + value }
}
for (entry in moduleOutputPatcher.getPatchedContent(moduleName)) {
sourceList.add(InMemoryContentSource(entry.key, entry.value, sizeConsumer))
}
// must be before module output to override
for (moduleOutputPatch in moduleOutputPatcher.getPatchedDir(moduleName)) {
sourceList.add(DirSource(dir = moduleOutputPatch, sizeConsumer = sizeConsumer))
}
val searchableOptionsModuleDir = context.searchableOptionDir.resolve(moduleName)
if (Files.exists(searchableOptionsModuleDir)) {
sourceList.add(DirSource(dir = searchableOptionsModuleDir, sizeConsumer = sizeConsumer))
}
val excludes = if (extraExcludes.isEmpty()) {
commonModuleExcludes
}
else {
val fileSystem = FileSystems.getDefault()
commonModuleExcludes + extraExcludes.map { fileSystem.getPathMatcher("glob:$it") }
}
sourceList.add(DirSource(dir = context.getModuleOutputDir(context.findRequiredModule(moduleName)),
excludes = excludes,
sizeConsumer = sizeConsumer))
if (layout != null) {
packModuleLibs(item = item, layout = layout, copiedFiles = copiedFiles, sources = descriptor.sources)
}
}
}
private fun packModuleLibs(item: ModuleItem,
layout: BaseLayout,
copiedFiles: MutableMap<Path, CopiedFor>,
sources: MutableList<Source>) {
if (item.relativeOutputFile.contains('/')) {
return
}
val moduleName = item.moduleName
if (layout.modulesWithExcludedModuleLibraries.contains(moduleName)) {
return
}
val excluded = layout.excludedModuleLibraries.get(moduleName)
for (element in context.findRequiredModule(moduleName).dependenciesList.dependencies) {
if (element !is JpsLibraryDependency || element.libraryReference.parentReference!!.resolve() !is JpsModule) {
continue
}
if (JpsJavaExtensionService.getInstance().getDependencyExtension(element)?.scope
?.isIncludedIn(JpsJavaClasspathKind.PRODUCTION_RUNTIME) != true) {
continue
}
val library = element.library!!
val libraryName = getLibraryFileName(library)
if (excluded.contains(libraryName) || layout.includedModuleLibraries.any { it.libraryName == libraryName }) {
continue
}
val targetFile = outputDir.resolve(item.relativeOutputFile)
val files = getLibraryFiles(library = library, copiedFiles = copiedFiles, isModuleLevel = true, targetFile = targetFile)
for (i in (files.size - 1) downTo 0) {
val file = files.get(i)
val fileName = file.fileName.toString()
if (fileName.endsWith("-rt.jar") || fileName.contains("-agent")) {
files.removeAt(i)
addLibrary(library, outputDir.resolve(removeVersionFromJar(fileName)), listOf(file))
}
}
for (file in files) {
sources.add(ZipSource(file) { size ->
libraryEntries.add(ModuleLibraryFileEntry(path = targetFile, moduleName = moduleName, libraryFile = file, size = size))
})
}
}
}
private fun mergeLibsByPredicate(jarName: String,
libraryToMerge: MutableMap<JpsLibrary, List<Path>>,
outputDir: Path,
@@ -296,57 +356,15 @@ class JarPackager private constructor(private val context: BuildContext) {
}
private fun filesToSourceWithMappings(uberJarFile: Path, libraryToMerge: Map<JpsLibrary, List<Path>>) {
val sources = getJarDescriptorSources(uberJarFile)
val sources = getJarDescriptorSources(targetFile = uberJarFile)
for ((key, value) in libraryToMerge) {
filesToSourceWithMapping(to = sources, files = value, library = key, targetFile = uberJarFile)
}
}
private fun packModuleOutputAndUnpackedProjectLibraries(modules: Collection<String>,
jarPath: String,
jarFile: Path,
moduleOutputPatcher: ModuleOutputPatcher,
layout: BaseLayout,
moduleNameToSize: MutableMap<String, Int>,
sourceList: MutableList<Source>): List<DistributionFileEntry> {
Span.current().addEvent("include module outputs", Attributes.of(AttributeKey.stringArrayKey("modules"), java.util.List.copyOf(modules)))
val searchableOptionsDir = context.searchableOptionDir
for (moduleName in modules) {
addModuleSources(moduleName = moduleName,
moduleNameToSize = moduleNameToSize,
moduleOutputDir = context.getModuleOutputDir(context.findRequiredModule(moduleName)),
modulePatches = moduleOutputPatcher.getPatchedDir(moduleName),
modulePatchContents = moduleOutputPatcher.getPatchedContent(moduleName),
searchableOptionsRootDir = searchableOptionsDir,
extraExcludes = layout.moduleExcludes.get(moduleName) ?: emptyList(),
sourceList = sourceList)
}
val list = mutableListOf<DistributionFileEntry>()
for (libraryName in layout.projectLibrariesToUnpack.get(jarPath)) {
val library = context.project.libraryCollection.findLibrary(libraryName)
check(library != null) {
"Project library \'$libraryName\' from $jarPath should be unpacked but it isn\'t found"
}
for (file in library.getPaths(JpsOrderRootType.COMPILED)) {
sourceList.add(ZipSource(file = file) { size ->
val libraryData = ProjectLibraryData(libraryName = library.name,
outPath = null,
packMode = LibraryPackMode.MERGED,
reason = "explicitUnpack")
libraryEntries.add(ProjectLibraryEntry(path = jarFile, data = libraryData, libraryFile = file, size = size))
})
}
}
return list
}
private fun packProjectLibraries(jarToModuleNames: Map<String, List<String>>,
outputDir: Path,
private fun packProjectLibraries(outputDir: Path,
layout: BaseLayout,
copiedFiles: MutableMap<Path, CopiedFor>,
extraLibSources: MutableMap<String, MutableList<Source>>): MutableMap<JpsLibrary, List<Path>> {
copiedFiles: MutableMap<Path, CopiedFor>): MutableMap<JpsLibrary, List<Path>> {
val toMerge = LinkedHashMap<JpsLibrary, List<Path>>()
val projectLibs = if (layout.includedProjectLibraries.isEmpty()) {
emptyList()
@@ -395,79 +413,9 @@ class JarPackager private constructor(private val context: BuildContext) {
}
}
}
for ((targetFilename, value) in jarToModuleNames) {
if (targetFilename.contains('/')) {
continue
}
for (moduleName in value) {
if (layout.modulesWithExcludedModuleLibraries.contains(moduleName)) {
continue
}
val excluded = layout.excludedModuleLibraries.get(moduleName)
for (element in context.findRequiredModule(moduleName).dependenciesList.dependencies) {
if (element !is JpsLibraryDependency) {
continue
}
packModuleLibs(moduleName = moduleName,
targetFilename = targetFilename,
libraryDependency = element,
excluded = excluded,
layout = layout,
outputDir = outputDir,
copiedFiles = copiedFiles,
extraLibSources = extraLibSources)
}
}
}
return toMerge
}
private fun packModuleLibs(moduleName: String,
targetFilename: String,
libraryDependency: JpsLibraryDependency,
excluded: Collection<String>,
layout: BaseLayout,
outputDir: Path,
copiedFiles: MutableMap<Path, CopiedFor>,
extraLibSources: MutableMap<String, MutableList<Source>>) {
if (libraryDependency.libraryReference.parentReference!!.resolve() !is JpsModule) {
return
}
if (JpsJavaExtensionService.getInstance().getDependencyExtension(libraryDependency)?.scope
?.isIncludedIn(JpsJavaClasspathKind.PRODUCTION_RUNTIME) != true) {
return
}
val library = libraryDependency.library!!
val libraryName = getLibraryFileName(library)
if (excluded.contains(libraryName) || layout.includedModuleLibraries.any { it.libraryName == libraryName }) {
return
}
val targetFile = outputDir.resolve(targetFilename)
val files = getLibraryFiles(library = library, copiedFiles = copiedFiles, isModuleLevel = true, targetFile = targetFile)
for (i in (files.size - 1) downTo 0) {
val file = files.get(i)
val fileName = file.fileName.toString()
if (fileName.endsWith("-rt.jar") || fileName.contains("-agent")) {
files.removeAt(i)
addLibrary(library, outputDir.resolve(removeVersionFromJar(fileName)), listOf(file))
}
}
if (!files.isEmpty()) {
val sources = extraLibSources.computeIfAbsent(targetFilename) { mutableListOf() }
for (file in files) {
sources.add(ZipSource(file) { size ->
libraryEntries.add(ModuleLibraryFileEntry(targetFile, moduleName, file, size))
})
}
}
}
private fun filesToSourceWithMapping(to: MutableList<Source>, files: List<Path>, library: JpsLibrary, targetFile: Path) {
val moduleName = (library.createReference().parentReference as? JpsModuleReference)?.moduleName
for (file in files) {
@@ -499,7 +447,12 @@ class JarPackager private constructor(private val context: BuildContext) {
}
private fun getJarDescriptorSources(targetFile: Path): MutableList<Source> {
return jarDescriptors.computeIfAbsent(targetFile) { JarDescriptor(targetFile) }.sources
return jarDescriptors.computeIfAbsent(targetFile) {
createJarDescriptor(outputDir = outputDir,
targetFile = targetFile,
collectNativeFiles = collectNativeFiles,
context = context)
}.sources
}
}
@@ -602,9 +555,14 @@ private suspend fun unpackNativeLibraries(sourceFile: Path, paths: List<String>,
}
}
private data class JarDescriptor(@JvmField val jarFile: Path) {
private data class JarDescriptor(@JvmField val file: Path,
@JvmField val pathInClassLog: String,
@JvmField val collectNativeFiles: Boolean) {
@JvmField
val sources: MutableList<Source> = mutableListOf()
var includedModules: MutableList<String>? = null
@JvmField
var includedModules: PersistentList<ModuleItem> = persistentListOf()
}
private fun removeVersionFromJar(fileName: String): String {
@@ -665,64 +623,22 @@ private fun isLibraryMergeable(libName: String): Boolean {
!libName.contains("groovy", ignoreCase = true)
}
internal val commonModuleExcludes = java.util.List.of(
FileSystems.getDefault().getPathMatcher("glob:**/icon-robots.txt"),
FileSystems.getDefault().getPathMatcher("glob:icon-robots.txt"),
FileSystems.getDefault().getPathMatcher("glob:.unmodified"),
// compilation cache on TC
FileSystems.getDefault().getPathMatcher("glob:.hash"),
FileSystems.getDefault().getPathMatcher("glob:classpath.index"),
)
private fun addModuleSources(moduleName: String,
moduleNameToSize: MutableMap<String, Int>,
moduleOutputDir: Path,
modulePatches: Collection<Path>,
modulePatchContents: Map<String, ByteArray>,
searchableOptionsRootDir: Path,
extraExcludes: Collection<String>,
sourceList: MutableList<Source>) {
val sizeConsumer = IntConsumer {
moduleNameToSize.merge(moduleName, it) { oldValue, value -> oldValue + value }
}
for (entry in modulePatchContents) {
sourceList.add(InMemoryContentSource(entry.key, entry.value, sizeConsumer))
}
// must be before module output to override
for (moduleOutputPatch in modulePatches) {
sourceList.add(DirSource(moduleOutputPatch, Collections.emptyList(), sizeConsumer))
}
val searchableOptionsModuleDir = searchableOptionsRootDir.resolve(moduleName)
if (Files.exists(searchableOptionsModuleDir)) {
sourceList.add(DirSource(searchableOptionsModuleDir, Collections.emptyList(), sizeConsumer))
}
val excludes = if (extraExcludes.isEmpty()) {
commonModuleExcludes
}
else {
commonModuleExcludes.plus(extraExcludes.map { FileSystems.getDefault().getPathMatcher("glob:$it") })
}
sourceList.add(DirSource(dir = moduleOutputDir, excludes = excludes, sizeConsumer = sizeConsumer))
internal val commonModuleExcludes: List<PathMatcher> = FileSystems.getDefault().let { fs ->
java.util.List.of(
fs.getPathMatcher("glob:**/icon-robots.txt"),
fs.getPathMatcher("glob:icon-robots.txt"),
fs.getPathMatcher("glob:.unmodified"),
// compilation cache on TC
fs.getPathMatcher("glob:.hash"),
fs.getPathMatcher("glob:classpath.index"),
)
}
private data class CopiedFor(@JvmField val library: JpsLibrary, @JvmField val targetFile: Path?)
private data class BuildJarDescriptor(@JvmField val descriptor: JarDescriptor,
@JvmField val relativePath: String,
@JvmField val collectNativeFiles: Boolean) {
val file: Path
get() = descriptor.jarFile
val sources: List<Source>
get() = descriptor.sources
}
private const val DO_NOT_EXPORT_TO_CONSOLE = "_CES_"
private suspend fun buildJars(descriptors: List<BuildJarDescriptor>, dryRun: Boolean): Map<ZipSource, MutableList<String>> {
private suspend fun buildJars(descriptors: Collection<JarDescriptor>, dryRun: Boolean): Map<ZipSource, MutableList<String>> {
val uniqueFiles = HashMap<Path, List<Source>>()
for (descriptor in descriptors) {
val existing = uniqueFiles.putIfAbsent(descriptor.file, descriptor.sources)
@@ -743,12 +659,17 @@ private suspend fun buildJars(descriptors: List<BuildJarDescriptor>, dryRun: Boo
.setAttribute("jar", file.toString())
.setAttribute(AttributeKey.stringArrayKey("sources"), item.sources.map(Source::toString))
.use {
buildJar(targetFile = file, sources = item.sources, dryRun = dryRun, nativeFiles = nativeFiles)
if (item.sources.isEmpty()) {
return@async emptyMap()
}
else {
buildJar(targetFile = file, sources = item.sources, dryRun = dryRun, nativeFiles = nativeFiles)
}
}
// app.jar is combined later with other JARs and then re-ordered
if (!dryRun && item.relativePath.isNotEmpty() && item.relativePath != "lib/app.jar") {
reorderJar(relativePath = item.relativePath, file = file)
if (!dryRun && item.pathInClassLog.isNotEmpty() && item.pathInClassLog != "lib/app.jar") {
reorderJar(relativePath = item.pathInClassLog, file = file)
}
nativeFiles
}
@@ -774,3 +695,30 @@ fun buildJar(targetFile: Path,
compress = compress,
)
}
private fun createJarDescriptor(outputDir: Path,
targetFile: Path,
collectNativeFiles: Boolean,
context: BuildContext): JarDescriptor {
var pathInClassLog = ""
val isReorderingEnabled = !context.options.buildStepsToSkip.contains(BuildOptions.GENERATE_JAR_ORDER_STEP)
if (isReorderingEnabled) {
if (context.paths.distAllDir == outputDir.parent) {
pathInClassLog = outputDir.parent.relativize(targetFile).toString().replace(File.separatorChar, '/')
}
else if (outputDir.startsWith(context.paths.distAllDir)) {
pathInClassLog = context.paths.distAllDir.relativize(targetFile).toString().replace(File.separatorChar, '/')
}
else {
val parent = outputDir.parent
if (parent?.fileName.toString() == "plugins") {
pathInClassLog = outputDir.parent.parent.relativize(targetFile).toString().replace(File.separatorChar, '/')
}
}
}
val fileName = targetFile.fileName.toString()
return JarDescriptor(file = targetFile,
pathInClassLog = pathInClassLog,
collectNativeFiles = collectNativeFiles && (fileName == APP_JAR || fileName.startsWith("3rd-party-")))
}

View File

@@ -1,5 +1,5 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("ReplaceGetOrSet")
@file:Suppress("ReplaceGetOrSet", "LiftReturnOrAssignment")
package org.jetbrains.intellij.build.impl
@@ -70,7 +70,7 @@ class ModuleOutputPatcher {
?: error("patched plugin.xml not found for $moduleName module")
}
fun getPatchedDir(moduleName: String): Collection<Path> = patchDirs.get(moduleName) ?: emptyList()
internal fun getPatchedDir(moduleName: String): Collection<Path> = patchDirs.get(moduleName) ?: emptyList()
fun getPatchedContent(moduleName: String): Map<String, ByteArray> = patches.get(moduleName) ?: emptyMap()
internal fun getPatchedContent(moduleName: String): Map<String, ByteArray> = patches.get(moduleName) ?: emptyMap()
}

View File

@@ -39,27 +39,15 @@ private val pathElements = hashSetOf("interface-class", "implementation-class")
private val predefinedTypes = hashSetOf("java.lang.Object")
private val ignoreModules = hashSetOf("intellij.java.testFramework", "intellij.platform.uast.tests")
class ModuleStructureValidator(
private val context: BuildContext,
moduleJars: Map<String, List<String>>,
) {
private val moduleJars = MultiMap<String, String>()
private val moduleNames = HashSet<String>()
class ModuleStructureValidator(private val context: BuildContext, modules: Collection<ModuleItem>) {
// filter out jars with relative paths in name
private val modules: Collection<ModuleItem> = modules.filter {
!it.relativeOutputFile.contains("\\") || !it.relativeOutputFile.contains('/')
}
private val errors = ArrayList<AssertionError>()
private val libraryFiles = HashMap<JpsLibrary, Set<String>>()
init {
for ((jar, modules) in moduleJars) {
// filter out jars with relative paths in name
if (jar.contains("\\") || jar.contains('/')) {
continue
}
this.moduleJars.put(jar, modules)
this.moduleNames.addAll(modules)
}
}
private fun getLibraryFiles(library: JpsLibrary): Set<String> {
@Suppress("NAME_SHADOWING")
return libraryFiles.computeIfAbsent(library) { library ->
@@ -79,23 +67,24 @@ class ModuleStructureValidator(
fun validate(): List<AssertionError> {
errors.clear()
context.messages.info("Validating jars...")
val messages = context.messages
messages.info("Validating jars...")
validateJarModules()
context.messages.info("Validating modules...")
messages.info("Validating modules...")
val visitedModules = HashSet<JpsModule>()
for (moduleName in moduleNames) {
for (moduleName in modules.map { it.moduleName }.distinct()) {
if (ignoreModules.contains(moduleName)) {
continue
}
validateModuleDependencies(visitedModules, context.findRequiredModule(moduleName))
}
context.messages.info("Validating xml descriptors...")
messages.info("Validating xml descriptors...")
validateXmlDescriptors()
if (errors.isEmpty()) {
context.messages.info("Validation finished successfully")
messages.info("Validation finished successfully")
}
return errors
@@ -103,10 +92,8 @@ class ModuleStructureValidator(
private fun validateJarModules() {
val modulesInJars = MultiMap<String, String>()
for (jar in moduleJars.keySet()) {
for (module in moduleJars.get(jar)) {
modulesInJars.putValue(module, jar)
}
for (item in modules) {
modulesInJars.putValue(item.moduleName, item.relativeOutputFile)
}
for (module in modulesInJars.keySet()) {
@@ -140,7 +127,7 @@ class ModuleStructureValidator(
continue
}
if (!moduleNames.contains(dependantModule.name)) {
if (modules.none { it.moduleName != dependantModule.name }) {
errors.add(AssertionError("Missing dependency found: ${module.name} -> ${dependantModule.name} [${role.scope.name}]", null))
continue
}
@@ -153,7 +140,7 @@ class ModuleStructureValidator(
private fun validateXmlDescriptors() {
val roots = ArrayList<Path>()
val libraries = HashSet<JpsLibrary>()
for (moduleName in moduleNames) {
for (moduleName in modules.map { it.moduleName }.distinct()) {
val module = context.findRequiredModule(moduleName)
for (root in module.sourceRoots) {
roots.add(root.path)
@@ -220,7 +207,7 @@ class ModuleStructureValidator(
private fun validateXmlRegistrations(descriptors: HashSet<Path>) {
val classes = HashSet<String>(predefinedTypes)
val visitedLibraries = HashSet<String>()
for (moduleName in moduleNames) {
for (moduleName in modules.map { it.moduleName }.distinct()) {
val jpsModule = context.findRequiredModule(moduleName)
val outputDirectory = JpsJavaExtensionService.getInstance().getOutputDirectory(jpsModule, false)!!.toPath()
@@ -270,7 +257,7 @@ class ModuleStructureValidator(
}
}
context.messages.info("Found ${classes.size} classes in ${moduleNames.size} modules and ${visitedLibraries.size} libraries")
context.messages.info("Found ${classes.size} classes in ${modules.map { it.moduleName }.distinct().size} modules and ${visitedLibraries.size} libraries")
for (descriptor in descriptors) {
val xml = Files.newInputStream(descriptor).use(::readXmlAsModel)

View File

@@ -27,35 +27,29 @@ class PlatformLayout: BaseLayout() {
includedProjectLibraries.add(data)
}
/**
* See [Spec.withoutProjectLibrary]
*/
override fun withModule(moduleName: String) {
withModule(moduleName, APP_JAR)
}
fun withoutProjectLibrary(libraryName: String) {
excludedProjectLibraries.add(libraryName)
}
inline fun collectProjectLibrariesFromIncludedModules(context: BuildContext, consumer: (JpsLibrary, JpsModule) -> Unit) {
val libsToUnpack = projectLibrariesToUnpack.values()
for (moduleName in includedModuleNames) {
val libsToUnpack = includedProjectLibraries.mapTo(HashSet()) { it.libraryName }
for (moduleName in includedModules.asSequence().map { it.moduleName }.distinct()) {
val module = context.findRequiredModule(moduleName)
for (library in JpsJavaExtensionService.dependencies(module).includedIn(JpsJavaClasspathKind.PRODUCTION_RUNTIME).libraries) {
if (library.createReference().parentReference is JpsModuleReference ||
libsToUnpack.contains(library.name) ||
excludedProjectLibraries.contains(library.name)) {
continue
if (!isSkippedLibrary(library, libsToUnpack)) {
consumer(library, module)
}
consumer(library, module)
}
}
}
class Spec(@Internal val layout: PlatformLayout) : BaseLayoutSpec(layout) {
/**
* Exclude project library [libraryName] even if it's added to dependencies of some module or plugin included into the product
*/
fun withoutProjectLibrary(libraryName: String) {
layout.withoutProjectLibrary(libraryName)
}
fun isSkippedLibrary(library: JpsLibrary, libsToUnpack: Collection<String>): Boolean {
return library.createReference().parentReference is JpsModuleReference ||
libsToUnpack.contains(library.name) ||
excludedProjectLibraries.contains(library.name)
}
}

View File

@@ -1,17 +1,23 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("ReplaceJavaStaticMethodWithKotlinAnalog")
@file:Suppress("ReplaceJavaStaticMethodWithKotlinAnalog", "ReplaceGetOrSet")
package org.jetbrains.intellij.build.impl
import com.intellij.util.xml.dom.readXmlAsModel
import kotlinx.collections.immutable.PersistentList
import kotlinx.collections.immutable.persistentListOf
import kotlinx.collections.immutable.persistentMapOf
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import org.jetbrains.intellij.build.BuildContext
import org.jetbrains.intellij.build.ProductModulesLayout
import org.jetbrains.jps.model.java.JpsJavaExtensionService
import org.w3c.dom.Element
import java.nio.file.Files
import org.jetbrains.jps.model.java.*
import org.jetbrains.jps.util.JpsPathUtil
import java.nio.file.Path
import java.util.*
import javax.xml.parsers.DocumentBuilderFactory
import kotlin.collections.HashSet
import kotlin.collections.LinkedHashSet
private const val UTIL_JAR = "util.jar"
private const val UTIL_RT_JAR = "util_rt.jar"
@@ -19,39 +25,22 @@ private const val UTIL_RT_JAR = "util_rt.jar"
private val PLATFORM_API_MODULES = persistentListOf(
"intellij.platform.analysis",
"intellij.platform.builtInServer",
"intellij.platform.core",
"intellij.platform.diff",
"intellij.platform.vcs.dvcs",
"intellij.platform.editor",
"intellij.platform.externalSystem",
"intellij.platform.externalSystem.dependencyUpdater",
"intellij.platform.codeStyle",
"intellij.platform.indexing",
"intellij.platform.jps.model",
"intellij.platform.lang.core",
"intellij.platform.lang",
"intellij.platform.lvcs",
"intellij.platform.ml",
"intellij.platform.ide",
"intellij.platform.ide.core",
"intellij.platform.projectModel",
"intellij.platform.remote.core",
"intellij.platform.remoteServers.agent.rt",
"intellij.platform.remoteServers",
"intellij.platform.tasks",
"intellij.platform.usageView",
"intellij.platform.vcs.core",
"intellij.platform.vcs",
"intellij.platform.vcs.log",
"intellij.platform.vcs.log.graph",
"intellij.platform.execution",
"intellij.platform.debugger",
"intellij.platform.webSymbols",
"intellij.xml.analysis",
"intellij.xml",
"intellij.xml.psi",
"intellij.xml.structureView",
"intellij.platform.concurrency",
)
/**
@@ -59,30 +48,17 @@ private val PLATFORM_API_MODULES = persistentListOf(
*/
private val PLATFORM_IMPLEMENTATION_MODULES = persistentListOf(
"intellij.platform.analysis.impl",
"intellij.platform.builtInServer.impl",
"intellij.platform.core.impl",
"intellij.platform.ide.core.impl",
"intellij.platform.diff.impl",
"intellij.platform.editor.ex",
"intellij.platform.codeStyle.impl",
"intellij.platform.indexing.impl",
"intellij.platform.elevation",
"intellij.platform.elevation.client",
"intellij.platform.elevation.common",
"intellij.platform.elevation.daemon",
"intellij.platform.externalProcessAuthHelper",
"intellij.platform.refactoring",
"intellij.platform.inspect",
"intellij.platform.lang.impl",
"intellij.platform.workspaceModel.storage",
"intellij.platform.workspaceModel.jps",
// lvcs.xml - convert into product module
"intellij.platform.lvcs.impl",
"intellij.platform.ide.impl",
"intellij.platform.projectModel.impl",
"intellij.platform.macro",
"intellij.platform.execution.impl",
"intellij.platform.wsl.impl",
"intellij.platform.externalSystem.impl",
"intellij.platform.scriptDebugger.protocolReaderRuntime",
"intellij.regexp",
"intellij.platform.remoteServers.impl",
@@ -93,298 +69,338 @@ private val PLATFORM_IMPLEMENTATION_MODULES = persistentListOf(
"intellij.platform.structureView.impl",
"intellij.platform.tasks.impl",
"intellij.platform.testRunner",
"intellij.platform.debugger.impl",
"intellij.platform.configurationStore.impl",
"intellij.platform.serviceContainer",
"intellij.platform.objectSerializer",
"intellij.platform.diagnostic",
"intellij.platform.diagnostic.telemetry",
"intellij.platform.core.ui",
"intellij.platform.credentialStore",
"intellij.platform.credentialStore.ui",
"intellij.platform.dependenciesToolwindow",
"intellij.platform.rd.community",
"intellij.platform.ml.impl",
"intellij.remoteDev.util",
"intellij.platform.feedback",
"intellij.platform.warmup",
"intellij.platform.buildScripts.downloader",
"intellij.idea.community.build.dependencies",
"intellij.platform.usageView.impl",
"intellij.platform.ml.impl",
"intellij.platform.tips",
"intellij.platform.bootstrap",
"intellij.relaxng",
"intellij.json",
"intellij.spellchecker",
"intellij.platform.webSymbols",
"intellij.xml.dom.impl",
"intellij.platform.vcs.dvcs.impl",
"intellij.platform.vcs.log.graph.impl",
"intellij.platform.vcs.log.impl",
"intellij.smart.update",
"intellij.platform.collaborationTools",
"intellij.platform.collaborationTools.auth",
"intellij.platform.markdown.utils",
)
private const val UTIL_8 = "util-8.jar"
object PlatformModules {
const val PRODUCT_JAR = "product.jar"
internal val PLATFORM_CUSTOM_PACK_MODE: Map<String, LibraryPackMode> = persistentMapOf(
"jetbrains-annotations-java5" to LibraryPackMode.STANDALONE_SEPARATE_WITHOUT_VERSION_NAME,
"intellij-coverage" to LibraryPackMode.STANDALONE_SEPARATE,
)
internal val CUSTOM_PACK_MODE: Map<String, LibraryPackMode> = persistentMapOf(
"jetbrains-annotations-java5" to LibraryPackMode.STANDALONE_SEPARATE_WITHOUT_VERSION_NAME,
"intellij-coverage" to LibraryPackMode.STANDALONE_SEPARATE,
)
internal fun collectPlatformModules(to: MutableCollection<String>) {
to.addAll(PLATFORM_API_MODULES)
to.addAll(PLATFORM_IMPLEMENTATION_MODULES)
}
internal fun collectPlatformModules(to: MutableCollection<String>) {
to.addAll(PLATFORM_API_MODULES)
to.addAll(PLATFORM_IMPLEMENTATION_MODULES)
internal fun hasPlatformCoverage(productLayout: ProductModulesLayout, enabledPluginModules: Set<String>, context: BuildContext): Boolean {
val modules = LinkedHashSet<String>()
modules.addAll(productLayout.getIncludedPluginModules(enabledPluginModules))
modules.addAll(PLATFORM_API_MODULES)
modules.addAll(PLATFORM_IMPLEMENTATION_MODULES)
modules.addAll(productLayout.productApiModules)
modules.addAll(productLayout.productImplementationModules)
val coverageModuleName = "intellij.platform.coverage"
if (modules.contains(coverageModuleName)) {
return true
}
internal fun hasPlatformCoverage(productLayout: ProductModulesLayout, enabledPluginModules: Set<String>, context: BuildContext): Boolean {
val modules = LinkedHashSet<String>()
modules.addAll(productLayout.getIncludedPluginModules(enabledPluginModules))
modules.addAll(PLATFORM_API_MODULES)
modules.addAll(PLATFORM_IMPLEMENTATION_MODULES)
modules.addAll(productLayout.productApiModules)
modules.addAll(productLayout.productImplementationModules)
modules.addAll(productLayout.additionalPlatformJars.values())
for (moduleName in modules) {
var contains = false
JpsJavaExtensionService.dependencies(context.findRequiredModule(moduleName))
.productionOnly()
.processModules { module ->
if (!contains && module.name == coverageModuleName) {
contains = true
}
}
val coverageModuleName = "intellij.platform.coverage"
if (modules.contains(coverageModuleName)) {
if (contains) {
return true
}
}
for (moduleName in modules) {
var contains = false
JpsJavaExtensionService.dependencies(context.findRequiredModule(moduleName))
.productionOnly()
.processModules { module ->
if (!contains && module.name == coverageModuleName) {
contains = true
}
}
return false
}
if (contains) {
return true
}
private fun addModule(relativeJarPath: String,
moduleNames: Collection<String>,
productLayout: ProductModulesLayout,
layout: PlatformLayout) {
layout.withModules(moduleNames.asSequence()
.filter { !productLayout.excludedModuleNames.contains(it) }
.map { ModuleItem(moduleName = it, relativeOutputFile = relativeJarPath, reason = "addModule") }.toList())
}
internal suspend fun createPlatformLayout(productLayout: ProductModulesLayout,
hasPlatformCoverage: Boolean,
additionalProjectLevelLibraries: SortedSet<ProjectLibraryData>,
context: BuildContext): PlatformLayout {
val layout = PlatformLayout()
// used only in modules that packed into Java
layout.withoutProjectLibrary("jps-javac-extension")
layout.withoutProjectLibrary("Eclipse")
for (customizer in productLayout.platformLayoutSpec) {
customizer(layout, context)
}
for ((module, patterns) in productLayout.moduleExcludes) {
layout.excludeFromModule(module, patterns)
}
addModule(UTIL_RT_JAR, listOf(
"intellij.platform.util.rt",
"intellij.platform.util.trove",
), productLayout, layout)
addModule(UTIL_8, listOf(
"intellij.platform.util.rt.java8",
"intellij.platform.util.classLoader",
"intellij.platform.util.jdom",
"intellij.platform.util.xmlDom",
), productLayout, layout)
// fastutil-min cannot be in libsThatUsedInJps - guava is used by JPS and also in this JAR,
// but it leads to conflict in some old 3rd-party JDBC driver, so, pack fastutil-min into another JAR
layout.withProjectLibrary(libraryName = "fastutil-min", jarName = UTIL_8)
// util.jar is loaded by JVM classloader as part of loading our custom PathClassLoader class - reduce file size
addModule(UTIL_JAR, listOf(
"intellij.platform.util.zip",
"intellij.platform.util",
"intellij.platform.util.base",
"intellij.platform.extensions",
"intellij.platform.tracing.rt",
"intellij.platform.core",
// Scala uses GeneralCommandLine in JPS plugin
"intellij.platform.ide.util.io",
"intellij.platform.boot",
), productLayout, layout)
addModule("externalProcess-rt.jar", listOf(
"intellij.platform.externalProcessAuthHelper.rt"
), productLayout, layout)
addModule("stats.jar", listOf(
"intellij.platform.statistics",
"intellij.platform.statistics.uploader",
"intellij.platform.statistics.config",
), productLayout, layout)
if (!productLayout.excludedModuleNames.contains("intellij.java.guiForms.rt")) {
layout.withModule("intellij.java.guiForms.rt", "forms_rt.jar")
}
addModule("jps-model.jar", listOf(
"intellij.platform.jps.model",
"intellij.platform.jps.model.serialization",
"intellij.platform.jps.model.impl"
), productLayout, layout)
addModule("external-system-rt.jar", listOf(
"intellij.platform.externalSystem.rt",
"intellij.platform.objectSerializer.annotations"
), productLayout, layout)
addModule("cds/classesLogAgent.jar", listOf("intellij.platform.cdsAgent"), productLayout, layout)
val productPluginSourceModuleName = context.productProperties.productPluginSourceModuleName
?: context.productProperties.applicationInfoModule
val productPluginContentModules = getProductPluginContentModules(context, productPluginSourceModuleName)
val explicit = mutableListOf<ModuleItem>()
for (moduleName in productLayout.productImplementationModules) {
if (productLayout.excludedModuleNames.contains(moduleName)) {
continue
}
explicit.add(ModuleItem(moduleName = moduleName,
relativeOutputFile = if (isModuleCloseSource(moduleName, context)) PRODUCT_JAR else APP_JAR,
reason = "productImplementationModules"))
}
explicit.addAll(toModuleItemSequence(PLATFORM_API_MODULES, productLayout, reason = "PLATFORM_API_MODULES"))
explicit.addAll(toModuleItemSequence(PLATFORM_IMPLEMENTATION_MODULES, productLayout, reason = "PLATFORM_IMPLEMENTATION_MODULES"))
explicit.addAll(toModuleItemSequence(productLayout.productApiModules, productLayout, reason = "productApiModules"))
if (hasPlatformCoverage && !productLayout.excludedModuleNames.contains("intellij.platform.coverage")) {
explicit.add(ModuleItem(moduleName = "intellij.platform.coverage", relativeOutputFile = APP_JAR, reason = "coverage"))
}
val implicit = computeImplicitRequiredModules(
explicit = explicit.map { it.moduleName }.toList(),
layout = layout,
productPluginContentModules = productPluginContentModules.mapTo(HashSet()) { it.moduleName },
productLayout = productLayout,
context = context,
)
layout.withModules((explicit +
productPluginContentModules +
implicit.asSequence().map {
ModuleItem(moduleName = it.first,
relativeOutputFile = APP_JAR,
reason = "<- " + it.second.asReversed().joinToString(separator = " <- "))
}).sortedBy { it.moduleName }.toList())
layout.withProjectLibrary(libraryName = "ion", jarName = UTIL_RT_JAR)
for (item in additionalProjectLevelLibraries) {
if (!layout.excludedProjectLibraries.contains(item.libraryName)) {
layout.includedProjectLibraries.add(item)
}
}
// as a separate step, not a part of computing implicitModules, as we should collect libraries from a such implicitly included modules
layout.collectProjectLibrariesFromIncludedModules(context) { lib, module ->
val name = lib.name
if (module.name == "intellij.platform.buildScripts.downloader" && (name == "zstd-jni" || name == "zstd-jni-windows-aarch64")) {
return@collectProjectLibrariesFromIncludedModules
}
layout.includedProjectLibraries
.addOrGet(ProjectLibraryData(name, PLATFORM_CUSTOM_PACK_MODE.getOrDefault(name, LibraryPackMode.MERGED)))
.dependentModules.computeIfAbsent("core") { mutableListOf() }.add(module.name)
}
return layout
}
private fun isModuleCloseSource(moduleName: String, context: BuildContext): Boolean {
if (moduleName.endsWith(".resources") || moduleName.endsWith(".icons")) {
return false
}
private fun jar(relativeJarPath: String, moduleNames: Collection<String>, productLayout: ProductModulesLayout, layout: PlatformLayout) {
for (moduleName in moduleNames) {
if (!productLayout.excludedModuleNames.contains(moduleName)) {
layout.withModule(moduleName, relativeJarPath)
}
}
val sourceRoots = context.findRequiredModule(moduleName).sourceRoots.filter { it.rootType == JavaSourceRootType.SOURCE }
if (sourceRoots.isEmpty()) {
return false
}
private fun addModule(moduleName: String, productLayout: ProductModulesLayout, layout: PlatformLayout) {
if (!productLayout.excludedModuleNames.contains(moduleName)) {
layout.withModule(moduleName)
}
return sourceRoots.any { moduleSourceRoot ->
!Path.of(JpsPathUtil.urlToPath(moduleSourceRoot.url)).startsWith(context.paths.communityHomeDir)
}
}
private fun addModule(moduleName: String, relativeJarPath: String, productLayout: ProductModulesLayout, layout: PlatformLayout) {
if (!productLayout.excludedModuleNames.contains(moduleName)) {
layout.withModule(moduleName, relativeJarPath)
private fun toModuleItemSequence(list: Collection<String>, productLayout: ProductModulesLayout, reason: String): Sequence<ModuleItem> {
return list.asSequence()
.filter { !productLayout.excludedModuleNames.contains(it) }
.map { ModuleItem(moduleName = it, relativeOutputFile = APP_JAR, reason = reason) }
}
private suspend fun computeImplicitRequiredModules(explicit: List<String>,
layout: PlatformLayout,
productPluginContentModules: Set<String>,
productLayout: ProductModulesLayout,
context: BuildContext): List<Pair<String, PersistentList<String>>> {
val rootChain = persistentListOf<String>()
val rootList = layout.filteredIncludedModuleNames(TEST_FRAMEWORK_JAR)
.plus(explicit)
.filter {
!productLayout.excludedModuleNames.contains(it) &&
!productPluginContentModules.contains(it) &&
!it.startsWith("intellij.pycharm.") &&
!it.startsWith("intellij.python.") &&
!it.startsWith("intellij.codeServer.") &&
!it.startsWith("intellij.clion.") &&
!it.startsWith("intellij.appcode.") &&
it != "fleet.backend" &&
it != "intellij.codeServer" &&
it != "intellij.goland"
}
}
.distinct()
.sorted()
.map { it to rootChain }
.toList()
internal fun createPlatformLayout(productLayout: ProductModulesLayout,
hasPlatformCoverage: Boolean,
additionalProjectLevelLibraries: SortedSet<ProjectLibraryData>,
context: BuildContext): PlatformLayout {
val layout = PlatformLayout()
// used only in modules that packed into Java
layout.withoutProjectLibrary("jps-javac-extension")
layout.withoutProjectLibrary("Eclipse")
val layoutSpec = PlatformLayout.Spec(layout)
for (platformLayoutSpec in productLayout.platformLayoutSpec) {
platformLayoutSpec(layoutSpec, context)
}
val unique = HashSet<String>()
layout.includedModules.mapTo(unique) { it.moduleName }
unique.addAll(explicit)
unique.addAll(productPluginContentModules)
unique.addAll(productLayout.excludedModuleNames)
unique.add("fleet.backend")
// Module intellij.featuresTrainer contains, so it is a plugin, but plugin must be not included in a platform
// (chain: [intellij.pycharm.community, intellij.python.featuresTrainer])
unique.add("intellij.pycharm.community")
unique.add("intellij.python.featuresTrainer")
unique.add("intellij.pycharm.ds")
val alreadyPackedModules = HashSet<String>()
for (entry in productLayout.additionalPlatformJars.entrySet()) {
jar(entry.key, entry.value, productLayout, layout)
alreadyPackedModules.addAll(entry.value)
}
val result = mutableListOf<Pair<String, PersistentList<String>>>()
compute(list = rootList, context = context, unique = unique, result = result)
for (moduleName in PLATFORM_API_MODULES) {
// intellij.platform.core is used in Kotlin and Scala JPS plugins (PathUtil) https://youtrack.jetbrains.com/issue/IDEA-292483
if (!productLayout.excludedModuleNames.contains(moduleName) && moduleName != "intellij.platform.core") {
layout.withModule(moduleName, if (moduleName == "intellij.platform.jps.model") "jps-model.jar" else BaseLayout.APP_JAR)
}
}
jar(BaseLayout.APP_JAR, productLayout.productApiModules, productLayout, layout)
for (module in productLayout.productImplementationModules) {
if (!productLayout.excludedModuleNames.contains(module) && !alreadyPackedModules.contains(module)) {
val isRelocated = module == "intellij.xml.dom.impl" ||
module == "intellij.platform.structuralSearch" ||
// todo why intellij.tools.testsBootstrap is included to RM?
module == "intellij.tools.testsBootstrap" ||
module == "intellij.platform.duplicates.analysis"
if (isRelocated) {
layout.withModule(module, BaseLayout.APP_JAR)
}
else if (!context.productProperties.useProductJar || module.startsWith("intellij.platform.commercial")) {
layout.withModule(module, productLayout.mainJarName)
}
else {
layout.withModule(module, PRODUCT_JAR)
if (context.options.validateImplicitPlatformModule) {
withContext(Dispatchers.IO) {
for ((name, chain) in result) {
launch {
val file = context.findFileInModuleSources(name, "META-INF/plugin.xml")
require(file == null) {
"Module $name contains $file, so it is a plugin, but plugin must be not included in a platform (chain: $chain)"
}
}
}
}
for ((module, patterns) in productLayout.moduleExcludes) {
layout.excludeFromModule(module, patterns)
}
jar(UTIL_RT_JAR, listOf(
"intellij.platform.util.rt",
"intellij.platform.util.trove",
), productLayout, layout)
jar(UTIL_8, listOf(
"intellij.platform.util.rt.java8",
"intellij.platform.util.classLoader",
"intellij.platform.util.jdom",
"intellij.platform.util.xmlDom",
), productLayout, layout)
// fastutil-min cannot be in libsThatUsedInJps - guava is used by JPS and also in this JAR,
// but it leads to conflict in some old 3rd-party JDBC driver, so, pack fastutil-min into another JAR
layout.projectLibrariesToUnpack.putValue(UTIL_8, "fastutil-min")
jar(UTIL_JAR, listOf(
"intellij.platform.util.zip",
"intellij.platform.util",
"intellij.platform.util.text.matching",
"intellij.platform.util.base",
"intellij.platform.util.diff",
"intellij.platform.extensions",
"intellij.platform.tracing.rt",
"intellij.platform.core",
// Scala uses GeneralCommandLine in JPS plugin
"intellij.platform.ide.util.io",
"intellij.platform.boot",
), productLayout, layout)
jar("externalProcess-rt.jar", listOf(
"intellij.platform.externalProcessAuthHelper.rt"
), productLayout, layout)
jar(BaseLayout.APP_JAR, PLATFORM_IMPLEMENTATION_MODULES, productLayout, layout)
// util.jar is loaded by JVM classloader as part of loading our custom PathClassLoader class - reduce file size
jar(BaseLayout.APP_JAR, listOf(
"intellij.platform.bootstrap",
"intellij.platform.util.ui",
"intellij.platform.util.ex",
"intellij.platform.ide.util.io.impl",
"intellij.platform.ide.util.netty",
"intellij.relaxng",
"intellij.json",
"intellij.spellchecker",
"intellij.platform.webSymbols",
"intellij.xml.analysis.impl",
"intellij.xml.psi.impl",
"intellij.xml.structureView.impl",
"intellij.xml.impl",
"intellij.platform.vcs.impl",
"intellij.platform.vcs.dvcs.impl",
"intellij.platform.vcs.log.graph.impl",
"intellij.platform.vcs.log.impl",
"intellij.smart.update",
"intellij.platform.collaborationTools",
"intellij.platform.collaborationTools.auth",
"intellij.platform.markdown.utils",
"intellij.platform.icons",
"intellij.platform.resources",
"intellij.platform.resources.en",
"intellij.platform.sqlite",
), productLayout, layout)
jar("stats.jar", listOf(
"intellij.platform.statistics",
"intellij.platform.statistics.uploader",
"intellij.platform.statistics.config",
), productLayout, layout)
addModule("intellij.platform.statistics.devkit", productLayout, layout)
addModule("intellij.platform.objectSerializer.annotations", productLayout, layout)
if (!productLayout.excludedModuleNames.contains("intellij.java.guiForms.rt")) {
layout.withModule("intellij.java.guiForms.rt", "forms_rt.jar")
}
addModule("intellij.platform.jps.model.serialization", "jps-model.jar", productLayout, layout)
addModule("intellij.platform.jps.model.impl", "jps-model.jar", productLayout, layout)
addModule("intellij.platform.externalSystem.rt", "external-system-rt.jar", productLayout, layout)
addModule("intellij.platform.cdsAgent", "cds/classesLogAgent.jar", productLayout, layout)
if (hasPlatformCoverage) {
addModule("intellij.platform.coverage", BaseLayout.APP_JAR, productLayout, layout)
}
for (libraryName in productLayout.projectLibrariesToUnpackIntoMainJar) {
layout.withProjectLibraryUnpackedIntoJar(libraryName, productLayout.mainJarName)
}
val productPluginSourceModuleName = context.productProperties.applicationInfoModule
for (name in getProductPluginContentModules(context, productPluginSourceModuleName)) {
layout.withModule(name, BaseLayout.APP_JAR)
}
layout.projectLibrariesToUnpack.putValue(UTIL_RT_JAR, "ion")
for (item in additionalProjectLevelLibraries) {
val name = item.libraryName
if (!productLayout.projectLibrariesToUnpackIntoMainJar.contains(name) &&
!layout.projectLibrariesToUnpack.values().contains(name) &&
!layout.excludedProjectLibraries.contains(name)) {
layout.includedProjectLibraries.add(item)
}
}
layout.collectProjectLibrariesFromIncludedModules(context) { lib, module ->
val name = lib.name
if (module.name == "intellij.platform.buildScripts.downloader" && (name == "zstd-jni" || name == "zstd-jni-windows-aarch64")) {
return@collectProjectLibrariesFromIncludedModules
}
layout.includedProjectLibraries
.addOrGet(ProjectLibraryData(name, CUSTOM_PACK_MODE.getOrDefault(name, LibraryPackMode.MERGED)))
.dependentModules.computeIfAbsent("core") { mutableListOf() }.add(module.name)
}
return layout
}
// result _must_ consistent, do not use Set.of or HashSet here
fun getProductPluginContentModules(buildContext: BuildContext, productPluginSourceModuleName: String): Set<String> {
var file = buildContext.findFileInModuleSources(productPluginSourceModuleName, "META-INF/plugin.xml")
return result
}
private fun compute(list: List<Pair<String, PersistentList<String>>>,
context: BuildContext,
unique: HashSet<String>,
result: MutableList<Pair<String, PersistentList<String>>>) {
val oldSize = result.size
for ((dependentName, dependentChain) in list) {
val dependentModule = context.findRequiredModule(dependentName)
val chain = dependentChain.add(dependentName)
JpsJavaExtensionService.dependencies(dependentModule).includedIn(JpsJavaClasspathKind.PRODUCTION_RUNTIME).processModules { module ->
val name = module.name
if (unique.add(name)) {
result.add(name to chain)
}
}
}
if (oldSize != result.size) {
compute(list = result.subList(oldSize, result.size).sortedBy { it.first },
context = context,
unique = unique,
result = result)
}
}
// result _must be_ consistent, do not use Set.of or HashSet here
private suspend fun getProductPluginContentModules(context: BuildContext, productPluginSourceModuleName: String): Set<ModuleItem> {
val content = withContext(Dispatchers.IO) {
var file = context.findFileInModuleSources(productPluginSourceModuleName, "META-INF/plugin.xml")
if (file == null) {
file = buildContext.findFileInModuleSources(productPluginSourceModuleName,
"META-INF/${buildContext.productProperties.platformPrefix}Plugin.xml")
file = context.findFileInModuleSources(productPluginSourceModuleName,
"META-INF/${context.productProperties.platformPrefix}Plugin.xml")
if (file == null) {
buildContext.messages.warning("Cannot find product plugin descriptor in '$productPluginSourceModuleName' module")
return emptySet()
context.messages.warning("Cannot find product plugin descriptor in '$productPluginSourceModuleName' module")
return@withContext null
}
}
Files.newInputStream(file).use {
val contentList = DocumentBuilderFactory.newDefaultInstance()
.newDocumentBuilder()
.parse(it, file.toString())
.documentElement
.getElementsByTagName("content")
if (contentList.length == 0) {
return emptySet()
}
readXmlAsModel(file).getChild("content")
} ?: return emptySet()
val modules = (contentList.item(0) as Element).getElementsByTagName("module")
val result = LinkedHashSet<String>(modules.length)
for (i in 0 until modules.length) {
result.add((modules.item(i) as Element).getAttribute("name"))
}
return result
}
val modules = content.children("module")
val result = LinkedHashSet<ModuleItem>()
for (module in modules) {
result.add(ModuleItem(moduleName = module.attributes.get("name") ?: continue, relativeOutputFile = APP_JAR, reason = "productModule"))
}
return result
}

View File

@@ -21,7 +21,6 @@ import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.attribute.BasicFileAttributes
import java.util.function.BiConsumer
import java.util.function.BiPredicate
import java.util.function.UnaryOperator
typealias ResourceGenerator = suspend (Path, BuildContext) -> Unit
@@ -29,14 +28,10 @@ typealias ResourceGenerator = suspend (Path, BuildContext) -> Unit
/**
* Describes layout of a plugin in the product distribution
*/
class PluginLayout private constructor(
val mainModule: String,
mainJarNameWithoutExtension: String,
) : BaseLayout() {
class PluginLayout private constructor(val mainModule: String, mainJarNameWithoutExtension: String) : BaseLayout() {
constructor(mainModule: String) : this(
mainModule,
convertModuleNameToFileName(mainModule),
mainModule = mainModule,
mainJarNameWithoutExtension = convertModuleNameToFileName(mainModule),
)
private var mainJarName = "$mainJarNameWithoutExtension.jar"
@@ -52,24 +47,30 @@ class PluginLayout private constructor(
var directoryNameSetExplicitly: Boolean = false
var bundlingRestrictions: PluginBundlingRestrictions = PluginBundlingRestrictions.NONE
internal set
var pathsToScramble: PersistentList<String> = persistentListOf()
val scrambleClasspathPlugins: MutableList<Pair<String /*plugin name*/, String /*relative path*/>> = mutableListOf()
var scrambleClasspathFilter: BiPredicate<BuildContext, Path> = BiPredicate { _, _ -> true }
private set
var scrambleClasspathPlugins: PersistentList<Pair<String /*plugin name*/, String /*relative path*/>> = persistentListOf()
private set
var scrambleClasspathFilter: (BuildContext, Path) -> Boolean = { _, _ -> true }
/**
* See {@link org.jetbrains.intellij.build.impl.PluginLayout.PluginLayoutSpec#zkmScriptStub}
*/
var zkmScriptStub: String? = null
var pluginCompatibilityExactVersion = false
var retainProductDescriptorForBundledPlugin = false
var enableSymlinksAndExecutableResources = false
var pluginCompatibilityExactVersion: Boolean = false
var retainProductDescriptorForBundledPlugin: Boolean = false
var enableSymlinksAndExecutableResources: Boolean = false
internal var resourceGenerators: PersistentList<ResourceGenerator> = persistentListOf()
private set
internal var patchers: PersistentList<suspend (ModuleOutputPatcher, BuildContext) -> Unit> = persistentListOf()
private set
fun getMainJarName() = mainJarName
fun getMainJarName(): String = mainJarName
companion object {
/**
@@ -82,7 +83,7 @@ class PluginLayout private constructor(
* [org.jetbrains.intellij.build.ProductModulesLayout.bundledPluginModules],
* [org.jetbrains.intellij.build.ProductModulesLayout.pluginModulesToPublish] list.
*
* <p>Note that project-level libraries on which the plugin modules depend, are automatically put to 'IDE_HOME/lib' directory for all IDEs
* <p>Note that project-level libraries on which the plugin modules depend are automatically put to 'IDE_HOME/lib' directory for all IDEs
* which are compatible with the plugin. If this isn't desired (e.g. a library is used in a single plugin only, or if plugins where
* a library is used aren't bundled with IDEs, so we don't want to increase the size of the distribution, you may invoke {@link PluginLayoutSpec#withProjectLibrary}
* to include such a library to the plugin distribution.</p>
@@ -107,7 +108,7 @@ class PluginLayout private constructor(
@JvmStatic
fun plugin(moduleNames: List<String>, body: (SimplePluginLayoutSpec) -> Unit): PluginLayout {
val layout = PluginLayout(mainModule = moduleNames.first())
moduleNames.forEach(layout::withModule)
layout.withModules(moduleNames)
body(SimplePluginLayoutSpec(layout))
return layout
}
@@ -115,7 +116,7 @@ class PluginLayout private constructor(
@JvmStatic
fun plugin(moduleNames: List<String>): PluginLayout {
val layout = PluginLayout(mainModule = moduleNames.first())
moduleNames.forEach(layout::withModule)
layout.withModules(moduleNames)
return layout
}
@@ -132,7 +133,7 @@ class PluginLayout private constructor(
override fun withModule(moduleName: String) {
if (moduleName.endsWith(".jps") || moduleName.endsWith(".rt")) {
// must be in a separate JAR
super.withModule(moduleName)
withModule(moduleName, "${convertModuleNameToFileName(moduleName)}.jar")
}
else {
withModule(moduleName, mainJarName)
@@ -359,16 +360,15 @@ class PluginLayout private constructor(
* @param pluginName - a name of dependent plugin, whose jars should be added to scramble classpath
* @param relativePath - a directory where jars should be searched (relative to plugin home directory, "lib" by default)
*/
@JvmOverloads
fun scrambleClasspathPlugin(pluginName: String, relativePath: String = "lib") {
layout.scrambleClasspathPlugins.add(Pair(pluginName, relativePath))
layout.scrambleClasspathPlugins = layout.scrambleClasspathPlugins.add(Pair(pluginName, relativePath))
}
/**
* Allows control over classpath entries that will be used by the scrambler to resolve references from jars being scrambled.
* By default, all platform jars are added to the 'scramble classpath'
*/
fun filterScrambleClasspath(filter: BiPredicate<BuildContext, Path>) {
fun filterScrambleClasspath(filter: (BuildContext, Path) -> Boolean) {
layout.scrambleClasspathFilter = filter
}
@@ -380,10 +380,11 @@ class PluginLayout private constructor(
withPatch { patcher, context ->
val discoveredServiceFiles = LinkedHashMap<String, LinkedHashSet<Pair<String, Path>>>()
for (moduleName in layout.jarToModules.get(layout.mainJarName)!!) {
for (moduleName in layout.includedModules.asSequence().filter { it.relativeOutputFile == layout.mainJarName }.map { it.moduleName }.distinct()) {
val path = context.findFileInModuleSources(moduleName, "META-INF/services") ?: continue
Files.list(path).use { stream ->
stream
Files.newDirectoryStream(path).use { dirStream ->
dirStream
.asSequence()
.filter { Files.isRegularFile(it) }
.forEach { serviceFile ->
discoveredServiceFiles.computeIfAbsent(serviceFile.fileName.toString()) { LinkedHashSet() }
@@ -421,3 +422,5 @@ class PluginLayout private constructor(
fun evaluate(pluginXml: Path, ideBuildVersion: String, context: BuildContext): String
}
}
private fun convertModuleNameToFileName(moduleName: String): String = moduleName.removePrefix("intellij.").replace('.', '-')

View File

@@ -16,8 +16,8 @@ import java.nio.file.Files
import java.nio.file.Path
import java.util.*
fun collectCompatiblePluginsToPublish(providedModulesFile: Path, context: BuildContext): List<PluginLayout> {
val parse = JSON.std.mapFrom(Files.readString(providedModulesFile))
fun collectCompatiblePluginsToPublish(providedModuleFile: Path, context: BuildContext): List<PluginLayout> {
val parse = JSON.std.mapFrom(Files.readString(providedModuleFile))
@Suppress("UNCHECKED_CAST")
val availableModulesAndPlugins = HashSet(parse.get("modules") as Collection<String>)
@@ -28,13 +28,13 @@ fun collectCompatiblePluginsToPublish(providedModulesFile: Path, context: BuildC
skipBundledPlugins = true,
honorCompatiblePluginsToIgnore = true,
context = context)
val descriptorsMapWithBundled = collectPluginDescriptors(skipImplementationDetailPlugins = true,
skipBundledPlugins = false,
honorCompatiblePluginsToIgnore = true,
context = context)
val descriptorMapWithBundled = collectPluginDescriptors(skipImplementationDetailPlugins = true,
skipBundledPlugins = false,
honorCompatiblePluginsToIgnore = true,
context = context)
val result = ArrayList<PluginLayout>(descriptorMap.size)
for (descriptor in descriptorMap.values) {
if (isPluginCompatible(descriptor, availableModulesAndPlugins, descriptorsMapWithBundled)) {
if (isPluginCompatible(descriptor, availableModulesAndPlugins, descriptorMapWithBundled)) {
result.add(descriptor.pluginLayout)
}
}
@@ -196,7 +196,7 @@ private class SourcesBasedXIncludeResolver(
) : JDOMXIncluder.PathResolver {
override fun resolvePath(relativePath: String, base: URL?): URL {
var result: URL? = null
for (moduleName in pluginLayout.includedModuleNames) {
for (moduleName in pluginLayout.includedModules.asSequence().map { it.moduleName }.distinct()) {
result = (context.findFileInModuleSources(moduleName, relativePath) ?: continue).toUri().toURL()
}
if (result == null) {

View File

@@ -15,10 +15,10 @@ enum class LibraryPackMode {
}
class ProjectLibraryData(
val libraryName: String,
val packMode: LibraryPackMode,
val outPath: String? = null,
val reason: String? = null,
@JvmField val libraryName: String,
@JvmField val packMode: LibraryPackMode,
@JvmField val outPath: String? = null,
@JvmField val reason: String? = null,
) {
init {
require(outPath == null || !outPath.isBlank()) {
@@ -26,20 +26,20 @@ class ProjectLibraryData(
}
}
// plugin to list of modules that uses the library
// plugin to a list of modules that uses the library
val dependentModules: MutableMap<String, MutableList<String>> = TreeMap()
override fun toString() = "ProjectLibraryData(name=$libraryName, packMode=$packMode, relativeOutputPath=$outPath)"
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
if (this === other) {
return true
}
if (javaClass != other?.javaClass) {
return false
}
other as ProjectLibraryData
if (libraryName != other.libraryName) return false
return true
return libraryName == other.libraryName
}
override fun hashCode() = libraryName.hashCode()

View File

@@ -75,5 +75,5 @@ class ModuleOutputEntry(
override val type: String
get() = "module-output"
override fun changePath(newFile: Path) = ModuleOutputEntry(newFile, moduleName, size, reason)
override fun changePath(newFile: Path) = ModuleOutputEntry(path = newFile, moduleName = moduleName, size = size, reason = reason)
}

View File

@@ -1,4 +1,6 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("LiftReturnOrAssignment")
package org.jetbrains.intellij.build.impl.projectStructureMapping
import com.fasterxml.jackson.core.JsonFactory
@@ -18,7 +20,7 @@ internal val Collection<DistributionFileEntry>.includedModules: Sequence<String>
/**
* Provides mapping between files in the product distribution and modules and libraries in the project configuration. The generated JSON file
* contains array of [DistributionFileEntry].
* contains an array of [DistributionFileEntry].
*/
internal fun buildJarContentReport(entries: Collection<DistributionFileEntry>, out: OutputStream?, buildPaths: BuildPaths) {
val writer = JsonFactory().createGenerator(out).setPrettyPrinter(IntelliJDefaultPrettyPrinter())
@@ -57,8 +59,7 @@ fun writeProjectStructureReport(entries: Collection<DistributionFileEntry>, file
writer.writeNumberField("size", entry.size)
}
is ModuleOutputEntry -> {
writer.writeStringField("module", entry.moduleName)
writer.writeNumberField("size", entry.size)
writeModuleItem(writer, entry)
}
is ModuleTestOutputEntry -> {
writer.writeStringField("module", entry.moduleName)
@@ -128,8 +129,7 @@ private fun writeModules(writer: JsonGenerator, fileEntries: List<DistributionFi
writer.writeStartObject()
val moduleName = entry.moduleName
writer.writeStringField("name", moduleName)
writer.writeNumberField("size", entry.size)
writeModuleItem(writer, entry)
writeModuleLibraries(fileEntries = fileEntries, moduleName = moduleName, writer = writer, buildPaths = buildPaths)
writer.writeEndObject()
}
@@ -138,6 +138,14 @@ private fun writeModules(writer: JsonGenerator, fileEntries: List<DistributionFi
}
}
private fun writeModuleItem(writer: JsonGenerator, entry: ModuleOutputEntry) {
writer.writeStringField("name", entry.moduleName)
writer.writeNumberField("size", entry.size)
entry.reason?.let {
writer.writeStringField("reason", it)
}
}
private fun writeModuleLibraries(fileEntries: List<DistributionFileEntry>,
moduleName: String,
writer: JsonGenerator,

View File

@@ -182,7 +182,6 @@ object KotlinPluginBuilder {
"kotlin.uast.uast-kotlin-idea-fir",
"kotlin.fir.fir-low-level-api-ide-impl",
"kotlin.navigation",
"kotlin.code-insight.line-markers-k2",
"kotlin.refactorings.common",
"kotlin.refactorings.k2",
"kotlin.refactorings.rename.k2",

View File

@@ -29,14 +29,13 @@ object PythonCommunityPluginModules {
"intellij.jupyter.core"
)
const val pythonCommunityName = "python-ce"
const val pythonCommunityName: String = "python-ce"
fun pythonCommunityPluginLayout(body: ((PluginLayout.PluginLayoutSpec) -> Unit)? = null): PluginLayout {
val communityOnlyModules = persistentListOf(
"intellij.python.community.plugin",
"intellij.python.community.plugin.minor",
)
return pythonPlugin("intellij.python.community.plugin", pythonCommunityName, COMMUNITY_MODULES.addAll(communityOnlyModules)) { spec ->
return pythonPlugin("intellij.python.community.plugin", pythonCommunityName, COMMUNITY_MODULES + communityOnlyModules) { spec ->
body?.invoke(spec)
}
}
@@ -50,7 +49,6 @@ object PythonCommunityPluginModules {
spec.directoryName = name
spec.mainJarName = "${name}.jar"
spec.withModules(modules)
spec.withModule(mainModuleName)
spec.withGeneratedResources { targetDir, context ->
val output = targetDir.resolve("helpers")
Files.createDirectories(output)

View File

@@ -1,4 +1,6 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("ReplaceGetOrSet")
package org.jetbrains.intellij.build.testFramework
import com.intellij.openapi.application.PathManager
@@ -10,12 +12,9 @@ import org.jetbrains.intellij.build.BuildContext
import org.jetbrains.intellij.build.IdeaProjectLoaderUtil
import org.jetbrains.intellij.build.ProductProperties
import org.jetbrains.intellij.build.ProprietaryBuildTools
import org.jetbrains.intellij.build.impl.DistributionJARsBuilder
import org.jetbrains.intellij.build.impl.ModuleStructureValidator
import org.jetbrains.jps.model.java.JpsJavaClasspathKind
import org.jetbrains.intellij.build.impl.createDistributionBuilderState
import org.jetbrains.jps.model.java.JpsJavaDependencyScope
import org.jetbrains.jps.model.java.JpsJavaExtensionService
import org.jetbrains.jps.model.module.JpsModuleDependency
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.extension.ExtendWith
import java.nio.file.Path
@@ -37,10 +36,10 @@ abstract class IdeStructureTestBase {
val productProperties = createProductProperties(projectHome)
return runBlocking(Dispatchers.Default) {
createBuildContext(homePath = projectHome,
productProperties = productProperties,
buildTools = createBuildTools(),
skipDependencySetup = false,
communityHomePath = IdeaProjectLoaderUtil.guessCommunityHome(javaClass))
productProperties = productProperties,
buildTools = createBuildTools(),
skipDependencySetup = false,
communityHomePath = IdeaProjectLoaderUtil.guessCommunityHome(javaClass))
}
}
@@ -48,62 +47,19 @@ abstract class IdeStructureTestBase {
@Test
fun moduleStructureValidation(softly: SoftAssertions) {
val context = createBuildContext()
val jarBuilder = DistributionJARsBuilder(context, emptySet())
println("Packed modules:")
val moduleToJar = jarBuilder.state.platform.jarToModules.entries.asSequence()
.flatMap { it.value.map { e -> e to it.key } }
.groupBy(keySelector = { it.first }, valueTransform = { it.second })
.toSortedMap()
for (kv in moduleToJar) {
println(" ${kv.key} ${kv.value}")
val state = runBlocking {
createDistributionBuilderState(pluginsToPublish = emptySet(), context = context)
}
val validator = ModuleStructureValidator(context, jarBuilder.state.platform.jarToModules)
println("Packed modules:")
for (item in state.platform.includedModules) {
println(" ${item.moduleName} ${item.relativeOutputFile}")
}
val validator = ModuleStructureValidator(context, state.platform.includedModules)
val errors = validator.validate()
for (error in errors) {
softly.collectAssertionError(error)
}
}
@Test
fun moduleClosureValidation(softly: SoftAssertions) {
val buildContext = createBuildContext()
val jarBuilder = DistributionJARsBuilder(buildContext, emptySet())
val exceptions = missingModulesException
val activeExceptions = mutableSetOf<MissingModuleException>()
val moduleToJar = jarBuilder.state.platform.jarToModules.asSequence()
.flatMap { it.value.map { e -> e to it.key } }
.toMap(TreeMap())
for (kv in moduleToJar) {
val module = buildContext.findRequiredModule(kv.key)
for (dependency in module.dependenciesList.dependencies) {
if (dependency !is JpsModuleDependency) {
continue
}
val dependencyExtension = JpsJavaExtensionService.getInstance().getDependencyExtension(dependency)!!
if (!dependencyExtension.scope.isIncludedIn(JpsJavaClasspathKind.PRODUCTION_RUNTIME)) {
continue
}
val moduleDependency = dependency.module!!
if (!moduleToJar.containsKey(moduleDependency.name)) {
val missingModuleException = MissingModuleException(module.name, moduleDependency.name, dependencyExtension.scope)
if (exceptions.contains(missingModuleException)) {
activeExceptions.add(missingModuleException)
}
else {
val message = "${buildContext.productProperties.productCode} (${javaClass.simpleName}): missing module from the product layout '${moduleDependency.name}' referenced from '${module.name}' scope ${dependencyExtension.scope}"
softly.fail<Unit>(message)
}
}
}
}
for (moduleName in exceptions.minus(activeExceptions)) {
softly.fail<Unit>("${buildContext.productProperties.productCode} (${javaClass.simpleName}): module '$moduleName' is mentioned in ${::missingModulesException.name}, but it was not used. Please remove it from the list")
}
}
}

View File

@@ -16,11 +16,11 @@ import kotlin.io.path.writeText
internal class BuildArtifactsReproducibilityTest {
private val randomSeedNumber = Random().nextLong()
private val iterationsChannel = Channel<BuildContext>()
val iterations = if (isEnabled) 2 else 1
private val iterationChannel = Channel<BuildContext>()
val iterations: Int = if (isEnabled) 2 else 1
companion object {
val isEnabled = System.getProperty("intellij.build.test.artifacts.reproducibility") == "true"
val isEnabled: Boolean = System.getProperty("intellij.build.test.artifacts.reproducibility") == "true"
}
fun configure(options: BuildOptions) {
@@ -35,16 +35,16 @@ internal class BuildArtifactsReproducibilityTest {
if (!isEnabled) return
build.cleanBuildOutput()
if (iterationNumber == 1) {
iterationsChannel.send(build)
iterationChannel.send(build)
/**
* Waiting for [compare] to complete not to clean up [BuildPaths.buildOutputDir] of a [build]
*/
iterationsChannel.receive()
iterationChannel.receive()
}
else {
val otherBuild = iterationsChannel.receive()
val otherBuild = iterationChannel.receive()
compare(build, otherBuild)
iterationsChannel.send(build)
iterationChannel.send(build)
}
}

View File

@@ -1,4 +1,6 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("LiftReturnOrAssignment")
package org.jetbrains.intellij.build.testFramework
import com.intellij.diagnostic.telemetry.useWithScope2
@@ -203,7 +205,7 @@ private fun copyDebugLog(productProperties: ProductProperties, messages: BuildMe
}
}
private suspend fun asSingleTraceFile(traceSpanName: String, build: suspend () -> Unit) {
private inline fun asSingleTraceFile(traceSpanName: String, build: () -> Unit) {
val traceFile = TestLoggerFactory.getTestLogDir().resolve("$traceSpanName-trace.json")
TracerProviderManager.setOutput(traceFile)
try {

View File

@@ -4,9 +4,7 @@ package org.jetbrains.intellij.build
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.runBlocking
import org.assertj.core.api.Assertions.assertThat
import org.jetbrains.intellij.build.impl.BuildContextImpl
import org.jetbrains.intellij.build.impl.DistributionBuilderState
import org.jetbrains.intellij.build.impl.DistributionJARsBuilder
import org.jetbrains.intellij.build.impl.*
import org.junit.Test
class DistributionJARsBuilderTest {
@@ -16,8 +14,8 @@ class DistributionJARsBuilderTest {
val productProperties = IdeaCommunityProperties(communityHome.communityRoot)
runBlocking(Dispatchers.Default) {
val context = BuildContextImpl.createContext(communityHome, communityHome.communityRoot, productProperties)
val ideClasspath1 = DistributionJARsBuilder(DistributionBuilderState(emptySet(), context)).createIdeClassPath(context)
val ideClasspath2 = DistributionJARsBuilder(DistributionBuilderState(emptySet(), context)).createIdeClassPath(context)
val ideClasspath1 = createIdeClassPath(createDistributionBuilderState(emptySet(), context), context)
val ideClasspath2 = createIdeClassPath(createDistributionBuilderState(emptySet(), context), context)
assertThat(ideClasspath1).isEqualTo(ideClasspath2)
}
}

View File

@@ -39,7 +39,6 @@
<orderEntry type="library" name="fastutil-min" level="project" />
<orderEntry type="library" name="kotlin-stdlib" level="project" />
<orderEntry type="module" module-name="intellij.platform.util.base" exported="" />
<orderEntry type="module" module-name="intellij.platform.util.diff" />
<orderEntry type="library" name="aalto-xml" level="project" />
<orderEntry type="module" module-name="intellij.platform.util.xmlDom" />
<orderEntry type="library" name="kotlinx-coroutines-jdk8" level="project" />

View File

@@ -2,7 +2,6 @@
package com.intellij.openapi.application
import com.fasterxml.aalto.`in`.ReaderConfig
import com.intellij.diff.comparison.ComparisonUtil
import com.intellij.openapi.util.SystemInfoRt
import com.intellij.openapi.util.text.Strings
import com.intellij.util.lang.UrlClassLoader
@@ -42,7 +41,6 @@ object ClassPathUtil {
Strings::class.java, // module 'intellij.platform.util.base'
classLoader.loadClass("com.intellij.util.xml.dom.XmlDomReader"), // module 'intellij.platform.util.xmlDom'
SystemInfoRt::class.java, // module 'intellij.platform.util.rt'
ComparisonUtil::class.java, // module 'intellij.platform.util.diff'
UrlClassLoader::class.java, // module 'intellij.platform.util.classLoader'
classLoader.loadClass("org.jetbrains.xxh3.Xx3UnencodedString"), // intellij.platform.util.rt.java8 (required for classLoader)
Flow::class.java, // jetbrains-annotations-java5

View File

@@ -1,22 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="FacetManager">
<facet type="kotlin-language" name="Kotlin">
<configuration version="5" platform="JVM 1.8" allPlatforms="JVM [1.8]" useProjectSettings="false">
<compilerSettings>
<option name="additionalArguments" value="-Xjvm-default=all -opt-in=com.intellij.openapi.util.IntellijInternalApi" />
</compilerSettings>
<compilerArguments>
<stringArguments>
<stringArg name="jvmTarget" arg="1.8" />
<stringArg name="apiVersion" arg="1.8" />
<stringArg name="languageVersion" arg="1.8" />
</stringArguments>
</compilerArguments>
</configuration>
</facet>
</component>
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8" inherit-compiler-output="true">
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />

View File

@@ -13,18 +13,7 @@ abstract class PyCharmPropertiesBase : JetBrainsProductProperties() {
init {
reassignAltClickToMultipleCarets = true
useSplash = true
productLayout.mainJarName = "pycharm.jar"
productLayout.withAdditionalPlatformJar(
"testFramework.jar",
"intellij.platform.testFramework.core",
"intellij.platform.testFramework.impl",
"intellij.platform.testFramework.common",
"intellij.platform.testFramework.junit5",
"intellij.platform.testFramework",
"intellij.tools.testsBootstrap",
"intellij.java.rt",
)
productLayout.addPlatformSpec(TEST_FRAMEWORK_WITH_JAVA_RT)
buildCrossPlatformDistribution = true
mavenArtifacts.additionalModules = mavenArtifacts.additionalModules.addAll(listOf(
"intellij.java.compiler.antTasks",