IJPL-866 refactor - extract index writer from ZipFileWriter

GitOrigin-RevId: 04b6d46b87bd373df5a41672c2406703044e1510
This commit is contained in:
Vladimir Krivosheev
2024-03-22 13:34:11 +01:00
committed by intellij-monorepo-bot
parent d2d23c77a1
commit 95c0264e49
21 changed files with 272 additions and 195 deletions

1
.idea/modules.xml generated
View File

@@ -1082,7 +1082,6 @@
<module fileurl="file://$PROJECT_DIR$/platform/util/diff/intellij.platform.util.diff.iml" filepath="$PROJECT_DIR$/platform/util/diff/intellij.platform.util.diff.iml" />
<module fileurl="file://$PROJECT_DIR$/platform/util-ex/intellij.platform.util.ex.iml" filepath="$PROJECT_DIR$/platform/util-ex/intellij.platform.util.ex.iml" />
<module fileurl="file://$PROJECT_DIR$/platform/util/http/intellij.platform.util.http.iml" filepath="$PROJECT_DIR$/platform/util/http/intellij.platform.util.http.iml" />
<module fileurl="file://$PROJECT_DIR$/platform/util/immutable-key-value-store/intellij.platform.util.immutableKeyValueStore.iml" filepath="$PROJECT_DIR$/platform/util/immutable-key-value-store/intellij.platform.util.immutableKeyValueStore.iml" />
<module fileurl="file://$PROJECT_DIR$/platform/util/immutable-key-value-store/benchmark/intellij.platform.util.immutableKeyValueStore.benchmark.iml" filepath="$PROJECT_DIR$/platform/util/immutable-key-value-store/benchmark/intellij.platform.util.immutableKeyValueStore.benchmark.iml" />
<module fileurl="file://$PROJECT_DIR$/platform/util/jdom/intellij.platform.util.jdom.iml" filepath="$PROJECT_DIR$/platform/util/jdom/intellij.platform.util.jdom.iml" />
<module fileurl="file://$PROJECT_DIR$/platform/util/nanoxml/intellij.platform.util.nanoxml.iml" filepath="$PROJECT_DIR$/platform/util/nanoxml/intellij.platform.util.nanoxml.iml" />

View File

@@ -42,7 +42,6 @@
<orderEntry type="library" name="jackson-jr-objects" level="project" />
<orderEntry type="library" name="jackson" level="project" />
<orderEntry type="library" name="kotlinx-coroutines-core" level="project" />
<orderEntry type="module" module-name="intellij.platform.util.immutableKeyValueStore" />
<orderEntry type="module" module-name="intellij.platform.util.rt.java8" />
<orderEntry type="module" module-name="intellij.java.rt" />
<orderEntry type="module" module-name="intellij.platform.util.rt" />

View File

@@ -1,5 +1,5 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.ikv.builder
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.io
import com.intellij.util.lang.ByteBufferCleaner
import java.nio.ByteBuffer
@@ -9,6 +9,7 @@ import java.nio.ByteOrder
class IkvIndexBuilder(private val writeSize: Boolean = true) {
private val entries = LinkedHashSet<IkvIndexEntry>()
@JvmField val names = mutableListOf<ByteArray>()
fun entry(key: Long, offset: Long, size: Int): IkvIndexEntry {
val entry = LongKeyedEntry(longKey = key, offset = offset)

View File

@@ -1,5 +1,5 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.ikv.builder
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.io
import java.nio.ByteBuffer
import java.nio.channels.FileChannel

View File

@@ -1,7 +1,8 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.io
import com.intellij.util.lang.Xx3UnencodedString
import com.intellij.util.lang.Xxh3
import it.unimi.dsi.fastutil.longs.LongOpenHashSet
class PackageIndexBuilder {
@@ -13,6 +14,8 @@ class PackageIndexBuilder {
private val dirsToRegister = HashSet<String>()
private var wasWritten = false
@JvmField val indexWriter = IkvIndexBuilder()
fun addFile(name: String, addClassDir: Boolean = false) {
val i = name.lastIndexOf('/')
val packageNameHash = if (i == -1) 0 else Xx3UnencodedString.hashUnencodedStringRange(name, i)
@@ -47,11 +50,15 @@ class PackageIndexBuilder {
val stream = zipCreator.resultStream
if (addDirEntriesMode == AddDirEntriesMode.NONE) {
stream.addDirsToIndex(sortedDirsToRegister)
for (dirName in sortedDirsToRegister) {
val nameBytes = dirName.encodeToByteArray()
indexWriter.add(indexWriter.entry(key = Xxh3.hash(nameBytes), offset = 0, size = -1))
indexWriter.names.add(nameBytes)
}
}
else {
for (dir in sortedDirsToRegister) {
stream.addDirEntry(dir)
stream.addDirEntry(dir, indexWriter)
}
}

View File

@@ -1,9 +1,8 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.io
import com.intellij.util.lang.ImmutableZipFile
import com.intellij.util.lang.Xxh3
import org.jetbrains.ikv.builder.IkvIndexBuilder
import java.io.IOException
import java.nio.ByteBuffer
import java.nio.ByteOrder
@@ -17,8 +16,9 @@ private const val INDEX_FORMAT_VERSION: Byte = 4
const val INDEX_FILENAME: String = "__index__"
internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
private val withOptimizedMetadataEnabled: Boolean) : AutoCloseable {
internal class ZipArchiveOutputStream(
private val channel: WritableByteChannel,
) : AutoCloseable {
private var classPackages: LongArray? = null
private var resourcePackages: LongArray? = null
@@ -26,17 +26,15 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
private var entryCount = 0
private var metadataBuffer = ByteBuffer.allocateDirect(2 * 1024 * 1024).order(ByteOrder.LITTLE_ENDIAN)
// 1 MB should be enough for the end of the central directory record
private val buffer = ByteBuffer.allocateDirect(1024 * 1024).order(ByteOrder.LITTLE_ENDIAN)
private val names = mutableListOf<ByteArray>()
private val indexWriter = IkvIndexBuilder()
private var channelPosition = 0L
private val fileChannel = channel as? FileChannel
fun addDirEntry(name: String) {
fun addDirEntry(name: String, indexWriter: IkvIndexBuilder?) {
if (finished) {
throw IOException("Stream has already been finished")
}
@@ -75,10 +73,19 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
buffer.flip()
writeBuffer(buffer)
writeCentralFileHeader(0, 0, ZipEntry.STORED, 0, nameInArchive, offset, dataOffset = -1, normalName = key)
writeCentralFileHeader(0, 0, ZipEntry.STORED, 0, nameInArchive, offset, dataOffset = -1, normalName = key, indexWriter = indexWriter)
}
fun writeRawEntry(header: ByteBuffer, content: ByteBuffer, name: ByteArray, size: Int, compressedSize: Int, method: Int, crc: Long) {
fun writeRawEntry(
header: ByteBuffer,
content: ByteBuffer,
name: ByteArray,
size: Int,
compressedSize: Int,
method: Int,
crc: Long,
indexWriter: IkvIndexBuilder?,
) {
if (finished) {
throw IOException("Stream has already been finished")
}
@@ -91,10 +98,28 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
writeBuffer(header)
writeBuffer(content)
writeCentralFileHeader(size, compressedSize, method, crc, name, offset, dataOffset = dataOffset)
writeCentralFileHeader(
size = size,
compressedSize = compressedSize,
method = method,
crc = crc,
name = name,
offset = offset,
dataOffset = dataOffset,
indexWriter = indexWriter,
)
}
fun writeRawEntry(content: ByteBuffer, name: ByteArray, size: Int, compressedSize: Int, method: Int, crc: Long, headerSize: Int) {
fun writeRawEntry(
content: ByteBuffer,
name: ByteArray,
size: Int,
compressedSize: Int,
method: Int,
crc: Long,
headerSize: Int,
indexWriter: IkvIndexBuilder?,
) {
if (finished) {
throw IOException("Stream has already been finished")
}
@@ -104,10 +129,19 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
assert(method != -1)
writeBuffer(content)
writeCentralFileHeader(size, compressedSize, method, crc, name, offset, dataOffset = offset + headerSize)
writeCentralFileHeader(size, compressedSize, method, crc, name, offset, dataOffset = offset + headerSize, indexWriter = indexWriter)
}
fun writeEntryHeaderAt(name: ByteArray, header: ByteBuffer, position: Long, size: Int, compressedSize: Int, crc: Long, method: Int) {
fun writeEntryHeaderAt(
name: ByteArray,
header: ByteBuffer,
position: Long,
size: Int,
compressedSize: Int,
crc: Long,
method: Int,
indexWriter: IkvIndexBuilder?,
) {
if (finished) {
throw IOException("Stream has already been finished")
}
@@ -134,16 +168,19 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
entryCount++
assert(channelPosition == dataOffset + compressedSize)
writeCentralFileHeader(size = size,
compressedSize = compressedSize,
method = method,
crc = crc,
name = name,
offset = position,
dataOffset = dataOffset)
writeCentralFileHeader(
size = size,
compressedSize = compressedSize,
method = method,
crc = crc,
name = name,
offset = position,
dataOffset = dataOffset,
indexWriter = indexWriter,
)
}
private fun writeIndex(crc32: CRC32): Int {
private fun writeIndex(crc32: CRC32, indexWriter: IkvIndexBuilder): Int {
// write one by one to channel to avoid buffer overflow
indexWriter.write {
crc32.update(it)
@@ -182,7 +219,7 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
}
// write names
for (list in names.asSequence().chunked(4096)) {
for (list in indexWriter.names.asSequence().chunked(4096)) {
writeData { buffer ->
val shortBuffer = buffer.asShortBuffer()
for (name in list) {
@@ -192,7 +229,7 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
}
}
for (list in names.asSequence().chunked(1024)) {
for (list in indexWriter.names.asSequence().chunked(1024)) {
writeData { buffer ->
for (name in list) {
buffer.put(name)
@@ -203,13 +240,13 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
return indexDataEnd
}
fun finish() {
fun finish(indexWriter: IkvIndexBuilder?) {
if (finished) {
throw IOException("This archive has already been finished")
}
val indexOffset: Int
if (withOptimizedMetadataEnabled && entryCount != 0) {
if (indexWriter != null && entryCount != 0) {
// ditto on macOS doesn't like arbitrary data in zip file - wrap into zip entry
val name = INDEX_FILENAME.toByteArray(Charsets.UTF_8)
val headerSize = 30 + name.size
@@ -217,7 +254,7 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
val entryDataPosition = channelPosition
val crc32 = CRC32()
indexOffset = writeIndex(crc32)
indexOffset = writeIndex(crc32, indexWriter)
val size = (channelPosition - entryDataPosition).toInt()
val crc = crc32.value
@@ -226,13 +263,16 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
writeLocalFileHeader(name = name, size = size, compressedSize = size, crc32 = crc, method = ZipEntry.STORED, buffer = buffer)
buffer.flip()
assert(buffer.remaining() == headerSize)
writeEntryHeaderAt(name = name,
header = buffer,
position = headerPosition,
size = size,
compressedSize = size,
crc = crc,
method = ZipEntry.STORED)
writeEntryHeaderAt(
name = name,
header = buffer,
position = headerPosition,
size = size,
compressedSize = size,
crc = crc,
method = ZipEntry.STORED,
indexWriter = indexWriter,
)
}
else {
indexOffset = -1
@@ -262,7 +302,7 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
buffer.putInt((centralDirectoryOffset and 0xffffffffL).toInt())
// comment length
if (withOptimizedMetadataEnabled) {
if (indexWriter != null) {
buffer.putShort((Byte.SIZE_BYTES + Integer.BYTES).toShort())
// version
buffer.put(INDEX_FORMAT_VERSION)
@@ -273,7 +313,11 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
}
}
else {
writeZip64End(centralDirectoryLength, centralDirectoryOffset, indexOffset)
writeZip64End(
centralDirectoryLength = centralDirectoryLength,
centralDirectoryOffset = centralDirectoryOffset,
optimizedMetadataOffset = indexOffset,
)
}
buffer.flip()
writeBuffer(buffer)
@@ -306,7 +350,7 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
buffer.putLong(centralDirectoryOffset)
// comment length
if (withOptimizedMetadataEnabled) {
if (optimizedMetadataOffset != -1) {
// version
buffer.put(INDEX_FORMAT_VERSION)
buffer.putInt(optimizedMetadataOffset)
@@ -375,7 +419,7 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
try {
if (!finished) {
channel.use {
finish()
finish(indexWriter = null)
}
}
}
@@ -385,23 +429,17 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
}
}
fun addDirsToIndex(dirNames: Collection<String>) {
assert(withOptimizedMetadataEnabled)
for (dirName in dirNames) {
val nameBytes = dirName.toByteArray(Charsets.UTF_8)
indexWriter.add(indexWriter.entry(key = Xxh3.hash(nameBytes), offset = 0, size = -1))
names.add(nameBytes)
}
}
private fun writeCentralFileHeader(size: Int,
compressedSize: Int,
method: Int,
crc: Long,
name: ByteArray,
offset: Long,
dataOffset: Long,
normalName: ByteArray = name) {
private fun writeCentralFileHeader(
size: Int,
compressedSize: Int,
method: Int,
crc: Long,
name: ByteArray,
offset: Long,
dataOffset: Long,
normalName: ByteArray = name,
indexWriter: IkvIndexBuilder?,
) {
var buffer = metadataBuffer
if (buffer.remaining() < (46 + name.size)) {
metadataBuffer = ByteBuffer.allocateDirect(buffer.capacity() * 2).order(ByteOrder.LITTLE_ENDIAN)
@@ -422,9 +460,9 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
// uncompressed size
buffer.putInt(headerOffset + 24, size)
if (withOptimizedMetadataEnabled) {
if (indexWriter != null) {
indexWriter.add(indexWriter.entry(offset = dataOffset, size = size, key = Xxh3.hash(normalName)))
names.add(normalName)
indexWriter.names.add(normalName)
}
// file name length

View File

@@ -1,4 +1,4 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("ConstPropertyName")
package org.jetbrains.intellij.build.io
@@ -36,25 +36,24 @@ fun transformZipUsingTempFile(file: Path, task: (ZipFileWriter) -> Unit) {
}
}
inline fun writeNewZip(file: Path,
compress: Boolean = false,
withOptimizedMetadataEnabled: Boolean = !compress,
task: (ZipFileWriter) -> Unit) {
inline fun writeNewZipWithoutIndex(
file: Path,
compress: Boolean = false,
task: (ZipFileWriter) -> Unit,
) {
Files.createDirectories(file.parent)
ZipFileWriter(channel = FileChannel.open(file, W_CREATE_NEW),
deflater = if (compress) Deflater(Deflater.DEFAULT_COMPRESSION, true) else null,
withOptimizedMetadataEnabled = !compress && withOptimizedMetadataEnabled).use {
ZipFileWriter(
channel = FileChannel.open(file, W_CREATE_NEW),
deflater = if (compress) Deflater(Deflater.DEFAULT_COMPRESSION, true) else null,
).use {
task(it)
}
}
// you must pass SeekableByteChannel if files are written (`file` method)
class ZipFileWriter(channel: WritableByteChannel,
private val deflater: Deflater? = null,
withOptimizedMetadataEnabled: Boolean = deflater == null) : AutoCloseable {
class ZipFileWriter(channel: WritableByteChannel, private val deflater: Deflater? = null) : AutoCloseable {
// size is written as part of optimized metadata - so, if compression is enabled, optimized metadata will be incorrect
internal val resultStream = ZipArchiveOutputStream(channel, withOptimizedMetadataEnabled = withOptimizedMetadataEnabled)
internal val resultStream = ZipArchiveOutputStream(channel)
private val crc32 = CRC32()
private val bufferAllocator = ByteBufferAllocator()
@@ -64,7 +63,7 @@ class ZipFileWriter(channel: WritableByteChannel,
get() = resultStream.getChannelPosition()
@Suppress("DuplicatedCode")
fun file(nameString: String, file: Path) {
fun file(nameString: String, file: Path, indexWriter: IkvIndexBuilder?) {
var isCompressed = deflater != null && !nameString.endsWith(".png")
val name = nameString.toByteArray()
@@ -77,7 +76,7 @@ class ZipFileWriter(channel: WritableByteChannel,
FileChannel.open(file, EnumSet.of(StandardOpenOption.READ)).use { channel ->
size = channel.size().toInt()
if (size == 0) {
writeEmptyFile(name, headerSize)
writeEmptyFile(name, headerSize, indexWriter)
return
}
if (size < compressThreshold) {
@@ -100,13 +99,16 @@ class ZipFileWriter(channel: WritableByteChannel,
writeLocalFileHeader(name = name, size = size, compressedSize = compressedSize, crc32 = crc, method = method, buffer = buffer)
buffer.position(0)
assert(buffer.remaining() == headerSize)
resultStream.writeEntryHeaderAt(name = name,
header = buffer,
position = headerPosition,
size = size,
compressedSize = compressedSize,
crc = crc,
method = method)
resultStream.writeEntryHeaderAt(
name = name,
header = buffer,
position = headerPosition,
size = size,
compressedSize = compressedSize,
crc = crc,
method = method,
indexWriter = indexWriter,
)
return
}
@@ -129,7 +131,7 @@ class ZipFileWriter(channel: WritableByteChannel,
writeLocalFileHeader(name, size, size, crc, ZipEntry.STORED, input)
input.position(0)
assert(input.remaining() == (size + headerSize))
resultStream.writeRawEntry(input, name, size, size, ZipEntry.STORED, crc, headerSize)
resultStream.writeRawEntry(input, name, size, size, ZipEntry.STORED, crc, headerSize, indexWriter)
}
private fun writeLargeFile(fileSize: Long, channel: FileChannel, deflater: Deflater?): Long {
@@ -226,19 +228,28 @@ class ZipFileWriter(channel: WritableByteChannel,
writeLocalFileHeader(name, size, compressedSize, crc, ZipEntry.DEFLATED, output)
output.position(0)
assert(output.remaining() == (compressedSize + headerSize))
resultStream.writeRawEntry(output, name, size, compressedSize, ZipEntry.DEFLATED, crc, headerSize)
resultStream.writeRawEntry(
content = output,
name = name,
size = size,
compressedSize = compressedSize,
method = ZipEntry.DEFLATED,
crc = crc,
headerSize = headerSize,
indexWriter = null,
)
}
fun uncompressedData(nameString: String, data: String) {
uncompressedData(nameString, ByteBuffer.wrap(data.toByteArray()))
fun uncompressedData(nameString: String, data: String, indexWriter: IkvIndexBuilder?) {
uncompressedData(nameString, ByteBuffer.wrap(data.toByteArray()), indexWriter)
}
fun uncompressedData(nameString: String, data: ByteBuffer) {
fun uncompressedData(nameString: String, data: ByteBuffer, indexWriter: IkvIndexBuilder?) {
val name = nameString.toByteArray()
val headerSize = 30 + name.size
if (!data.hasRemaining()) {
writeEmptyFile(name, headerSize)
writeEmptyFile(name, headerSize, indexWriter)
return
}
@@ -252,10 +263,10 @@ class ZipFileWriter(channel: WritableByteChannel,
val header = bufferAllocator.allocate(headerSize)
writeLocalFileHeader(name, size, size, crc, ZipEntry.STORED, header)
header.position(0)
resultStream.writeRawEntry(header, data, name, size, size, ZipEntry.STORED, crc)
resultStream.writeRawEntry(header, data, name, size, size, ZipEntry.STORED, crc, indexWriter = indexWriter)
}
fun uncompressedData(nameString: String, maxSize: Int, dataWriter: (ByteBuffer) -> Unit) {
fun uncompressedData(nameString: String, maxSize: Int, indexWriter: IkvIndexBuilder?, dataWriter: (ByteBuffer) -> Unit) {
val name = nameString.toByteArray()
val headerSize = 30 + name.size
@@ -274,19 +285,19 @@ class ZipFileWriter(channel: WritableByteChannel,
writeLocalFileHeader(name, size, size, crc, ZipEntry.STORED, output)
output.position(0)
assert(output.remaining() == (size + headerSize))
resultStream.writeRawEntry(output, name, size, size, ZipEntry.STORED, crc, headerSize)
resultStream.writeRawEntry(output, name, size, size, ZipEntry.STORED, crc, headerSize, indexWriter = indexWriter)
}
private fun writeEmptyFile(name: ByteArray, headerSize: Int) {
private fun writeEmptyFile(name: ByteArray, headerSize: Int, indexWriter: IkvIndexBuilder?) {
val input = bufferAllocator.allocate(headerSize)
writeLocalFileHeader(name, size = 0, compressedSize = 0, crc32 = 0, method = ZipEntry.STORED, buffer = input)
input.position(0)
input.limit(headerSize)
resultStream.writeRawEntry(input, name, 0, 0, ZipEntry.STORED, 0, headerSize)
resultStream.writeRawEntry(input, name, 0, 0, ZipEntry.STORED, 0, headerSize, indexWriter)
}
fun dir(name: String) {
resultStream.addDirEntry(name)
fun dir(name: String, indexWriter: IkvIndexBuilder?) {
resultStream.addDirEntry(name, indexWriter)
}
override fun close() {

View File

@@ -1,4 +1,4 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.io
import java.nio.channels.FileChannel
@@ -14,17 +14,19 @@ enum class AddDirEntriesMode {
ALL
}
fun zipWithCompression(targetFile: Path,
dirs: Map<Path, String>,
compressionLevel: Int = Deflater.DEFAULT_COMPRESSION,
addDirEntriesMode: AddDirEntriesMode = AddDirEntriesMode.NONE,
overwrite: Boolean = false,
fileFilter: ((name: String) -> Boolean)? = null) {
fun zipWithCompression(
targetFile: Path,
dirs: Map<Path, String>,
compressionLevel: Int = Deflater.DEFAULT_COMPRESSION,
addDirEntriesMode: AddDirEntriesMode = AddDirEntriesMode.NONE,
overwrite: Boolean = false,
fileFilter: ((name: String) -> Boolean)? = null,
) {
Files.createDirectories(targetFile.parent)
ZipFileWriter(channel = FileChannel.open(targetFile, if (overwrite) W_OVERWRITE else W_CREATE_NEW),
deflater = if (compressionLevel == Deflater.NO_COMPRESSION) null else Deflater(compressionLevel, true)).use { zipFileWriter ->
if (addDirEntriesMode == AddDirEntriesMode.NONE) {
doArchive(zipFileWriter = zipFileWriter, fileAdded = fileFilter, dirs = dirs)
doArchive(zipFileWriter = zipFileWriter, fileAdded = fileFilter, dirs = dirs, indexWriter = null)
}
else {
val dirNameSetToAdd = LinkedHashSet<String>()
@@ -42,48 +44,54 @@ fun zipWithCompression(targetFile: Path,
}
}
doArchive(zipFileWriter = zipFileWriter, fileAdded = fileAdded, dirs = dirs)
doArchive(zipFileWriter = zipFileWriter, fileAdded = fileAdded, dirs = dirs, indexWriter = null)
for (dir in dirNameSetToAdd) {
zipFileWriter.dir(dir)
zipFileWriter.dir(name = dir, indexWriter = null)
}
}
}
}
// symlinks are not supported but can be easily implemented - see CollectingVisitor.visitFile
fun zip(targetFile: Path,
dirs: Map<Path, String>,
addDirEntriesMode: AddDirEntriesMode = AddDirEntriesMode.RESOURCE_ONLY,
overwrite: Boolean = false,
fileFilter: ((name: String) -> Boolean)? = null) {
fun zip(
targetFile: Path,
dirs: Map<Path, String>,
addDirEntriesMode: AddDirEntriesMode = AddDirEntriesMode.RESOURCE_ONLY,
overwrite: Boolean = false,
fileFilter: ((name: String) -> Boolean)? = null,
) {
Files.createDirectories(targetFile.parent)
ZipFileWriter(channel = FileChannel.open(targetFile, if (overwrite) W_OVERWRITE else W_CREATE_NEW)).use { zipFileWriter ->
if (addDirEntriesMode == AddDirEntriesMode.NONE) {
doArchive(zipFileWriter = zipFileWriter, fileAdded = fileFilter, dirs = dirs)
doArchive(zipFileWriter = zipFileWriter, fileAdded = fileFilter, dirs = dirs, indexWriter = null)
}
else {
val packageIndexBuilder = PackageIndexBuilder()
val fileAdded = { name: String ->
if (fileFilter != null && !fileFilter(name)) {
false
}
else {
packageIndexBuilder.addFile(name, addClassDir = addDirEntriesMode == AddDirEntriesMode.ALL)
true
}
}
doArchive(zipFileWriter = zipFileWriter, fileAdded = fileAdded, dirs = dirs)
doArchive(
zipFileWriter = zipFileWriter,
fileAdded = { name ->
if (fileFilter != null && !fileFilter(name)) {
false
}
else {
packageIndexBuilder.addFile(name, addClassDir = addDirEntriesMode == AddDirEntriesMode.ALL)
true
}
},
dirs = dirs,
indexWriter = packageIndexBuilder.indexWriter,
)
packageIndexBuilder.writePackageIndex(zipFileWriter, addDirEntriesMode = addDirEntriesMode)
}
}
}
private fun doArchive(zipFileWriter: ZipFileWriter, fileAdded: ((String) -> Boolean)?, dirs: Map<Path, String>) {
private fun doArchive(zipFileWriter: ZipFileWriter, fileAdded: ((String) -> Boolean)?, dirs: Map<Path, String>, indexWriter: IkvIndexBuilder?) {
val archiver = ZipArchiver(zipFileWriter, fileAdded)
for ((dir, prefix) in dirs.entries) {
val normalizedDir = dir.toAbsolutePath().normalize()
archiver.setRootDir(normalizedDir, prefix)
archiveDir(normalizedDir, archiver, excludes = null)
archiveDir(normalizedDir, archiver, excludes = null, indexWriter = indexWriter)
}
}
@@ -99,7 +107,7 @@ private fun addDirWithParents(name: String, dirNameSetToAdd: MutableSet<String>)
}
}
class ZipArchiver(private val zipCreator: ZipFileWriter, val fileAdded: ((String) -> Boolean)? = null) : AutoCloseable {
class ZipArchiver(private val zipCreator: ZipFileWriter, @JvmField val fileAdded: ((String) -> Boolean)? = null) : AutoCloseable {
private var localPrefixLength = -1
private var archivePrefix = ""
@@ -114,10 +122,10 @@ class ZipArchiver(private val zipCreator: ZipFileWriter, val fileAdded: ((String
localPrefixLength = rootDir.toString().length + 1
}
fun addFile(file: Path) {
fun addFile(file: Path, indexWriter: IkvIndexBuilder?) {
val name = archivePrefix + file.toString().substring(localPrefixLength).replace('\\', '/')
if (fileAdded == null || fileAdded.invoke(name)) {
zipCreator.file(name, file)
zipCreator.file(name, file, indexWriter)
}
}
@@ -126,7 +134,7 @@ class ZipArchiver(private val zipCreator: ZipFileWriter, val fileAdded: ((String
}
}
fun archiveDir(startDir: Path, archiver: ZipArchiver, excludes: List<PathMatcher>? = emptyList()) {
fun archiveDir(startDir: Path, archiver: ZipArchiver, indexWriter: IkvIndexBuilder?, excludes: List<PathMatcher>? = emptyList()) {
val dirCandidates = ArrayDeque<Path>()
dirCandidates.add(startDir)
val tempList = ArrayList<Path>()
@@ -163,20 +171,8 @@ fun archiveDir(startDir: Path, archiver: ZipArchiver, excludes: List<PathMatcher
dirCandidates.add(file)
}
else {
archiver.addFile(file)
archiver.addFile(file, indexWriter)
}
}
}
}
inline fun copyZipRaw(sourceFile: Path,
packageIndexBuilder: PackageIndexBuilder,
zipCreator: ZipFileWriter,
crossinline filter: (entryName: String) -> Boolean = { true }) {
readZipFile(sourceFile) { name, data ->
if (filter(name)) {
packageIndexBuilder.addFile(name)
zipCreator.uncompressedData(name, data())
}
}
}

View File

@@ -13,7 +13,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.util.lang;
package org.jetbrains.intellij.build.io;
import com.intellij.util.lang.Xxh3;
import org.assertj.core.api.AssertionsForClassTypes;
import java.nio.Buffer;
import java.nio.ByteBuffer;
@@ -30,7 +33,7 @@ final class HashFunctionTest {
}
private static void testArrays(byte[] data, long eh, int len) {
assertThat(Xxh3.hash(data)).isEqualTo(eh);
AssertionsForClassTypes.assertThat(Xxh3.hash(data)).isEqualTo(eh);
byte[] data2 = new byte[len + 2];
System.arraycopy(data, 0, data2, 1, len);

View File

@@ -1,5 +1,5 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.ikv.builder
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.io
import com.intellij.util.lang.Ikv
import com.intellij.util.lang.Xxh3

View File

@@ -13,20 +13,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.util.lang;
package org.jetbrains.intellij.build.io;
import com.intellij.util.lang.Xx3UnencodedString;
import com.intellij.util.lang.Xxh3;
import org.assertj.core.api.AssertionsForClassTypes;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
import org.junit.jupiter.params.provider.ValueSource;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Random;
import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat;
@@ -60,7 +59,7 @@ public class XxHash3Test {
case 10_000 -> -4959357597963000776L;
default -> throw new UnsupportedOperationException("Unknown size");
};
assertThat(Xxh3.hashLongs(data)).isEqualTo(expected);
AssertionsForClassTypes.assertThat(Xxh3.hashLongs(data)).isEqualTo(expected);
}
@Test
@@ -100,7 +99,7 @@ public class XxHash3Test {
}
private static void checkPackage(String s, long expected) {
assertThat(Xx3UnencodedString.hashUnencodedString(s.replace('.', '/'))).describedAs("Hash as string of: " + s).isEqualTo(expected);
AssertionsForClassTypes.assertThat(Xx3UnencodedString.hashUnencodedString(s.replace('.', '/'))).describedAs("Hash as string of: " + s).isEqualTo(expected);
}
private static void testUnencodedString(String s, long expected) {

View File

@@ -25,7 +25,6 @@
<orderEntry type="module" module-name="intellij.platform.util.xmlDom" scope="PROVIDED" />
<orderEntry type="library" name="okhttp" level="project" />
<orderEntry type="module" module-name="intellij.idea.community.build.tasks" />
<orderEntry type="module" module-name="intellij.platform.util.immutableKeyValueStore" />
<orderEntry type="module" module-name="intellij.platform.util.rt.java8" />
<orderEntry type="library" name="gson" level="project" />
<orderEntry type="library" name="kotlinx-coroutines-core" level="project" />

View File

@@ -151,7 +151,11 @@ fun reorderJar(jarFile: Path, orderedNames: List<String>) {
for (entry in entries) {
packageIndexBuilder.addFile(entry.name)
zipCreator.uncompressedData(entry.name, entry.getByteBuffer(sourceZip))
zipCreator.uncompressedData(
nameString = entry.name,
data = entry.getByteBuffer(sourceZip),
indexWriter = packageIndexBuilder.indexWriter,
)
}
packageIndexBuilder.writePackageIndex(zipCreator)
}

View File

@@ -206,11 +206,10 @@ class ModuleItem(
}
override fun equals(other: Any?): Boolean {
return this === other ||
other is ModuleItem && moduleName == other.moduleName && relativeOutputFile == other.relativeOutputFile
return this === other || other is ModuleItem && moduleName == other.moduleName && relativeOutputFile == other.relativeOutputFile
}
override fun hashCode(): Int = 31 * moduleName.hashCode() + relativeOutputFile.hashCode()
override fun toString(): String = "ModuleItem(moduleName=${moduleName}, relativeOutputFile=${relativeOutputFile}, reason=${reason})"
override fun toString(): String = "ModuleItem(moduleName=$moduleName, relativeOutputFile=$relativeOutputFile, reason=$reason)"
}

View File

@@ -1,4 +1,4 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.impl
import com.intellij.platform.diagnostic.telemetry.helpers.use
@@ -10,7 +10,7 @@ import org.jetbrains.intellij.build.CompilationContext
import org.jetbrains.intellij.build.TraceManager.spanBuilder
import org.jetbrains.intellij.build.io.ZipArchiver
import org.jetbrains.intellij.build.io.archiveDir
import org.jetbrains.intellij.build.io.writeNewZip
import org.jetbrains.intellij.build.io.writeNewZipWithoutIndex
import java.nio.file.Files
import java.nio.file.Path
@@ -96,12 +96,12 @@ private suspend fun buildResourcesForHelpPlugin(resourceRoot: Path, classPath: L
jvmArgs = emptyList(),
classPath = classPath)
}
writeNewZip(assetJar, compress = true) { zipCreator ->
writeNewZipWithoutIndex(assetJar, compress = true) { zipCreator ->
val archiver = ZipArchiver(zipCreator)
archiver.setRootDir(resourceRoot)
archiveDir(resourceRoot.resolve("topics"), archiver)
archiveDir(resourceRoot.resolve("images"), archiver)
archiveDir(resourceRoot.resolve("search"), archiver)
archiveDir(resourceRoot.resolve("topics"), archiver, null)
archiveDir(resourceRoot.resolve("images"), archiver, null)
archiveDir(resourceRoot.resolve("search"), archiver, null)
}
}
}

View File

@@ -1262,15 +1262,15 @@ private fun archivePlugin(optimized: Boolean,
source: Path,
context: BuildContext) {
if (optimized) {
writeNewZip(target, compress = compress, withOptimizedMetadataEnabled = false) { zipCreator ->
writeNewZipWithoutIndex(target, compress = compress) { zipCreator ->
ZipArchiver(zipCreator).use { archiver ->
if (Files.isDirectory(source)) {
archiver.setRootDir(source, source.fileName.toString())
archiveDir(startDir = source, archiver = archiver, excludes = null)
archiveDir(startDir = source, archiver = archiver, excludes = null, indexWriter = null)
}
else {
archiver.setRootDir(source.parent)
archiver.addFile(source)
archiver.addFile(source, indexWriter = null)
}
}
}
@@ -1300,7 +1300,7 @@ private fun buildBlockMap(file: Path, json: JSON) {
val fileParent = file.parent
val fileName = file.fileName.toString()
writeNewZip(fileParent.resolve("$fileName.blockmap.zip"), compress = true) {
writeNewZipWithoutIndex(fileParent.resolve("$fileName.blockmap.zip"), compress = true) {
it.compressedData("blockmap.json", ByteBuffer.wrap(bytes))
}

View File

@@ -2,8 +2,9 @@
package org.jetbrains.intellij.build.impl
import org.jetbrains.intellij.build.io.PackageIndexBuilder
import org.jetbrains.intellij.build.io.ZipFileWriter
import org.jetbrains.intellij.build.io.writeNewZip
import org.jetbrains.intellij.build.io.writeNewZipWithoutIndex
import java.nio.ByteBuffer
import java.nio.channels.SeekableByteChannel
import java.nio.channels.WritableByteChannel
@@ -23,8 +24,11 @@ internal fun buildKeymapPlugin(keymaps: Array<String>, buildNumber: String, targ
val buffer = ByteBuffer.allocate(128 * 1024)
ZipFileWriter(WritableByteChannelBackedByByteBuffer(buffer)).use { zipCreator ->
zipCreator.uncompressedData("META-INF/plugin.xml", ByteBuffer.wrap(pluginXmlData))
val packageIndexBuilder = PackageIndexBuilder()
zipCreator.uncompressedData("META-INF/plugin.xml", ByteBuffer.wrap(pluginXmlData), packageIndexBuilder.indexWriter)
packageIndexBuilder.addFile("META-INF/plugin.xml")
packageIndexBuilder.addFile("META-INF/pluginIcon.svg")
@Suppress("SpellCheckingInspection")
zipCreator.uncompressedData("META-INF/pluginIcon.svg",
"<svg xmlns=\"http://www.w3.org/2000/svg\" width=\"40\" height=\"40\" viewBox=\"0 0 40 40\">\n" +
@@ -33,7 +37,9 @@ internal fun buildKeymapPlugin(keymaps: Array<String>, buildNumber: String, targ
"15 L28,15 L28,11 Z M22,11 L26,11 L26,15 L22,15 L22,11 Z M10,11 L14,11 L14,15 L10,15 L10,11 Z M4," +
"11 L8,11 L8,15 L4,15 L4,11 Z M4,5 L8,5 L8,9 L4,9 L4,5 Z M10,5 L14,5 L14,9 L10,9 L10,5 Z M16,11 L20," +
"11 L20,15 L16,15 L16,11 Z M25,21 L11,21 L11,17 L25,17 L25,21 Z\" transform=\"translate(2 7)\"/>\n" +
"</svg>\n")
"</svg>\n", packageIndexBuilder.indexWriter)
packageIndexBuilder.addFile("META-INF/pluginIcon_dark.svg")
@Suppress("SpellCheckingInspection")
zipCreator.uncompressedData("META-INF/pluginIcon_dark.svg",
"<svg xmlns=\"http://www.w3.org/2000/svg\" width=\"40\" height=\"40\" viewBox=\"0 0 40 40\">\n" +
@@ -42,17 +48,21 @@ internal fun buildKeymapPlugin(keymaps: Array<String>, buildNumber: String, targ
"15 L28,15 L28,11 Z M22,11 L26,11 L26,15 L22,15 L22,11 Z M10,11 L14,11 L14,15 L10,15 L10,11 Z M4," +
"11 L8,11 L8,15 L4,15 L4,11 Z M4,5 L8,5 L8,9 L4,9 L4,5 Z M10,5 L14,5 L14,9 L10,9 L10,5 Z M16,11 L20," +
"11 L20,15 L16,15 L16,11 Z M25,21 L11,21 L11,17 L25,17 L25,21 Z\" transform=\"translate(2 7)\"/>\n" +
"</svg>\n")
"</svg>\n", packageIndexBuilder.indexWriter)
for (name in keymaps) {
zipCreator.file("keymaps/$name.xml", keymapDir.resolve("$name.xml"))
val keymapFile = "keymaps/$name.xml"
packageIndexBuilder.addFile(keymapFile)
zipCreator.file(keymapFile, keymapDir.resolve("$name.xml"), packageIndexBuilder.indexWriter)
}
packageIndexBuilder.writePackageIndex(zipCreator)
}
buffer.flip()
val resultFile = targetDir.resolve("${shortName}Keymap.zip")
writeNewZip(resultFile, compress = true) {
it.uncompressedData("${shortName}Keymap/lib/${shortName}Keymap.jar", buffer)
writeNewZipWithoutIndex(resultFile, compress = true) {
it.uncompressedData("${shortName}Keymap/lib/${shortName}Keymap.jar", buffer, null)
}
return Pair(resultFile, pluginXmlData)
}

View File

@@ -1,10 +1,10 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("ReplacePutWithAssignment")
package org.jetbrains.intellij.build.impl
import com.intellij.openapi.util.SystemInfoRt
import com.intellij.platform.diagnostic.telemetry.helpers.useWithScope
import com.intellij.platform.diagnostic.telemetry.helpers.use
import com.intellij.platform.diagnostic.telemetry.helpers.useWithScope
import com.jetbrains.signatureverifier.ILogger
import com.jetbrains.signatureverifier.InvalidDataException
import com.jetbrains.signatureverifier.crypt.SignatureVerificationParams
@@ -123,17 +123,17 @@ private fun copyZipReplacing(origin: Path, entries: Map<String, Path>, context:
.setAttribute(AttributeKey.stringArrayKey("unsigned"), entries.keys.toList())
.use {
transformZipUsingTempFile(origin) { zipWriter ->
val index = PackageIndexBuilder()
val packageIndexBuilder = PackageIndexBuilder()
readZipFile(origin) { name, dataSupplier ->
index.addFile(name)
packageIndexBuilder.addFile(name)
if (entries.containsKey(name)) {
zipWriter.file(name, entries.getValue(name))
zipWriter.file(name, entries.getValue(name), packageIndexBuilder.indexWriter)
}
else {
zipWriter.uncompressedData(name, dataSupplier())
zipWriter.uncompressedData(name, dataSupplier(), packageIndexBuilder.indexWriter)
}
}
index.writePackageIndex(zipWriter)
packageIndexBuilder.writePackageIndex(zipWriter)
}
Files.setLastModifiedTime(origin, FileTime.from(context.options.buildDateInSeconds, TimeUnit.SECONDS))
}

View File

@@ -180,7 +180,14 @@ internal suspend fun buildJar(
})
val normalizedDir = source.dir.toAbsolutePath().normalize()
archiver.setRootDir(normalizedDir, source.prefix)
archiveDir(startDir = normalizedDir, archiver = archiver, excludes = source.excludes.takeIf(List<PathMatcher>::isNotEmpty))
archiveDir(
startDir = normalizedDir,
archiver = archiver,
indexWriter = packageIndexBuilder?.indexWriter,
excludes = source.excludes.takeIf(
List<PathMatcher>::isNotEmpty,
)
)
}
is InMemoryContentSource -> {
@@ -192,9 +199,14 @@ internal suspend fun buildJar(
}
packageIndexBuilder?.addFile(source.relativePath)
zipCreator.uncompressedData(source.relativePath, source.data.size) {
it.put(source.data)
}
zipCreator.uncompressedData(
nameString = source.relativePath,
maxSize = source.data.size,
indexWriter = packageIndexBuilder?.indexWriter,
dataWriter = {
it.put(source.data)
},
)
}
is ZipSource -> {
@@ -276,11 +288,11 @@ private suspend fun handleZipSource(source: ZipSource,
zipCreator.compressedData(name, dataSupplier())
}
else {
zipCreator.uncompressedData(name, dataSupplier())
zipCreator.uncompressedData(name, dataSupplier(), indexWriter = packageIndexBuilder?.indexWriter)
}
}
else {
zipCreator.file(name, file)
zipCreator.file(name, file, indexWriter = packageIndexBuilder?.indexWriter)
Files.delete(file)
}
}
@@ -293,7 +305,7 @@ private suspend fun handleZipSource(source: ZipSource,
zipCreator.compressedData(name, data)
}
else {
zipCreator.uncompressedData(name, data)
zipCreator.uncompressedData(name, data, indexWriter = packageIndexBuilder?.indexWriter)
}
}
}

View File

@@ -10,8 +10,8 @@
<orderEntry type="library" scope="TEST" name="jmh-core" level="project" />
<orderEntry type="library" scope="TEST" name="jmh-generator-annprocess" level="project" />
<orderEntry type="library" scope="TEST" name="kotlin-stdlib" level="project" />
<orderEntry type="module" module-name="intellij.platform.util.immutableKeyValueStore" scope="TEST" />
<orderEntry type="module" module-name="intellij.platform.util.zip" scope="TEST" />
<orderEntry type="module" module-name="intellij.platform.util.rt.java8" scope="TEST" />
<orderEntry type="module" module-name="intellij.idea.community.build.tasks" scope="TEST" />
</component>
</module>

View File

@@ -2,7 +2,7 @@
package com.intellij.platform.util.immutableKeyValueStore.benchmark
import com.intellij.util.lang.Xxh3
import org.jetbrains.ikv.builder.IkvWriter
import org.jetbrains.intellij.build.io.IkvWriter
import java.nio.channels.FileChannel
import java.nio.file.Files
import java.nio.file.Path