mirror of
https://gitflic.ru/project/openide/openide.git
synced 2026-03-22 15:19:59 +07:00
IJPL-866 refactor - extract index writer from ZipFileWriter
GitOrigin-RevId: 04b6d46b87bd373df5a41672c2406703044e1510
This commit is contained in:
committed by
intellij-monorepo-bot
parent
d2d23c77a1
commit
95c0264e49
1
.idea/modules.xml
generated
1
.idea/modules.xml
generated
@@ -1082,7 +1082,6 @@
|
||||
<module fileurl="file://$PROJECT_DIR$/platform/util/diff/intellij.platform.util.diff.iml" filepath="$PROJECT_DIR$/platform/util/diff/intellij.platform.util.diff.iml" />
|
||||
<module fileurl="file://$PROJECT_DIR$/platform/util-ex/intellij.platform.util.ex.iml" filepath="$PROJECT_DIR$/platform/util-ex/intellij.platform.util.ex.iml" />
|
||||
<module fileurl="file://$PROJECT_DIR$/platform/util/http/intellij.platform.util.http.iml" filepath="$PROJECT_DIR$/platform/util/http/intellij.platform.util.http.iml" />
|
||||
<module fileurl="file://$PROJECT_DIR$/platform/util/immutable-key-value-store/intellij.platform.util.immutableKeyValueStore.iml" filepath="$PROJECT_DIR$/platform/util/immutable-key-value-store/intellij.platform.util.immutableKeyValueStore.iml" />
|
||||
<module fileurl="file://$PROJECT_DIR$/platform/util/immutable-key-value-store/benchmark/intellij.platform.util.immutableKeyValueStore.benchmark.iml" filepath="$PROJECT_DIR$/platform/util/immutable-key-value-store/benchmark/intellij.platform.util.immutableKeyValueStore.benchmark.iml" />
|
||||
<module fileurl="file://$PROJECT_DIR$/platform/util/jdom/intellij.platform.util.jdom.iml" filepath="$PROJECT_DIR$/platform/util/jdom/intellij.platform.util.jdom.iml" />
|
||||
<module fileurl="file://$PROJECT_DIR$/platform/util/nanoxml/intellij.platform.util.nanoxml.iml" filepath="$PROJECT_DIR$/platform/util/nanoxml/intellij.platform.util.nanoxml.iml" />
|
||||
|
||||
@@ -42,7 +42,6 @@
|
||||
<orderEntry type="library" name="jackson-jr-objects" level="project" />
|
||||
<orderEntry type="library" name="jackson" level="project" />
|
||||
<orderEntry type="library" name="kotlinx-coroutines-core" level="project" />
|
||||
<orderEntry type="module" module-name="intellij.platform.util.immutableKeyValueStore" />
|
||||
<orderEntry type="module" module-name="intellij.platform.util.rt.java8" />
|
||||
<orderEntry type="module" module-name="intellij.java.rt" />
|
||||
<orderEntry type="module" module-name="intellij.platform.util.rt" />
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
package org.jetbrains.ikv.builder
|
||||
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
package org.jetbrains.intellij.build.io
|
||||
|
||||
import com.intellij.util.lang.ByteBufferCleaner
|
||||
import java.nio.ByteBuffer
|
||||
@@ -9,6 +9,7 @@ import java.nio.ByteOrder
|
||||
|
||||
class IkvIndexBuilder(private val writeSize: Boolean = true) {
|
||||
private val entries = LinkedHashSet<IkvIndexEntry>()
|
||||
@JvmField val names = mutableListOf<ByteArray>()
|
||||
|
||||
fun entry(key: Long, offset: Long, size: Int): IkvIndexEntry {
|
||||
val entry = LongKeyedEntry(longKey = key, offset = offset)
|
||||
@@ -1,5 +1,5 @@
|
||||
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
package org.jetbrains.ikv.builder
|
||||
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
package org.jetbrains.intellij.build.io
|
||||
|
||||
import java.nio.ByteBuffer
|
||||
import java.nio.channels.FileChannel
|
||||
@@ -1,7 +1,8 @@
|
||||
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
package org.jetbrains.intellij.build.io
|
||||
|
||||
import com.intellij.util.lang.Xx3UnencodedString
|
||||
import com.intellij.util.lang.Xxh3
|
||||
import it.unimi.dsi.fastutil.longs.LongOpenHashSet
|
||||
|
||||
class PackageIndexBuilder {
|
||||
@@ -13,6 +14,8 @@ class PackageIndexBuilder {
|
||||
private val dirsToRegister = HashSet<String>()
|
||||
private var wasWritten = false
|
||||
|
||||
@JvmField val indexWriter = IkvIndexBuilder()
|
||||
|
||||
fun addFile(name: String, addClassDir: Boolean = false) {
|
||||
val i = name.lastIndexOf('/')
|
||||
val packageNameHash = if (i == -1) 0 else Xx3UnencodedString.hashUnencodedStringRange(name, i)
|
||||
@@ -47,11 +50,15 @@ class PackageIndexBuilder {
|
||||
|
||||
val stream = zipCreator.resultStream
|
||||
if (addDirEntriesMode == AddDirEntriesMode.NONE) {
|
||||
stream.addDirsToIndex(sortedDirsToRegister)
|
||||
for (dirName in sortedDirsToRegister) {
|
||||
val nameBytes = dirName.encodeToByteArray()
|
||||
indexWriter.add(indexWriter.entry(key = Xxh3.hash(nameBytes), offset = 0, size = -1))
|
||||
indexWriter.names.add(nameBytes)
|
||||
}
|
||||
}
|
||||
else {
|
||||
for (dir in sortedDirsToRegister) {
|
||||
stream.addDirEntry(dir)
|
||||
stream.addDirEntry(dir, indexWriter)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
package org.jetbrains.intellij.build.io
|
||||
|
||||
import com.intellij.util.lang.ImmutableZipFile
|
||||
import com.intellij.util.lang.Xxh3
|
||||
import org.jetbrains.ikv.builder.IkvIndexBuilder
|
||||
import java.io.IOException
|
||||
import java.nio.ByteBuffer
|
||||
import java.nio.ByteOrder
|
||||
@@ -17,8 +16,9 @@ private const val INDEX_FORMAT_VERSION: Byte = 4
|
||||
|
||||
const val INDEX_FILENAME: String = "__index__"
|
||||
|
||||
internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
|
||||
private val withOptimizedMetadataEnabled: Boolean) : AutoCloseable {
|
||||
internal class ZipArchiveOutputStream(
|
||||
private val channel: WritableByteChannel,
|
||||
) : AutoCloseable {
|
||||
private var classPackages: LongArray? = null
|
||||
private var resourcePackages: LongArray? = null
|
||||
|
||||
@@ -26,17 +26,15 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
|
||||
private var entryCount = 0
|
||||
|
||||
private var metadataBuffer = ByteBuffer.allocateDirect(2 * 1024 * 1024).order(ByteOrder.LITTLE_ENDIAN)
|
||||
|
||||
// 1 MB should be enough for the end of the central directory record
|
||||
private val buffer = ByteBuffer.allocateDirect(1024 * 1024).order(ByteOrder.LITTLE_ENDIAN)
|
||||
|
||||
private val names = mutableListOf<ByteArray>()
|
||||
private val indexWriter = IkvIndexBuilder()
|
||||
|
||||
private var channelPosition = 0L
|
||||
|
||||
private val fileChannel = channel as? FileChannel
|
||||
|
||||
fun addDirEntry(name: String) {
|
||||
fun addDirEntry(name: String, indexWriter: IkvIndexBuilder?) {
|
||||
if (finished) {
|
||||
throw IOException("Stream has already been finished")
|
||||
}
|
||||
@@ -75,10 +73,19 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
|
||||
buffer.flip()
|
||||
writeBuffer(buffer)
|
||||
|
||||
writeCentralFileHeader(0, 0, ZipEntry.STORED, 0, nameInArchive, offset, dataOffset = -1, normalName = key)
|
||||
writeCentralFileHeader(0, 0, ZipEntry.STORED, 0, nameInArchive, offset, dataOffset = -1, normalName = key, indexWriter = indexWriter)
|
||||
}
|
||||
|
||||
fun writeRawEntry(header: ByteBuffer, content: ByteBuffer, name: ByteArray, size: Int, compressedSize: Int, method: Int, crc: Long) {
|
||||
fun writeRawEntry(
|
||||
header: ByteBuffer,
|
||||
content: ByteBuffer,
|
||||
name: ByteArray,
|
||||
size: Int,
|
||||
compressedSize: Int,
|
||||
method: Int,
|
||||
crc: Long,
|
||||
indexWriter: IkvIndexBuilder?,
|
||||
) {
|
||||
if (finished) {
|
||||
throw IOException("Stream has already been finished")
|
||||
}
|
||||
@@ -91,10 +98,28 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
|
||||
writeBuffer(header)
|
||||
writeBuffer(content)
|
||||
|
||||
writeCentralFileHeader(size, compressedSize, method, crc, name, offset, dataOffset = dataOffset)
|
||||
writeCentralFileHeader(
|
||||
size = size,
|
||||
compressedSize = compressedSize,
|
||||
method = method,
|
||||
crc = crc,
|
||||
name = name,
|
||||
offset = offset,
|
||||
dataOffset = dataOffset,
|
||||
indexWriter = indexWriter,
|
||||
)
|
||||
}
|
||||
|
||||
fun writeRawEntry(content: ByteBuffer, name: ByteArray, size: Int, compressedSize: Int, method: Int, crc: Long, headerSize: Int) {
|
||||
fun writeRawEntry(
|
||||
content: ByteBuffer,
|
||||
name: ByteArray,
|
||||
size: Int,
|
||||
compressedSize: Int,
|
||||
method: Int,
|
||||
crc: Long,
|
||||
headerSize: Int,
|
||||
indexWriter: IkvIndexBuilder?,
|
||||
) {
|
||||
if (finished) {
|
||||
throw IOException("Stream has already been finished")
|
||||
}
|
||||
@@ -104,10 +129,19 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
|
||||
assert(method != -1)
|
||||
|
||||
writeBuffer(content)
|
||||
writeCentralFileHeader(size, compressedSize, method, crc, name, offset, dataOffset = offset + headerSize)
|
||||
writeCentralFileHeader(size, compressedSize, method, crc, name, offset, dataOffset = offset + headerSize, indexWriter = indexWriter)
|
||||
}
|
||||
|
||||
fun writeEntryHeaderAt(name: ByteArray, header: ByteBuffer, position: Long, size: Int, compressedSize: Int, crc: Long, method: Int) {
|
||||
fun writeEntryHeaderAt(
|
||||
name: ByteArray,
|
||||
header: ByteBuffer,
|
||||
position: Long,
|
||||
size: Int,
|
||||
compressedSize: Int,
|
||||
crc: Long,
|
||||
method: Int,
|
||||
indexWriter: IkvIndexBuilder?,
|
||||
) {
|
||||
if (finished) {
|
||||
throw IOException("Stream has already been finished")
|
||||
}
|
||||
@@ -134,16 +168,19 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
|
||||
entryCount++
|
||||
|
||||
assert(channelPosition == dataOffset + compressedSize)
|
||||
writeCentralFileHeader(size = size,
|
||||
compressedSize = compressedSize,
|
||||
method = method,
|
||||
crc = crc,
|
||||
name = name,
|
||||
offset = position,
|
||||
dataOffset = dataOffset)
|
||||
writeCentralFileHeader(
|
||||
size = size,
|
||||
compressedSize = compressedSize,
|
||||
method = method,
|
||||
crc = crc,
|
||||
name = name,
|
||||
offset = position,
|
||||
dataOffset = dataOffset,
|
||||
indexWriter = indexWriter,
|
||||
)
|
||||
}
|
||||
|
||||
private fun writeIndex(crc32: CRC32): Int {
|
||||
private fun writeIndex(crc32: CRC32, indexWriter: IkvIndexBuilder): Int {
|
||||
// write one by one to channel to avoid buffer overflow
|
||||
indexWriter.write {
|
||||
crc32.update(it)
|
||||
@@ -182,7 +219,7 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
|
||||
}
|
||||
|
||||
// write names
|
||||
for (list in names.asSequence().chunked(4096)) {
|
||||
for (list in indexWriter.names.asSequence().chunked(4096)) {
|
||||
writeData { buffer ->
|
||||
val shortBuffer = buffer.asShortBuffer()
|
||||
for (name in list) {
|
||||
@@ -192,7 +229,7 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
|
||||
}
|
||||
}
|
||||
|
||||
for (list in names.asSequence().chunked(1024)) {
|
||||
for (list in indexWriter.names.asSequence().chunked(1024)) {
|
||||
writeData { buffer ->
|
||||
for (name in list) {
|
||||
buffer.put(name)
|
||||
@@ -203,13 +240,13 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
|
||||
return indexDataEnd
|
||||
}
|
||||
|
||||
fun finish() {
|
||||
fun finish(indexWriter: IkvIndexBuilder?) {
|
||||
if (finished) {
|
||||
throw IOException("This archive has already been finished")
|
||||
}
|
||||
|
||||
val indexOffset: Int
|
||||
if (withOptimizedMetadataEnabled && entryCount != 0) {
|
||||
if (indexWriter != null && entryCount != 0) {
|
||||
// ditto on macOS doesn't like arbitrary data in zip file - wrap into zip entry
|
||||
val name = INDEX_FILENAME.toByteArray(Charsets.UTF_8)
|
||||
val headerSize = 30 + name.size
|
||||
@@ -217,7 +254,7 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
|
||||
val entryDataPosition = channelPosition
|
||||
|
||||
val crc32 = CRC32()
|
||||
indexOffset = writeIndex(crc32)
|
||||
indexOffset = writeIndex(crc32, indexWriter)
|
||||
|
||||
val size = (channelPosition - entryDataPosition).toInt()
|
||||
val crc = crc32.value
|
||||
@@ -226,13 +263,16 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
|
||||
writeLocalFileHeader(name = name, size = size, compressedSize = size, crc32 = crc, method = ZipEntry.STORED, buffer = buffer)
|
||||
buffer.flip()
|
||||
assert(buffer.remaining() == headerSize)
|
||||
writeEntryHeaderAt(name = name,
|
||||
header = buffer,
|
||||
position = headerPosition,
|
||||
size = size,
|
||||
compressedSize = size,
|
||||
crc = crc,
|
||||
method = ZipEntry.STORED)
|
||||
writeEntryHeaderAt(
|
||||
name = name,
|
||||
header = buffer,
|
||||
position = headerPosition,
|
||||
size = size,
|
||||
compressedSize = size,
|
||||
crc = crc,
|
||||
method = ZipEntry.STORED,
|
||||
indexWriter = indexWriter,
|
||||
)
|
||||
}
|
||||
else {
|
||||
indexOffset = -1
|
||||
@@ -262,7 +302,7 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
|
||||
buffer.putInt((centralDirectoryOffset and 0xffffffffL).toInt())
|
||||
|
||||
// comment length
|
||||
if (withOptimizedMetadataEnabled) {
|
||||
if (indexWriter != null) {
|
||||
buffer.putShort((Byte.SIZE_BYTES + Integer.BYTES).toShort())
|
||||
// version
|
||||
buffer.put(INDEX_FORMAT_VERSION)
|
||||
@@ -273,7 +313,11 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
|
||||
}
|
||||
}
|
||||
else {
|
||||
writeZip64End(centralDirectoryLength, centralDirectoryOffset, indexOffset)
|
||||
writeZip64End(
|
||||
centralDirectoryLength = centralDirectoryLength,
|
||||
centralDirectoryOffset = centralDirectoryOffset,
|
||||
optimizedMetadataOffset = indexOffset,
|
||||
)
|
||||
}
|
||||
buffer.flip()
|
||||
writeBuffer(buffer)
|
||||
@@ -306,7 +350,7 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
|
||||
buffer.putLong(centralDirectoryOffset)
|
||||
|
||||
// comment length
|
||||
if (withOptimizedMetadataEnabled) {
|
||||
if (optimizedMetadataOffset != -1) {
|
||||
// version
|
||||
buffer.put(INDEX_FORMAT_VERSION)
|
||||
buffer.putInt(optimizedMetadataOffset)
|
||||
@@ -375,7 +419,7 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
|
||||
try {
|
||||
if (!finished) {
|
||||
channel.use {
|
||||
finish()
|
||||
finish(indexWriter = null)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -385,23 +429,17 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
|
||||
}
|
||||
}
|
||||
|
||||
fun addDirsToIndex(dirNames: Collection<String>) {
|
||||
assert(withOptimizedMetadataEnabled)
|
||||
for (dirName in dirNames) {
|
||||
val nameBytes = dirName.toByteArray(Charsets.UTF_8)
|
||||
indexWriter.add(indexWriter.entry(key = Xxh3.hash(nameBytes), offset = 0, size = -1))
|
||||
names.add(nameBytes)
|
||||
}
|
||||
}
|
||||
|
||||
private fun writeCentralFileHeader(size: Int,
|
||||
compressedSize: Int,
|
||||
method: Int,
|
||||
crc: Long,
|
||||
name: ByteArray,
|
||||
offset: Long,
|
||||
dataOffset: Long,
|
||||
normalName: ByteArray = name) {
|
||||
private fun writeCentralFileHeader(
|
||||
size: Int,
|
||||
compressedSize: Int,
|
||||
method: Int,
|
||||
crc: Long,
|
||||
name: ByteArray,
|
||||
offset: Long,
|
||||
dataOffset: Long,
|
||||
normalName: ByteArray = name,
|
||||
indexWriter: IkvIndexBuilder?,
|
||||
) {
|
||||
var buffer = metadataBuffer
|
||||
if (buffer.remaining() < (46 + name.size)) {
|
||||
metadataBuffer = ByteBuffer.allocateDirect(buffer.capacity() * 2).order(ByteOrder.LITTLE_ENDIAN)
|
||||
@@ -422,9 +460,9 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel,
|
||||
// uncompressed size
|
||||
buffer.putInt(headerOffset + 24, size)
|
||||
|
||||
if (withOptimizedMetadataEnabled) {
|
||||
if (indexWriter != null) {
|
||||
indexWriter.add(indexWriter.entry(offset = dataOffset, size = size, key = Xxh3.hash(normalName)))
|
||||
names.add(normalName)
|
||||
indexWriter.names.add(normalName)
|
||||
}
|
||||
|
||||
// file name length
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
@file:Suppress("ConstPropertyName")
|
||||
|
||||
package org.jetbrains.intellij.build.io
|
||||
@@ -36,25 +36,24 @@ fun transformZipUsingTempFile(file: Path, task: (ZipFileWriter) -> Unit) {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
inline fun writeNewZip(file: Path,
|
||||
compress: Boolean = false,
|
||||
withOptimizedMetadataEnabled: Boolean = !compress,
|
||||
task: (ZipFileWriter) -> Unit) {
|
||||
inline fun writeNewZipWithoutIndex(
|
||||
file: Path,
|
||||
compress: Boolean = false,
|
||||
task: (ZipFileWriter) -> Unit,
|
||||
) {
|
||||
Files.createDirectories(file.parent)
|
||||
ZipFileWriter(channel = FileChannel.open(file, W_CREATE_NEW),
|
||||
deflater = if (compress) Deflater(Deflater.DEFAULT_COMPRESSION, true) else null,
|
||||
withOptimizedMetadataEnabled = !compress && withOptimizedMetadataEnabled).use {
|
||||
ZipFileWriter(
|
||||
channel = FileChannel.open(file, W_CREATE_NEW),
|
||||
deflater = if (compress) Deflater(Deflater.DEFAULT_COMPRESSION, true) else null,
|
||||
).use {
|
||||
task(it)
|
||||
}
|
||||
}
|
||||
|
||||
// you must pass SeekableByteChannel if files are written (`file` method)
|
||||
class ZipFileWriter(channel: WritableByteChannel,
|
||||
private val deflater: Deflater? = null,
|
||||
withOptimizedMetadataEnabled: Boolean = deflater == null) : AutoCloseable {
|
||||
class ZipFileWriter(channel: WritableByteChannel, private val deflater: Deflater? = null) : AutoCloseable {
|
||||
// size is written as part of optimized metadata - so, if compression is enabled, optimized metadata will be incorrect
|
||||
internal val resultStream = ZipArchiveOutputStream(channel, withOptimizedMetadataEnabled = withOptimizedMetadataEnabled)
|
||||
internal val resultStream = ZipArchiveOutputStream(channel)
|
||||
private val crc32 = CRC32()
|
||||
|
||||
private val bufferAllocator = ByteBufferAllocator()
|
||||
@@ -64,7 +63,7 @@ class ZipFileWriter(channel: WritableByteChannel,
|
||||
get() = resultStream.getChannelPosition()
|
||||
|
||||
@Suppress("DuplicatedCode")
|
||||
fun file(nameString: String, file: Path) {
|
||||
fun file(nameString: String, file: Path, indexWriter: IkvIndexBuilder?) {
|
||||
var isCompressed = deflater != null && !nameString.endsWith(".png")
|
||||
|
||||
val name = nameString.toByteArray()
|
||||
@@ -77,7 +76,7 @@ class ZipFileWriter(channel: WritableByteChannel,
|
||||
FileChannel.open(file, EnumSet.of(StandardOpenOption.READ)).use { channel ->
|
||||
size = channel.size().toInt()
|
||||
if (size == 0) {
|
||||
writeEmptyFile(name, headerSize)
|
||||
writeEmptyFile(name, headerSize, indexWriter)
|
||||
return
|
||||
}
|
||||
if (size < compressThreshold) {
|
||||
@@ -100,13 +99,16 @@ class ZipFileWriter(channel: WritableByteChannel,
|
||||
writeLocalFileHeader(name = name, size = size, compressedSize = compressedSize, crc32 = crc, method = method, buffer = buffer)
|
||||
buffer.position(0)
|
||||
assert(buffer.remaining() == headerSize)
|
||||
resultStream.writeEntryHeaderAt(name = name,
|
||||
header = buffer,
|
||||
position = headerPosition,
|
||||
size = size,
|
||||
compressedSize = compressedSize,
|
||||
crc = crc,
|
||||
method = method)
|
||||
resultStream.writeEntryHeaderAt(
|
||||
name = name,
|
||||
header = buffer,
|
||||
position = headerPosition,
|
||||
size = size,
|
||||
compressedSize = compressedSize,
|
||||
crc = crc,
|
||||
method = method,
|
||||
indexWriter = indexWriter,
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -129,7 +131,7 @@ class ZipFileWriter(channel: WritableByteChannel,
|
||||
writeLocalFileHeader(name, size, size, crc, ZipEntry.STORED, input)
|
||||
input.position(0)
|
||||
assert(input.remaining() == (size + headerSize))
|
||||
resultStream.writeRawEntry(input, name, size, size, ZipEntry.STORED, crc, headerSize)
|
||||
resultStream.writeRawEntry(input, name, size, size, ZipEntry.STORED, crc, headerSize, indexWriter)
|
||||
}
|
||||
|
||||
private fun writeLargeFile(fileSize: Long, channel: FileChannel, deflater: Deflater?): Long {
|
||||
@@ -226,19 +228,28 @@ class ZipFileWriter(channel: WritableByteChannel,
|
||||
writeLocalFileHeader(name, size, compressedSize, crc, ZipEntry.DEFLATED, output)
|
||||
output.position(0)
|
||||
assert(output.remaining() == (compressedSize + headerSize))
|
||||
resultStream.writeRawEntry(output, name, size, compressedSize, ZipEntry.DEFLATED, crc, headerSize)
|
||||
resultStream.writeRawEntry(
|
||||
content = output,
|
||||
name = name,
|
||||
size = size,
|
||||
compressedSize = compressedSize,
|
||||
method = ZipEntry.DEFLATED,
|
||||
crc = crc,
|
||||
headerSize = headerSize,
|
||||
indexWriter = null,
|
||||
)
|
||||
}
|
||||
|
||||
fun uncompressedData(nameString: String, data: String) {
|
||||
uncompressedData(nameString, ByteBuffer.wrap(data.toByteArray()))
|
||||
fun uncompressedData(nameString: String, data: String, indexWriter: IkvIndexBuilder?) {
|
||||
uncompressedData(nameString, ByteBuffer.wrap(data.toByteArray()), indexWriter)
|
||||
}
|
||||
|
||||
fun uncompressedData(nameString: String, data: ByteBuffer) {
|
||||
fun uncompressedData(nameString: String, data: ByteBuffer, indexWriter: IkvIndexBuilder?) {
|
||||
val name = nameString.toByteArray()
|
||||
val headerSize = 30 + name.size
|
||||
|
||||
if (!data.hasRemaining()) {
|
||||
writeEmptyFile(name, headerSize)
|
||||
writeEmptyFile(name, headerSize, indexWriter)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -252,10 +263,10 @@ class ZipFileWriter(channel: WritableByteChannel,
|
||||
val header = bufferAllocator.allocate(headerSize)
|
||||
writeLocalFileHeader(name, size, size, crc, ZipEntry.STORED, header)
|
||||
header.position(0)
|
||||
resultStream.writeRawEntry(header, data, name, size, size, ZipEntry.STORED, crc)
|
||||
resultStream.writeRawEntry(header, data, name, size, size, ZipEntry.STORED, crc, indexWriter = indexWriter)
|
||||
}
|
||||
|
||||
fun uncompressedData(nameString: String, maxSize: Int, dataWriter: (ByteBuffer) -> Unit) {
|
||||
fun uncompressedData(nameString: String, maxSize: Int, indexWriter: IkvIndexBuilder?, dataWriter: (ByteBuffer) -> Unit) {
|
||||
val name = nameString.toByteArray()
|
||||
val headerSize = 30 + name.size
|
||||
|
||||
@@ -274,19 +285,19 @@ class ZipFileWriter(channel: WritableByteChannel,
|
||||
writeLocalFileHeader(name, size, size, crc, ZipEntry.STORED, output)
|
||||
output.position(0)
|
||||
assert(output.remaining() == (size + headerSize))
|
||||
resultStream.writeRawEntry(output, name, size, size, ZipEntry.STORED, crc, headerSize)
|
||||
resultStream.writeRawEntry(output, name, size, size, ZipEntry.STORED, crc, headerSize, indexWriter = indexWriter)
|
||||
}
|
||||
|
||||
private fun writeEmptyFile(name: ByteArray, headerSize: Int) {
|
||||
private fun writeEmptyFile(name: ByteArray, headerSize: Int, indexWriter: IkvIndexBuilder?) {
|
||||
val input = bufferAllocator.allocate(headerSize)
|
||||
writeLocalFileHeader(name, size = 0, compressedSize = 0, crc32 = 0, method = ZipEntry.STORED, buffer = input)
|
||||
input.position(0)
|
||||
input.limit(headerSize)
|
||||
resultStream.writeRawEntry(input, name, 0, 0, ZipEntry.STORED, 0, headerSize)
|
||||
resultStream.writeRawEntry(input, name, 0, 0, ZipEntry.STORED, 0, headerSize, indexWriter)
|
||||
}
|
||||
|
||||
fun dir(name: String) {
|
||||
resultStream.addDirEntry(name)
|
||||
fun dir(name: String, indexWriter: IkvIndexBuilder?) {
|
||||
resultStream.addDirEntry(name, indexWriter)
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
package org.jetbrains.intellij.build.io
|
||||
|
||||
import java.nio.channels.FileChannel
|
||||
@@ -14,17 +14,19 @@ enum class AddDirEntriesMode {
|
||||
ALL
|
||||
}
|
||||
|
||||
fun zipWithCompression(targetFile: Path,
|
||||
dirs: Map<Path, String>,
|
||||
compressionLevel: Int = Deflater.DEFAULT_COMPRESSION,
|
||||
addDirEntriesMode: AddDirEntriesMode = AddDirEntriesMode.NONE,
|
||||
overwrite: Boolean = false,
|
||||
fileFilter: ((name: String) -> Boolean)? = null) {
|
||||
fun zipWithCompression(
|
||||
targetFile: Path,
|
||||
dirs: Map<Path, String>,
|
||||
compressionLevel: Int = Deflater.DEFAULT_COMPRESSION,
|
||||
addDirEntriesMode: AddDirEntriesMode = AddDirEntriesMode.NONE,
|
||||
overwrite: Boolean = false,
|
||||
fileFilter: ((name: String) -> Boolean)? = null,
|
||||
) {
|
||||
Files.createDirectories(targetFile.parent)
|
||||
ZipFileWriter(channel = FileChannel.open(targetFile, if (overwrite) W_OVERWRITE else W_CREATE_NEW),
|
||||
deflater = if (compressionLevel == Deflater.NO_COMPRESSION) null else Deflater(compressionLevel, true)).use { zipFileWriter ->
|
||||
if (addDirEntriesMode == AddDirEntriesMode.NONE) {
|
||||
doArchive(zipFileWriter = zipFileWriter, fileAdded = fileFilter, dirs = dirs)
|
||||
doArchive(zipFileWriter = zipFileWriter, fileAdded = fileFilter, dirs = dirs, indexWriter = null)
|
||||
}
|
||||
else {
|
||||
val dirNameSetToAdd = LinkedHashSet<String>()
|
||||
@@ -42,48 +44,54 @@ fun zipWithCompression(targetFile: Path,
|
||||
}
|
||||
}
|
||||
|
||||
doArchive(zipFileWriter = zipFileWriter, fileAdded = fileAdded, dirs = dirs)
|
||||
doArchive(zipFileWriter = zipFileWriter, fileAdded = fileAdded, dirs = dirs, indexWriter = null)
|
||||
for (dir in dirNameSetToAdd) {
|
||||
zipFileWriter.dir(dir)
|
||||
zipFileWriter.dir(name = dir, indexWriter = null)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// symlinks are not supported but can be easily implemented - see CollectingVisitor.visitFile
|
||||
fun zip(targetFile: Path,
|
||||
dirs: Map<Path, String>,
|
||||
addDirEntriesMode: AddDirEntriesMode = AddDirEntriesMode.RESOURCE_ONLY,
|
||||
overwrite: Boolean = false,
|
||||
fileFilter: ((name: String) -> Boolean)? = null) {
|
||||
fun zip(
|
||||
targetFile: Path,
|
||||
dirs: Map<Path, String>,
|
||||
addDirEntriesMode: AddDirEntriesMode = AddDirEntriesMode.RESOURCE_ONLY,
|
||||
overwrite: Boolean = false,
|
||||
fileFilter: ((name: String) -> Boolean)? = null,
|
||||
) {
|
||||
Files.createDirectories(targetFile.parent)
|
||||
ZipFileWriter(channel = FileChannel.open(targetFile, if (overwrite) W_OVERWRITE else W_CREATE_NEW)).use { zipFileWriter ->
|
||||
if (addDirEntriesMode == AddDirEntriesMode.NONE) {
|
||||
doArchive(zipFileWriter = zipFileWriter, fileAdded = fileFilter, dirs = dirs)
|
||||
doArchive(zipFileWriter = zipFileWriter, fileAdded = fileFilter, dirs = dirs, indexWriter = null)
|
||||
}
|
||||
else {
|
||||
val packageIndexBuilder = PackageIndexBuilder()
|
||||
val fileAdded = { name: String ->
|
||||
if (fileFilter != null && !fileFilter(name)) {
|
||||
false
|
||||
}
|
||||
else {
|
||||
packageIndexBuilder.addFile(name, addClassDir = addDirEntriesMode == AddDirEntriesMode.ALL)
|
||||
true
|
||||
}
|
||||
}
|
||||
doArchive(zipFileWriter = zipFileWriter, fileAdded = fileAdded, dirs = dirs)
|
||||
doArchive(
|
||||
zipFileWriter = zipFileWriter,
|
||||
fileAdded = { name ->
|
||||
if (fileFilter != null && !fileFilter(name)) {
|
||||
false
|
||||
}
|
||||
else {
|
||||
packageIndexBuilder.addFile(name, addClassDir = addDirEntriesMode == AddDirEntriesMode.ALL)
|
||||
true
|
||||
}
|
||||
},
|
||||
dirs = dirs,
|
||||
indexWriter = packageIndexBuilder.indexWriter,
|
||||
)
|
||||
packageIndexBuilder.writePackageIndex(zipFileWriter, addDirEntriesMode = addDirEntriesMode)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun doArchive(zipFileWriter: ZipFileWriter, fileAdded: ((String) -> Boolean)?, dirs: Map<Path, String>) {
|
||||
private fun doArchive(zipFileWriter: ZipFileWriter, fileAdded: ((String) -> Boolean)?, dirs: Map<Path, String>, indexWriter: IkvIndexBuilder?) {
|
||||
val archiver = ZipArchiver(zipFileWriter, fileAdded)
|
||||
for ((dir, prefix) in dirs.entries) {
|
||||
val normalizedDir = dir.toAbsolutePath().normalize()
|
||||
archiver.setRootDir(normalizedDir, prefix)
|
||||
archiveDir(normalizedDir, archiver, excludes = null)
|
||||
archiveDir(normalizedDir, archiver, excludes = null, indexWriter = indexWriter)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -99,7 +107,7 @@ private fun addDirWithParents(name: String, dirNameSetToAdd: MutableSet<String>)
|
||||
}
|
||||
}
|
||||
|
||||
class ZipArchiver(private val zipCreator: ZipFileWriter, val fileAdded: ((String) -> Boolean)? = null) : AutoCloseable {
|
||||
class ZipArchiver(private val zipCreator: ZipFileWriter, @JvmField val fileAdded: ((String) -> Boolean)? = null) : AutoCloseable {
|
||||
private var localPrefixLength = -1
|
||||
private var archivePrefix = ""
|
||||
|
||||
@@ -114,10 +122,10 @@ class ZipArchiver(private val zipCreator: ZipFileWriter, val fileAdded: ((String
|
||||
localPrefixLength = rootDir.toString().length + 1
|
||||
}
|
||||
|
||||
fun addFile(file: Path) {
|
||||
fun addFile(file: Path, indexWriter: IkvIndexBuilder?) {
|
||||
val name = archivePrefix + file.toString().substring(localPrefixLength).replace('\\', '/')
|
||||
if (fileAdded == null || fileAdded.invoke(name)) {
|
||||
zipCreator.file(name, file)
|
||||
zipCreator.file(name, file, indexWriter)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -126,7 +134,7 @@ class ZipArchiver(private val zipCreator: ZipFileWriter, val fileAdded: ((String
|
||||
}
|
||||
}
|
||||
|
||||
fun archiveDir(startDir: Path, archiver: ZipArchiver, excludes: List<PathMatcher>? = emptyList()) {
|
||||
fun archiveDir(startDir: Path, archiver: ZipArchiver, indexWriter: IkvIndexBuilder?, excludes: List<PathMatcher>? = emptyList()) {
|
||||
val dirCandidates = ArrayDeque<Path>()
|
||||
dirCandidates.add(startDir)
|
||||
val tempList = ArrayList<Path>()
|
||||
@@ -163,20 +171,8 @@ fun archiveDir(startDir: Path, archiver: ZipArchiver, excludes: List<PathMatcher
|
||||
dirCandidates.add(file)
|
||||
}
|
||||
else {
|
||||
archiver.addFile(file)
|
||||
archiver.addFile(file, indexWriter)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
inline fun copyZipRaw(sourceFile: Path,
|
||||
packageIndexBuilder: PackageIndexBuilder,
|
||||
zipCreator: ZipFileWriter,
|
||||
crossinline filter: (entryName: String) -> Boolean = { true }) {
|
||||
readZipFile(sourceFile) { name, data ->
|
||||
if (filter(name)) {
|
||||
packageIndexBuilder.addFile(name)
|
||||
zipCreator.uncompressedData(name, data())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -13,7 +13,10 @@
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.intellij.util.lang;
|
||||
package org.jetbrains.intellij.build.io;
|
||||
|
||||
import com.intellij.util.lang.Xxh3;
|
||||
import org.assertj.core.api.AssertionsForClassTypes;
|
||||
|
||||
import java.nio.Buffer;
|
||||
import java.nio.ByteBuffer;
|
||||
@@ -30,7 +33,7 @@ final class HashFunctionTest {
|
||||
}
|
||||
|
||||
private static void testArrays(byte[] data, long eh, int len) {
|
||||
assertThat(Xxh3.hash(data)).isEqualTo(eh);
|
||||
AssertionsForClassTypes.assertThat(Xxh3.hash(data)).isEqualTo(eh);
|
||||
|
||||
byte[] data2 = new byte[len + 2];
|
||||
System.arraycopy(data, 0, data2, 1, len);
|
||||
@@ -1,5 +1,5 @@
|
||||
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
package org.jetbrains.ikv.builder
|
||||
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
package org.jetbrains.intellij.build.io
|
||||
|
||||
import com.intellij.util.lang.Ikv
|
||||
import com.intellij.util.lang.Xxh3
|
||||
@@ -13,20 +13,19 @@
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.intellij.util.lang;
|
||||
package org.jetbrains.intellij.build.io;
|
||||
|
||||
import com.intellij.util.lang.Xx3UnencodedString;
|
||||
import com.intellij.util.lang.Xxh3;
|
||||
import org.assertj.core.api.AssertionsForClassTypes;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.MethodSource;
|
||||
import org.junit.jupiter.params.provider.ValueSource;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Random;
|
||||
|
||||
import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat;
|
||||
@@ -60,7 +59,7 @@ public class XxHash3Test {
|
||||
case 10_000 -> -4959357597963000776L;
|
||||
default -> throw new UnsupportedOperationException("Unknown size");
|
||||
};
|
||||
assertThat(Xxh3.hashLongs(data)).isEqualTo(expected);
|
||||
AssertionsForClassTypes.assertThat(Xxh3.hashLongs(data)).isEqualTo(expected);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -100,7 +99,7 @@ public class XxHash3Test {
|
||||
}
|
||||
|
||||
private static void checkPackage(String s, long expected) {
|
||||
assertThat(Xx3UnencodedString.hashUnencodedString(s.replace('.', '/'))).describedAs("Hash as string of: " + s).isEqualTo(expected);
|
||||
AssertionsForClassTypes.assertThat(Xx3UnencodedString.hashUnencodedString(s.replace('.', '/'))).describedAs("Hash as string of: " + s).isEqualTo(expected);
|
||||
}
|
||||
|
||||
private static void testUnencodedString(String s, long expected) {
|
||||
@@ -25,7 +25,6 @@
|
||||
<orderEntry type="module" module-name="intellij.platform.util.xmlDom" scope="PROVIDED" />
|
||||
<orderEntry type="library" name="okhttp" level="project" />
|
||||
<orderEntry type="module" module-name="intellij.idea.community.build.tasks" />
|
||||
<orderEntry type="module" module-name="intellij.platform.util.immutableKeyValueStore" />
|
||||
<orderEntry type="module" module-name="intellij.platform.util.rt.java8" />
|
||||
<orderEntry type="library" name="gson" level="project" />
|
||||
<orderEntry type="library" name="kotlinx-coroutines-core" level="project" />
|
||||
|
||||
@@ -151,7 +151,11 @@ fun reorderJar(jarFile: Path, orderedNames: List<String>) {
|
||||
|
||||
for (entry in entries) {
|
||||
packageIndexBuilder.addFile(entry.name)
|
||||
zipCreator.uncompressedData(entry.name, entry.getByteBuffer(sourceZip))
|
||||
zipCreator.uncompressedData(
|
||||
nameString = entry.name,
|
||||
data = entry.getByteBuffer(sourceZip),
|
||||
indexWriter = packageIndexBuilder.indexWriter,
|
||||
)
|
||||
}
|
||||
packageIndexBuilder.writePackageIndex(zipCreator)
|
||||
}
|
||||
|
||||
@@ -206,11 +206,10 @@ class ModuleItem(
|
||||
}
|
||||
|
||||
override fun equals(other: Any?): Boolean {
|
||||
return this === other ||
|
||||
other is ModuleItem && moduleName == other.moduleName && relativeOutputFile == other.relativeOutputFile
|
||||
return this === other || other is ModuleItem && moduleName == other.moduleName && relativeOutputFile == other.relativeOutputFile
|
||||
}
|
||||
|
||||
override fun hashCode(): Int = 31 * moduleName.hashCode() + relativeOutputFile.hashCode()
|
||||
|
||||
override fun toString(): String = "ModuleItem(moduleName=${moduleName}, relativeOutputFile=${relativeOutputFile}, reason=${reason})"
|
||||
override fun toString(): String = "ModuleItem(moduleName=$moduleName, relativeOutputFile=$relativeOutputFile, reason=$reason)"
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
package org.jetbrains.intellij.build.impl
|
||||
|
||||
import com.intellij.platform.diagnostic.telemetry.helpers.use
|
||||
@@ -10,7 +10,7 @@ import org.jetbrains.intellij.build.CompilationContext
|
||||
import org.jetbrains.intellij.build.TraceManager.spanBuilder
|
||||
import org.jetbrains.intellij.build.io.ZipArchiver
|
||||
import org.jetbrains.intellij.build.io.archiveDir
|
||||
import org.jetbrains.intellij.build.io.writeNewZip
|
||||
import org.jetbrains.intellij.build.io.writeNewZipWithoutIndex
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
|
||||
@@ -96,12 +96,12 @@ private suspend fun buildResourcesForHelpPlugin(resourceRoot: Path, classPath: L
|
||||
jvmArgs = emptyList(),
|
||||
classPath = classPath)
|
||||
}
|
||||
writeNewZip(assetJar, compress = true) { zipCreator ->
|
||||
writeNewZipWithoutIndex(assetJar, compress = true) { zipCreator ->
|
||||
val archiver = ZipArchiver(zipCreator)
|
||||
archiver.setRootDir(resourceRoot)
|
||||
archiveDir(resourceRoot.resolve("topics"), archiver)
|
||||
archiveDir(resourceRoot.resolve("images"), archiver)
|
||||
archiveDir(resourceRoot.resolve("search"), archiver)
|
||||
archiveDir(resourceRoot.resolve("topics"), archiver, null)
|
||||
archiveDir(resourceRoot.resolve("images"), archiver, null)
|
||||
archiveDir(resourceRoot.resolve("search"), archiver, null)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1262,15 +1262,15 @@ private fun archivePlugin(optimized: Boolean,
|
||||
source: Path,
|
||||
context: BuildContext) {
|
||||
if (optimized) {
|
||||
writeNewZip(target, compress = compress, withOptimizedMetadataEnabled = false) { zipCreator ->
|
||||
writeNewZipWithoutIndex(target, compress = compress) { zipCreator ->
|
||||
ZipArchiver(zipCreator).use { archiver ->
|
||||
if (Files.isDirectory(source)) {
|
||||
archiver.setRootDir(source, source.fileName.toString())
|
||||
archiveDir(startDir = source, archiver = archiver, excludes = null)
|
||||
archiveDir(startDir = source, archiver = archiver, excludes = null, indexWriter = null)
|
||||
}
|
||||
else {
|
||||
archiver.setRootDir(source.parent)
|
||||
archiver.addFile(source)
|
||||
archiver.addFile(source, indexWriter = null)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1300,7 +1300,7 @@ private fun buildBlockMap(file: Path, json: JSON) {
|
||||
|
||||
val fileParent = file.parent
|
||||
val fileName = file.fileName.toString()
|
||||
writeNewZip(fileParent.resolve("$fileName.blockmap.zip"), compress = true) {
|
||||
writeNewZipWithoutIndex(fileParent.resolve("$fileName.blockmap.zip"), compress = true) {
|
||||
it.compressedData("blockmap.json", ByteBuffer.wrap(bytes))
|
||||
}
|
||||
|
||||
|
||||
@@ -2,8 +2,9 @@
|
||||
|
||||
package org.jetbrains.intellij.build.impl
|
||||
|
||||
import org.jetbrains.intellij.build.io.PackageIndexBuilder
|
||||
import org.jetbrains.intellij.build.io.ZipFileWriter
|
||||
import org.jetbrains.intellij.build.io.writeNewZip
|
||||
import org.jetbrains.intellij.build.io.writeNewZipWithoutIndex
|
||||
import java.nio.ByteBuffer
|
||||
import java.nio.channels.SeekableByteChannel
|
||||
import java.nio.channels.WritableByteChannel
|
||||
@@ -23,8 +24,11 @@ internal fun buildKeymapPlugin(keymaps: Array<String>, buildNumber: String, targ
|
||||
|
||||
val buffer = ByteBuffer.allocate(128 * 1024)
|
||||
ZipFileWriter(WritableByteChannelBackedByByteBuffer(buffer)).use { zipCreator ->
|
||||
zipCreator.uncompressedData("META-INF/plugin.xml", ByteBuffer.wrap(pluginXmlData))
|
||||
val packageIndexBuilder = PackageIndexBuilder()
|
||||
zipCreator.uncompressedData("META-INF/plugin.xml", ByteBuffer.wrap(pluginXmlData), packageIndexBuilder.indexWriter)
|
||||
packageIndexBuilder.addFile("META-INF/plugin.xml")
|
||||
|
||||
packageIndexBuilder.addFile("META-INF/pluginIcon.svg")
|
||||
@Suppress("SpellCheckingInspection")
|
||||
zipCreator.uncompressedData("META-INF/pluginIcon.svg",
|
||||
"<svg xmlns=\"http://www.w3.org/2000/svg\" width=\"40\" height=\"40\" viewBox=\"0 0 40 40\">\n" +
|
||||
@@ -33,7 +37,9 @@ internal fun buildKeymapPlugin(keymaps: Array<String>, buildNumber: String, targ
|
||||
"15 L28,15 L28,11 Z M22,11 L26,11 L26,15 L22,15 L22,11 Z M10,11 L14,11 L14,15 L10,15 L10,11 Z M4," +
|
||||
"11 L8,11 L8,15 L4,15 L4,11 Z M4,5 L8,5 L8,9 L4,9 L4,5 Z M10,5 L14,5 L14,9 L10,9 L10,5 Z M16,11 L20," +
|
||||
"11 L20,15 L16,15 L16,11 Z M25,21 L11,21 L11,17 L25,17 L25,21 Z\" transform=\"translate(2 7)\"/>\n" +
|
||||
"</svg>\n")
|
||||
"</svg>\n", packageIndexBuilder.indexWriter)
|
||||
|
||||
packageIndexBuilder.addFile("META-INF/pluginIcon_dark.svg")
|
||||
@Suppress("SpellCheckingInspection")
|
||||
zipCreator.uncompressedData("META-INF/pluginIcon_dark.svg",
|
||||
"<svg xmlns=\"http://www.w3.org/2000/svg\" width=\"40\" height=\"40\" viewBox=\"0 0 40 40\">\n" +
|
||||
@@ -42,17 +48,21 @@ internal fun buildKeymapPlugin(keymaps: Array<String>, buildNumber: String, targ
|
||||
"15 L28,15 L28,11 Z M22,11 L26,11 L26,15 L22,15 L22,11 Z M10,11 L14,11 L14,15 L10,15 L10,11 Z M4," +
|
||||
"11 L8,11 L8,15 L4,15 L4,11 Z M4,5 L8,5 L8,9 L4,9 L4,5 Z M10,5 L14,5 L14,9 L10,9 L10,5 Z M16,11 L20," +
|
||||
"11 L20,15 L16,15 L16,11 Z M25,21 L11,21 L11,17 L25,17 L25,21 Z\" transform=\"translate(2 7)\"/>\n" +
|
||||
"</svg>\n")
|
||||
"</svg>\n", packageIndexBuilder.indexWriter)
|
||||
for (name in keymaps) {
|
||||
zipCreator.file("keymaps/$name.xml", keymapDir.resolve("$name.xml"))
|
||||
val keymapFile = "keymaps/$name.xml"
|
||||
packageIndexBuilder.addFile(keymapFile)
|
||||
zipCreator.file(keymapFile, keymapDir.resolve("$name.xml"), packageIndexBuilder.indexWriter)
|
||||
}
|
||||
|
||||
packageIndexBuilder.writePackageIndex(zipCreator)
|
||||
}
|
||||
|
||||
buffer.flip()
|
||||
|
||||
val resultFile = targetDir.resolve("${shortName}Keymap.zip")
|
||||
writeNewZip(resultFile, compress = true) {
|
||||
it.uncompressedData("${shortName}Keymap/lib/${shortName}Keymap.jar", buffer)
|
||||
writeNewZipWithoutIndex(resultFile, compress = true) {
|
||||
it.uncompressedData("${shortName}Keymap/lib/${shortName}Keymap.jar", buffer, null)
|
||||
}
|
||||
return Pair(resultFile, pluginXmlData)
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
@file:Suppress("ReplacePutWithAssignment")
|
||||
package org.jetbrains.intellij.build.impl
|
||||
|
||||
import com.intellij.openapi.util.SystemInfoRt
|
||||
import com.intellij.platform.diagnostic.telemetry.helpers.useWithScope
|
||||
import com.intellij.platform.diagnostic.telemetry.helpers.use
|
||||
import com.intellij.platform.diagnostic.telemetry.helpers.useWithScope
|
||||
import com.jetbrains.signatureverifier.ILogger
|
||||
import com.jetbrains.signatureverifier.InvalidDataException
|
||||
import com.jetbrains.signatureverifier.crypt.SignatureVerificationParams
|
||||
@@ -123,17 +123,17 @@ private fun copyZipReplacing(origin: Path, entries: Map<String, Path>, context:
|
||||
.setAttribute(AttributeKey.stringArrayKey("unsigned"), entries.keys.toList())
|
||||
.use {
|
||||
transformZipUsingTempFile(origin) { zipWriter ->
|
||||
val index = PackageIndexBuilder()
|
||||
val packageIndexBuilder = PackageIndexBuilder()
|
||||
readZipFile(origin) { name, dataSupplier ->
|
||||
index.addFile(name)
|
||||
packageIndexBuilder.addFile(name)
|
||||
if (entries.containsKey(name)) {
|
||||
zipWriter.file(name, entries.getValue(name))
|
||||
zipWriter.file(name, entries.getValue(name), packageIndexBuilder.indexWriter)
|
||||
}
|
||||
else {
|
||||
zipWriter.uncompressedData(name, dataSupplier())
|
||||
zipWriter.uncompressedData(name, dataSupplier(), packageIndexBuilder.indexWriter)
|
||||
}
|
||||
}
|
||||
index.writePackageIndex(zipWriter)
|
||||
packageIndexBuilder.writePackageIndex(zipWriter)
|
||||
}
|
||||
Files.setLastModifiedTime(origin, FileTime.from(context.options.buildDateInSeconds, TimeUnit.SECONDS))
|
||||
}
|
||||
|
||||
@@ -180,7 +180,14 @@ internal suspend fun buildJar(
|
||||
})
|
||||
val normalizedDir = source.dir.toAbsolutePath().normalize()
|
||||
archiver.setRootDir(normalizedDir, source.prefix)
|
||||
archiveDir(startDir = normalizedDir, archiver = archiver, excludes = source.excludes.takeIf(List<PathMatcher>::isNotEmpty))
|
||||
archiveDir(
|
||||
startDir = normalizedDir,
|
||||
archiver = archiver,
|
||||
indexWriter = packageIndexBuilder?.indexWriter,
|
||||
excludes = source.excludes.takeIf(
|
||||
List<PathMatcher>::isNotEmpty,
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
is InMemoryContentSource -> {
|
||||
@@ -192,9 +199,14 @@ internal suspend fun buildJar(
|
||||
}
|
||||
|
||||
packageIndexBuilder?.addFile(source.relativePath)
|
||||
zipCreator.uncompressedData(source.relativePath, source.data.size) {
|
||||
it.put(source.data)
|
||||
}
|
||||
zipCreator.uncompressedData(
|
||||
nameString = source.relativePath,
|
||||
maxSize = source.data.size,
|
||||
indexWriter = packageIndexBuilder?.indexWriter,
|
||||
dataWriter = {
|
||||
it.put(source.data)
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
is ZipSource -> {
|
||||
@@ -276,11 +288,11 @@ private suspend fun handleZipSource(source: ZipSource,
|
||||
zipCreator.compressedData(name, dataSupplier())
|
||||
}
|
||||
else {
|
||||
zipCreator.uncompressedData(name, dataSupplier())
|
||||
zipCreator.uncompressedData(name, dataSupplier(), indexWriter = packageIndexBuilder?.indexWriter)
|
||||
}
|
||||
}
|
||||
else {
|
||||
zipCreator.file(name, file)
|
||||
zipCreator.file(name, file, indexWriter = packageIndexBuilder?.indexWriter)
|
||||
Files.delete(file)
|
||||
}
|
||||
}
|
||||
@@ -293,7 +305,7 @@ private suspend fun handleZipSource(source: ZipSource,
|
||||
zipCreator.compressedData(name, data)
|
||||
}
|
||||
else {
|
||||
zipCreator.uncompressedData(name, data)
|
||||
zipCreator.uncompressedData(name, data, indexWriter = packageIndexBuilder?.indexWriter)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,8 +10,8 @@
|
||||
<orderEntry type="library" scope="TEST" name="jmh-core" level="project" />
|
||||
<orderEntry type="library" scope="TEST" name="jmh-generator-annprocess" level="project" />
|
||||
<orderEntry type="library" scope="TEST" name="kotlin-stdlib" level="project" />
|
||||
<orderEntry type="module" module-name="intellij.platform.util.immutableKeyValueStore" scope="TEST" />
|
||||
<orderEntry type="module" module-name="intellij.platform.util.zip" scope="TEST" />
|
||||
<orderEntry type="module" module-name="intellij.platform.util.rt.java8" scope="TEST" />
|
||||
<orderEntry type="module" module-name="intellij.idea.community.build.tasks" scope="TEST" />
|
||||
</component>
|
||||
</module>
|
||||
@@ -2,7 +2,7 @@
|
||||
package com.intellij.platform.util.immutableKeyValueStore.benchmark
|
||||
|
||||
import com.intellij.util.lang.Xxh3
|
||||
import org.jetbrains.ikv.builder.IkvWriter
|
||||
import org.jetbrains.intellij.build.io.IkvWriter
|
||||
import java.nio.channels.FileChannel
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
|
||||
Reference in New Issue
Block a user