From c183daca7d4da68ef8fe260e1a32cbfd79e4d740 Mon Sep 17 00:00:00 2001 From: Vladimir Krivosheev Date: Fri, 28 Apr 2017 17:13:33 +0200 Subject: [PATCH] =?UTF-8?q?BinaryXml=20=E2=80=94=20deduplicate=20strings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../configuration-store-impl/src/StateMap.kt | 2 +- .../testSrc/BinaryXmlOutputterTest.kt | 56 +++++++- .../ExternalSystemStorage.kt | 2 +- .../configurationStore/BinaryXmlOutputter.kt | 128 +++++++++--------- .../configurationStore/BinaryXmlWriter.kt | 126 +++++++++++++++++ 5 files changed, 243 insertions(+), 71 deletions(-) create mode 100644 platform/projectModel-impl/src/com/intellij/configurationStore/BinaryXmlWriter.kt diff --git a/platform/configuration-store-impl/src/StateMap.kt b/platform/configuration-store-impl/src/StateMap.kt index f441e240c69a..0c1c4d867090 100644 --- a/platform/configuration-store-impl/src/StateMap.kt +++ b/platform/configuration-store-impl/src/StateMap.kt @@ -37,7 +37,7 @@ fun archiveState(state: Element): BufferExposingByteArrayOutputStream { return byteOut } -private fun unarchiveState(state: ByteArray) = SnappyFramedInputStream(ByteArrayInputStream(state), false).use { readElement(it) } +private fun unarchiveState(state: ByteArray) = SnappyFramedInputStream(ByteArrayInputStream(state), false).use { deserializeElementFromBinary(it) } fun getNewByteIfDiffers(key: String, newState: Any, oldState: ByteArray): ByteArray? { val newBytes: ByteArray diff --git a/platform/configuration-store-impl/testSrc/BinaryXmlOutputterTest.kt b/platform/configuration-store-impl/testSrc/BinaryXmlOutputterTest.kt index d15acabe782d..6d1298aa7902 100644 --- a/platform/configuration-store-impl/testSrc/BinaryXmlOutputterTest.kt +++ b/platform/configuration-store-impl/testSrc/BinaryXmlOutputterTest.kt @@ -12,7 +12,59 @@ class BinaryXmlOutputterTest { } @Test fun attributes() { - test("""""") + test(""" + + + + + + + + + + + + + + + + Foo + + + + + + + + 1.4 + + + + + + + + """) } private fun test(xml: String) { @@ -21,7 +73,7 @@ class BinaryXmlOutputterTest { serializeElementToBinary(loadElement(xml), it) } - val xmlAfter = JDOMUtil.writeElement(byteOut.toByteArray().inputStream().use { readElement(it) }) + val xmlAfter = JDOMUtil.writeElement(byteOut.toByteArray().inputStream().use { deserializeElementFromBinary(it) }) assertThat(xml.trimIndent()).isEqualTo(xmlAfter) } diff --git a/platform/external-system-impl/src/com/intellij/openapi/externalSystem/configurationStore/ExternalSystemStorage.kt b/platform/external-system-impl/src/com/intellij/openapi/externalSystem/configurationStore/ExternalSystemStorage.kt index 4d4b9e3aecea..98115bbbe621 100644 --- a/platform/external-system-impl/src/com/intellij/openapi/externalSystem/configurationStore/ExternalSystemStorage.kt +++ b/platform/external-system-impl/src/com/intellij/openapi/externalSystem/configurationStore/ExternalSystemStorage.kt @@ -28,7 +28,7 @@ internal class ExternalProjectStorage(private val module: Module, storageManager override public fun loadLocalData(): Element? { val data = manager.nameToData.get(module.name) ?: return null - return ByteArrayInputStream(data).use { readElement(it) } + return ByteArrayInputStream(data).use { deserializeElementFromBinary(it) } } override fun createSaveSession(states: StateMap) = object : XmlElementStorageSaveSession(states, this) { diff --git a/platform/projectModel-impl/src/com/intellij/configurationStore/BinaryXmlOutputter.kt b/platform/projectModel-impl/src/com/intellij/configurationStore/BinaryXmlOutputter.kt index bb90ebcfd677..95ddc67ddd55 100644 --- a/platform/projectModel-impl/src/com/intellij/configurationStore/BinaryXmlOutputter.kt +++ b/platform/projectModel-impl/src/com/intellij/configurationStore/BinaryXmlOutputter.kt @@ -15,97 +15,91 @@ */ package com.intellij.configurationStore +import com.intellij.util.io.DataOutputStream import com.intellij.util.io.IOUtil -import org.jdom.* +import org.jdom.Attribute +import org.jdom.CDATA +import org.jdom.Element +import org.jdom.Text import java.io.DataInputStream -import java.io.DataOutputStream import java.io.InputStream import java.io.OutputStream +import java.util.* -private enum class TypeMarker { +internal enum class TypeMarker { ELEMENT, CDATA, TEXT, ELEMENT_END } fun serializeElementToBinary(element: Element, out: OutputStream) { - writeElement(element, DataOutputStream(out)) + BinaryXmlWriter(DataOutputStream(out)).write(element) } -fun readElement(input: InputStream) = readElement(DataInputStream(input)) +fun deserializeElementFromBinary(input: InputStream) = BinaryXmlReader(DataInputStream(input)).read() -fun readElement(input: DataInputStream): Element { - val element = Element(IOUtil.readUTF(input)) - readAttributes(element, input) - readContent(element, input) - return element -} +private class BinaryXmlReader(private val input: DataInputStream) { + private val strings = ArrayList() -private fun readContent(element: Element, input: DataInputStream) { - while (true) { - when (input.read()) { - TypeMarker.ELEMENT.ordinal -> element.addContent(readElement(input)) - TypeMarker.TEXT.ordinal -> element.addContent(Text(IOUtil.readUTF(input))) - TypeMarker.CDATA.ordinal -> element.addContent(CDATA(IOUtil.readUTF(input))) - TypeMarker.ELEMENT_END.ordinal -> return + fun read() = readElement() + + private fun readString(): String { + val lengthOrIndex = readUInt29() + if (lengthOrIndex == 1) { + return "" } + + if ((lengthOrIndex and 1) == 1) { + val string = IOUtil.readUTF(input) + strings.add(string) + return string + } + + return strings.get(lengthOrIndex shr 1) } -} -private fun writeElement(element: Element, out: DataOutputStream) { - IOUtil.writeUTF(out, element.name) + private fun readElement(): Element { + val element = Element(readString()) + readAttributes(element) + readContent(element) + return element + } - writeAttributes(out, element.attributes) - - val content = element.content - for (item in content) { - if (item is Element) { - out.writeByte(TypeMarker.ELEMENT.ordinal) - writeElement(item, out) - } - else if (item is Text) { - if (!isAllWhitespace(item)) { - out.writeByte(TypeMarker.TEXT.ordinal) - IOUtil.writeUTF(out, item.text) + private fun readContent(element: Element) { + while (true) { + when (input.read()) { + TypeMarker.ELEMENT.ordinal -> element.addContent(readElement()) + TypeMarker.TEXT.ordinal -> element.addContent(Text(readString())) + TypeMarker.CDATA.ordinal -> element.addContent(CDATA(readString())) + TypeMarker.ELEMENT_END.ordinal -> return } } - else if (item is CDATA) { - out.writeByte(TypeMarker.CDATA.ordinal) - IOUtil.writeUTF(out, item.text) + } + + private fun readAttributes(element: Element) { + val size = input.readUnsignedByte() + for (i in 0..size - 1) { + element.setAttribute(Attribute(readString(), readString())) } } - out.writeByte(TypeMarker.ELEMENT_END.ordinal) -} -private fun writeAttributes(out: DataOutputStream, attributes: List?) { - val size = attributes?.size ?: 0 - out.write(size) - if (size == 0) { - return - } - - if (size > 255) { - throw UnsupportedOperationException("attributes size > 255") - } - else { - for (attribute in attributes!!) { - IOUtil.writeUTF(out, attribute.name) - IOUtil.writeUTF(out, attribute.value) + private fun readUInt29(): Int { + var value: Int + var b = input.read() + if ((b and 0xFF) < 128) { + return b } - } -} -private fun readAttributes(element: Element, input: DataInputStream) { - val size = input.readUnsignedByte() - for (i in 0..size - 1) { - element.setAttribute(Attribute(IOUtil.readUTF(input), IOUtil.readUTF(input))) - } -} - -private fun isAllWhitespace(obj: Content): Boolean { - val str = (obj as? Text)?.text ?: return false - for (i in 0..str.length - 1) { - if (!Verifier.isXMLWhitespace(str[i])) { - return false + value = b and 0x7F shl 7 + b = input.read() + if ((b and 0xFF) < 128) { + return value or b } + + value = value or (b and 0x7F) shl 7 + b = input.read() + if ((b and 0xFF) < 128) { + return value or b + } + + return value or (b and 0x7F) shl 8 or (input.read() and 0xFF) } - return true } \ No newline at end of file diff --git a/platform/projectModel-impl/src/com/intellij/configurationStore/BinaryXmlWriter.kt b/platform/projectModel-impl/src/com/intellij/configurationStore/BinaryXmlWriter.kt new file mode 100644 index 000000000000..9c8ec4016c10 --- /dev/null +++ b/platform/projectModel-impl/src/com/intellij/configurationStore/BinaryXmlWriter.kt @@ -0,0 +1,126 @@ +/* + * Copyright 2000-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.configurationStore + +import com.intellij.util.containers.ObjectIntHashMap +import com.intellij.util.io.IOUtil +import org.jdom.* +import java.io.DataOutputStream + +internal class BinaryXmlWriter(private val out: DataOutputStream) { + private val strings = ObjectIntHashMap() + + fun write(element: Element) { + writeElement(element) + } + + private fun writeString(string: String) { + if (string.isEmpty()) { + out.write(1) + return + } + + val reference = strings.get(string) + if (reference != -1) { + writeUInt29(reference shl 1) + return + } + + strings.put(string, strings.size()) + // don't write actual length, IOUtil does it + out.write((1 shl 1) or 1) + IOUtil.writeUTF(out, string) + } + + private fun writeElement(element: Element) { + writeString(element.name) + + writeAttributes(element.attributes) + + val content = element.content + for (item in content) { + if (item is Element) { + out.writeByte(TypeMarker.ELEMENT.ordinal) + writeElement(item) + } + else if (item is Text) { + if (!isAllWhitespace(item)) { + out.writeByte(TypeMarker.TEXT.ordinal) + writeString(item.text) + } + } + else if (item is CDATA) { + out.writeByte(TypeMarker.CDATA.ordinal) + writeString(item.text) + } + } + out.writeByte(TypeMarker.ELEMENT_END.ordinal) + } + + private fun writeAttributes(attributes: List?) { + val size = attributes?.size ?: 0 + out.write(size) + if (size == 0) { + return + } + + if (size > 255) { + throw UnsupportedOperationException("attributes size > 255") + } + else { + for (attribute in attributes!!) { + writeString(attribute.name) + writeString(attribute.value) + } + } + } + + // Represent smaller integers with fewer bytes using the most significant bit of each byte. The worst case uses 32-bits + // to represent a 29-bit number, which is what we would have done with no compression. + private fun writeUInt29(v: Int) { + if (v < 0x80) { + out.write(v) + } + else if (v < 0x4000) { + out.write((v shr 7 and 0x7F or 0x80)) + out.write(v and 0x7F) + } + else if (v < 0x200000) { + out.write((v shr 14 and 0x7F or 0x80)) + out.write(v shr 7 and 0x7F or 0x80) + out.write(v and 0x7F) + } + else if (v < 0x40000000) { + out.write(v shr 22 and 0x7F or 0x80) + out.write (v shr 15 and 0x7F or 0x80) + out.write(v shr 8 and 0x7F or 0x80) + out.write(v and 0xFF) + } + else { + throw IllegalArgumentException("Integer out of range: $v") + } + } + + private fun isAllWhitespace(obj: Content): Boolean { + val str = (obj as? Text)?.text ?: return false + for (i in 0..str.length - 1) { + if (!Verifier.isXMLWhitespace(str[i])) { + return false + } + } + return true + } +} \ No newline at end of file