BinaryXml — deduplicate strings

platform/configuration-store-impl/src/StateMap.kt

@@ -37,7 +37,7 @@ fun archiveState(state: Element): BufferExposingByteArrayOutputStream {
   return byteOut
 }
 
-private fun unarchiveState(state: ByteArray) = SnappyFramedInputStream(ByteArrayInputStream(state), false).use { readElement(it) }
+private fun unarchiveState(state: ByteArray) = SnappyFramedInputStream(ByteArrayInputStream(state), false).use { deserializeElementFromBinary(it) }
 
 fun getNewByteIfDiffers(key: String, newState: Any, oldState: ByteArray): ByteArray? {
   val newBytes: ByteArray

platform/configuration-store-impl/testSrc/BinaryXmlOutputterTest.kt

@@ -12,7 +12,59 @@ class BinaryXmlOutputterTest {
   }
 
   @Test fun attributes() {
-    test("""<foo bar="1" />""")
+    test("""
+    <state>
+      <component name="CopyrightManager">
+        <copyright>
+          <option name="myName" value="Foo" />
+          <option name="notice" value="where" />
+        </copyright>
+      </component>
+      <component name="InspectionProjectProfileManager">
+        <profile version="1.0">
+          <option name="myName" value="Project Default" />
+          <inspection_tool class="AntDuplicateTargetsInspection" enabled="false" level="ERROR" enabled_by_default="false" />
+          <inspection_tool class="AntMissingPropertiesFileInspection" enabled="false" level="ERROR" enabled_by_default="false" />
+          <inspection_tool class="AntResolveInspection" enabled="false" level="ERROR" enabled_by_default="false" />
+          <inspection_tool class="ArgNamesErrorsInspection" enabled="false" level="ERROR" enabled_by_default="false" />
+          <inspection_tool class="ArgNamesWarningsInspection" enabled="false" level="WARNING" enabled_by_default="false" />
+          <inspection_tool class="AroundAdviceStyleInspection" enabled="false" level="WARNING" enabled_by_default="false" />
+          <inspection_tool class="DeclareParentsInspection" enabled="false" level="ERROR" enabled_by_default="false" />
+          <inspection_tool class="EmptyEventHandler" enabled="false" level="WARNING" enabled_by_default="false" />
+          <inspection_tool class="PointcutMethodStyleInspection" enabled="false" level="WARNING" enabled_by_default="false" />
+        </profile>
+        <version value="1.0" />
+      </component>
+      <component name="ProjectLevelVcsManager" settingsEditedManually="false" />
+      <component name="masterDetails">
+        <states>
+          <state key="Copyright.UI">
+            <settings>
+              <last-edited>Foo</last-edited>
+              <splitter-proportions>
+                <option name="proportions">
+                  <list>
+                    <option value="0.2" />
+                  </list>
+                </option>
+              </splitter-proportions>
+            </settings>
+          </state>
+          <state key="ProjectJDKs.UI">
+            <settings>
+              <last-edited>1.4</last-edited>
+              <splitter-proportions>
+                <option name="proportions">
+                  <list>
+                    <option value="0.2" />
+                  </list>
+                </option>
+              </splitter-proportions>
+            </settings>
+          </state>
+        </states>
+      </component>
+    </state>""")
   }
 
   private fun test(xml: String) {
@@ -21,7 +73,7 @@ class BinaryXmlOutputterTest {
       serializeElementToBinary(loadElement(xml), it)
     }
 
-    val xmlAfter = JDOMUtil.writeElement(byteOut.toByteArray().inputStream().use { readElement(it) })
+    val xmlAfter = JDOMUtil.writeElement(byteOut.toByteArray().inputStream().use { deserializeElementFromBinary(it) })
 
     assertThat(xml.trimIndent()).isEqualTo(xmlAfter)
   }

platform/external-system-impl/src/com/intellij/openapi/externalSystem/configurationStore/ExternalSystemStorage.kt

@@ -28,7 +28,7 @@ internal class ExternalProjectStorage(private val module: Module, storageManager
 
   override public fun loadLocalData(): Element? {
     val data = manager.nameToData.get(module.name) ?: return null
-    return ByteArrayInputStream(data).use { readElement(it) }
+    return ByteArrayInputStream(data).use { deserializeElementFromBinary(it) }
   }
 
   override fun createSaveSession(states: StateMap) = object : XmlElementStorageSaveSession<ExternalProjectStorage>(states, this) {

platform/projectModel-impl/src/com/intellij/configurationStore/BinaryXmlOutputter.kt

@@ -15,97 +15,91 @@
  */
 package com.intellij.configurationStore
 
+import com.intellij.util.io.DataOutputStream
 import com.intellij.util.io.IOUtil
-import org.jdom.*
+import org.jdom.Attribute
+import org.jdom.CDATA
+import org.jdom.Element
+import org.jdom.Text
 import java.io.DataInputStream
-import java.io.DataOutputStream
 import java.io.InputStream
 import java.io.OutputStream
+import java.util.*
 
-private enum class TypeMarker {
+internal enum class TypeMarker {
   ELEMENT, CDATA, TEXT, ELEMENT_END
 }
 
 fun serializeElementToBinary(element: Element, out: OutputStream) {
-  writeElement(element, DataOutputStream(out))
+  BinaryXmlWriter(DataOutputStream(out)).write(element)
 }
 
-fun readElement(input: InputStream) = readElement(DataInputStream(input))
+fun deserializeElementFromBinary(input: InputStream) = BinaryXmlReader(DataInputStream(input)).read()
 
-fun readElement(input: DataInputStream): Element {
-  val element = Element(IOUtil.readUTF(input))
-  readAttributes(element, input)
-  readContent(element, input)
-  return element
-}
+private class BinaryXmlReader(private val input: DataInputStream) {
+  private val strings = ArrayList<String>()
 
-private fun readContent(element: Element, input: DataInputStream) {
-  while (true) {
-    when (input.read()) {
-      TypeMarker.ELEMENT.ordinal -> element.addContent(readElement(input))
-      TypeMarker.TEXT.ordinal -> element.addContent(Text(IOUtil.readUTF(input)))
-      TypeMarker.CDATA.ordinal -> element.addContent(CDATA(IOUtil.readUTF(input)))
-      TypeMarker.ELEMENT_END.ordinal -> return
+  fun read() = readElement()
+
+  private fun readString(): String {
+    val lengthOrIndex = readUInt29()
+    if (lengthOrIndex == 1) {
+      return ""
     }
+
+    if ((lengthOrIndex and 1) == 1) {
+      val string = IOUtil.readUTF(input)
+      strings.add(string)
+      return string
+    }
+
+    return strings.get(lengthOrIndex shr 1)
   }
-}
 
-private fun writeElement(element: Element, out: DataOutputStream) {
-  IOUtil.writeUTF(out, element.name)
+  private fun readElement(): Element {
+    val element = Element(readString())
+    readAttributes(element)
+    readContent(element)
+    return element
+  }
 
-  writeAttributes(out, element.attributes)
-
-  val content = element.content
-  for (item in content) {
-    if (item is Element) {
-      out.writeByte(TypeMarker.ELEMENT.ordinal)
-      writeElement(item, out)
-    }
-    else if (item is Text) {
-      if (!isAllWhitespace(item)) {
-        out.writeByte(TypeMarker.TEXT.ordinal)
-        IOUtil.writeUTF(out, item.text)
+  private fun readContent(element: Element) {
+    while (true) {
+      when (input.read()) {
+        TypeMarker.ELEMENT.ordinal -> element.addContent(readElement())
+        TypeMarker.TEXT.ordinal -> element.addContent(Text(readString()))
+        TypeMarker.CDATA.ordinal -> element.addContent(CDATA(readString()))
+        TypeMarker.ELEMENT_END.ordinal -> return
       }
     }
-    else if (item is CDATA) {
-      out.writeByte(TypeMarker.CDATA.ordinal)
-      IOUtil.writeUTF(out, item.text)
+  }
+
+  private fun readAttributes(element: Element) {
+    val size = input.readUnsignedByte()
+    for (i in 0..size - 1) {
+      element.setAttribute(Attribute(readString(), readString()))
     }
   }
-  out.writeByte(TypeMarker.ELEMENT_END.ordinal)
-}
 
-private fun writeAttributes(out: DataOutputStream, attributes: List<Attribute>?) {
-  val size = attributes?.size ?: 0
-  out.write(size)
-  if (size == 0) {
-    return
-  }
-
-  if (size > 255) {
-    throw UnsupportedOperationException("attributes size > 255")
-  }
-  else {
-    for (attribute in attributes!!) {
-      IOUtil.writeUTF(out, attribute.name)
-      IOUtil.writeUTF(out, attribute.value)
+  private fun readUInt29(): Int {
+    var value: Int
+    var b = input.read()
+    if ((b and 0xFF) < 128) {
+      return b
     }
-  }
-}
 
-private fun readAttributes(element: Element, input: DataInputStream) {
-  val size = input.readUnsignedByte()
-  for (i in 0..size - 1) {
-    element.setAttribute(Attribute(IOUtil.readUTF(input), IOUtil.readUTF(input)))
-  }
-}
-
-private fun isAllWhitespace(obj: Content): Boolean {
-  val str = (obj as? Text)?.text ?: return false
-  for (i in 0..str.length - 1) {
-    if (!Verifier.isXMLWhitespace(str[i])) {
-      return false
+    value = b and 0x7F shl 7
+    b = input.read()
+    if ((b and 0xFF) < 128) {
+      return value or b
     }
+
+    value = value or (b and 0x7F) shl 7
+    b = input.read()
+    if ((b and 0xFF) < 128) {
+      return value or b
+    }
+
+    return value or (b and 0x7F) shl 8 or (input.read() and 0xFF)
   }
-  return true
 }

platform/projectModel-impl/src/com/intellij/configurationStore/BinaryXmlWriter.kt

@@ -0,0 +1,126 @@
+/*
+ * Copyright 2000-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.configurationStore
+
+import com.intellij.util.containers.ObjectIntHashMap
+import com.intellij.util.io.IOUtil
+import org.jdom.*
+import java.io.DataOutputStream
+
+internal class BinaryXmlWriter(private val out: DataOutputStream) {
+  private val strings = ObjectIntHashMap<String>()
+
+  fun write(element: Element) {
+    writeElement(element)
+  }
+
+  private fun writeString(string: String) {
+    if (string.isEmpty()) {
+      out.write(1)
+      return
+    }
+
+    val reference = strings.get(string)
+    if (reference != -1) {
+      writeUInt29(reference shl 1)
+      return
+    }
+
+    strings.put(string, strings.size())
+    // don't write actual length, IOUtil does it
+    out.write((1 shl 1) or 1)
+    IOUtil.writeUTF(out, string)
+  }
+
+  private fun writeElement(element: Element) {
+    writeString(element.name)
+
+    writeAttributes(element.attributes)
+
+    val content = element.content
+    for (item in content) {
+      if (item is Element) {
+        out.writeByte(TypeMarker.ELEMENT.ordinal)
+        writeElement(item)
+      }
+      else if (item is Text) {
+        if (!isAllWhitespace(item)) {
+          out.writeByte(TypeMarker.TEXT.ordinal)
+          writeString(item.text)
+        }
+      }
+      else if (item is CDATA) {
+        out.writeByte(TypeMarker.CDATA.ordinal)
+        writeString(item.text)
+      }
+    }
+    out.writeByte(TypeMarker.ELEMENT_END.ordinal)
+  }
+
+  private fun writeAttributes(attributes: List<Attribute>?) {
+    val size = attributes?.size ?: 0
+    out.write(size)
+    if (size == 0) {
+      return
+    }
+
+    if (size > 255) {
+      throw UnsupportedOperationException("attributes size > 255")
+    }
+    else {
+      for (attribute in attributes!!) {
+        writeString(attribute.name)
+        writeString(attribute.value)
+      }
+    }
+  }
+
+  // Represent smaller integers with fewer bytes using the most significant bit of each byte. The worst case uses 32-bits
+  // to represent a 29-bit number, which is what we would have done with no compression.
+  private fun writeUInt29(v: Int) {
+    if (v < 0x80) {
+      out.write(v)
+    }
+    else if (v < 0x4000) {
+      out.write((v shr 7 and 0x7F or 0x80))
+      out.write(v and 0x7F)
+    }
+    else if (v < 0x200000) {
+      out.write((v shr 14 and 0x7F or 0x80))
+      out.write(v shr 7 and 0x7F or 0x80)
+      out.write(v and 0x7F)
+    }
+    else if (v < 0x40000000) {
+      out.write(v shr 22 and 0x7F or 0x80)
+      out.write (v shr 15 and 0x7F or 0x80)
+      out.write(v shr 8 and 0x7F or 0x80)
+      out.write(v and 0xFF)
+    }
+    else {
+      throw IllegalArgumentException("Integer out of range: $v")
+    }
+  }
+
+  private fun isAllWhitespace(obj: Content): Boolean {
+    val str = (obj as? Text)?.text ?: return false
+    for (i in 0..str.length - 1) {
+      if (!Verifier.isXMLWhitespace(str[i])) {
+        return false
+      }
+    }
+    return true
+  }
+}