use a hash4j library as it provides well-tested and proven seeded hash implementations

GitOrigin-RevId: c43ed1e1037a317b8bcbde33c49f26197748ee02
This commit is contained in:
Vladimir Krivosheev
2023-03-24 15:49:55 +00:00
committed by intellij-monorepo-bot
parent 5d02826f3c
commit 6800ed28dc
13 changed files with 106 additions and 151 deletions
+12
View File
@@ -0,0 +1,12 @@
<component name="libraryTable">
<library name="hash4j" type="repository">
<properties maven-id="com.dynatrace.hash4j:hash4j:0.8.0" />
<CLASSES>
<root url="jar://$MAVEN_REPOSITORY$/com/dynatrace/hash4j/hash4j/0.8.0/hash4j-0.8.0.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES>
<root url="jar://$MAVEN_REPOSITORY$/com/dynatrace/hash4j/hash4j/0.8.0/hash4j-0.8.0-sources.jar!/" />
</SOURCES>
</library>
</component>
@@ -134,6 +134,7 @@ object CommunityLibraryLicenses {
LibraryLicense(name = "Apache Commons Discovery", libraryName = "commons-discovery",
url = "https://commons.apache.org/dormant/commons-discovery/")
.apache("https://commons.apache.org/dormant/commons-discovery/license.html"),
LibraryLicense(name = "Apache Commons HTTPClient", libraryName = "http-client-3.1", version = "3.1&nbsp; (with patch by JetBrains)",
url = "https://hc.apache.org/httpclient-3.x").apache(),
LibraryLicense(name = "Apache Commons Imaging (JetBrains's fork)", libraryName = "commons-imaging",
@@ -351,6 +352,10 @@ object CommunityLibraryLicenses {
.apache("https://raw.githubusercontent.com/google/guava/master/LICENSE"),
LibraryLicense(name = "Hamcrest", libraryName = "hamcrest", url = "https://hamcrest.org/")
.newBsd("https://github.com/hamcrest/JavaHamcrest/blob/master/LICENSE.txt"),
LibraryLicense(libraryName = "hash4j", url = "https://github.com/dynatrace-oss/hash4j")
.apache("https://github.com/dynatrace-oss/hash4j/blob/main/LICENSE"),
LibraryLicense(name = "HDR Histogram", libraryName = "HdrHistogram", license = "CC0 1.0 Universal",
url = "https://github.com/HdrHistogram/HdrHistogram",
licenseUrl = "https://github.com/HdrHistogram/HdrHistogram/blob/master/LICENSE.txt"),
@@ -143,5 +143,6 @@
<orderEntry type="module" module-name="intellij.platform.codeStyle.impl" />
<orderEntry type="module" module-name="intellij.platform.util.text.matching" />
<orderEntry type="library" name="kotlinx-collections-immutable" level="project" />
<orderEntry type="library" name="hash4j" level="project" />
</component>
</module>
@@ -229,23 +229,24 @@ public final class UITheme {
initializeNamedColors(theme);
PaletteScopeManager paletteScopeManager = new PaletteScopeManager();
if (theme.iconColorsOnSelection != null && !theme.iconColorsOnSelection.isEmpty()) {
Map<String, String> colors = new TreeMap<>();
for (Map.Entry<String, Object> entry : theme.iconColorsOnSelection.entrySet()) {
colors.put(entry.getKey(), entry.getValue().toString());
Map<String, Object> colorsOnSelection = theme.iconColorsOnSelection;
if (colorsOnSelection != null && !colorsOnSelection.isEmpty()) {
Map<String, String> colors = new HashMap<>(colorsOnSelection.size());
Set<String> alphaColors = new HashSet<>(colorsOnSelection.size());
for (Map.Entry<String, Object> entry : colorsOnSelection.entrySet()) {
String value = entry.getValue().toString();
colors.put(entry.getKey(), value);
alphaColors.add(value);
}
Map<String, Integer> alpha = new TreeMap<>();
colors.forEach((key, value) -> alpha.put(value, 255));
theme.selectionColorPatcher = new SVGLoader.SvgElementColorPatcherProvider() {
@Override
public @Nullable SvgAttributePatcher attributeForPath(@Nullable String path) {
PaletteScope scope = paletteScopeManager.getScopeByPath(path);
InsecureHashBuilder hash = new InsecureHashBuilder()
.stringMap(colors)
.stringIntMap(alpha)
.update(scope == null ? ArrayUtilRt.EMPTY_LONG_ARRAY : scope.digest());
return SvgKt.newSvgPatcher(hash.build(), colors, alpha);
return SvgKt.newSvgPatcher(hash.build(), colors, color -> alphaColors.contains(color) ? 255 : null);
}
};
}
@@ -318,11 +319,10 @@ public final class UITheme {
}
theme.colorPatcher = new SVGLoader.SvgElementColorPatcherProvider() {
@Nullable
@Override
public SvgAttributePatcher attributeForPath(@Nullable String path) {
public @Nullable SvgAttributePatcher attributeForPath(@Nullable String path) {
PaletteScope scope = paletteScopeManager.getScopeByPath(path);
return scope == null ? null : SvgKt.newSvgPatcher(scope.digest(), scope.newPalette, scope.alphas);
return scope == null ? null : SvgKt.newSvgPatcher(scope.digest(), scope.newPalette, scope.alphas::get);
}
};
}
@@ -9,21 +9,21 @@ class InsecureHashBuilderTest {
fun `empty map`() {
val builder = InsecureHashBuilder()
builder.stringMap(mapOf())
assertThat(builder.build()).isEqualTo(longArrayOf(5238470482016868669L))
assertThat(builder.build()).isEqualTo(longArrayOf(204526195655617521L))
}
@Test
fun `string map`() {
val builder = InsecureHashBuilder()
builder.stringMap(mapOf("foo" to "bar"))
assertThat(builder.build()).isEqualTo(longArrayOf(-2665511616818817272L, -4484417689633454546L, -3459221722170984053L))
assertThat(builder.build()).isEqualTo(longArrayOf(5110489462189080232L))
}
@Test
fun `string int map`() {
val builder = InsecureHashBuilder()
builder.stringIntMap(mapOf("foo" to 123, "bar" to 543))
assertThat(builder.build()).isEqualTo(longArrayOf(8139571457004014537L, 3354812099852591849L, 3303764292165953682L))
assertThat(builder.build()).isEqualTo(longArrayOf(8624971787684361546L))
}
@Test
@@ -31,6 +31,6 @@ class InsecureHashBuilderTest {
val builder = InsecureHashBuilder()
builder.stringMap(mapOf("foo" to "bar"))
builder.stringIntMap(mapOf("foo" to 123, "bar" to 543))
assertThat(builder.build()).isEqualTo(longArrayOf(-2665511616818817272L, -4484417689633454546L, -3459221722170984053L, 8139571457004014537L, 3354812099852591849L, 3303764292165953682L))
assertThat(builder.build()).isEqualTo(longArrayOf(5110489462189080232L, 8624971787684361546L))
}
}
@@ -25,5 +25,6 @@
<orderEntry type="library" name="jsvg" level="project" />
<orderEntry type="module" module-name="intellij.platform.util.rt.java8" />
<orderEntry type="module" module-name="intellij.platform.sqlite" />
<orderEntry type="library" name="hash4j" level="project" />
</component>
</module>
@@ -24,7 +24,6 @@ import com.intellij.util.ui.*
import org.jetbrains.annotations.ApiStatus.Internal
import org.jetbrains.annotations.NonNls
import org.jetbrains.annotations.TestOnly
import org.jetbrains.xxh3.Xxh3
import java.awt.*
import java.awt.image.BufferedImage
import java.awt.image.ImageFilter
@@ -295,7 +294,9 @@ object IconLoader {
}
@Internal
fun patchColorsInCacheImageIcon(imageIcon: com.intellij.ui.icons.CachedImageIcon, colorPatcher: SvgElementColorPatcherProvider, isDark: Boolean?): Icon {
fun patchColorsInCacheImageIcon(imageIcon: com.intellij.ui.icons.CachedImageIcon,
colorPatcher: SvgElementColorPatcherProvider,
isDark: Boolean?): Icon {
var result = imageIcon
if (isDark != null) {
val variant = result.getDarkIcon(isDark)
@@ -309,7 +310,7 @@ object IconLoader {
@Suppress("DEPRECATION")
val bytes = colorPatcher.wholeDigest()
if (bytes != null) {
digest = longArrayOf(Xxh3.hash(bytes), Xxh3.seededHash(bytes, 5238470482016868669L))
digest = longArrayOf(hasher.hashBytesToLong(bytes), seededHasher.hashBytesToLong(bytes))
}
}
@@ -0,0 +1,8 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.ui
import com.dynatrace.hash4j.hashing.Hasher64
import com.dynatrace.hash4j.hashing.Hashing
internal val hasher: Hasher64 = Hashing.komihash4_3()
internal val seededHasher: Hasher64 = Hashing.komihash4_3(4812324275)
@@ -7,8 +7,10 @@ import com.github.benmanes.caffeine.cache.Weigher
import com.intellij.diagnostic.StartUpMeasurer
import com.intellij.openapi.diagnostic.Logger
import com.intellij.openapi.diagnostic.logger
import com.intellij.ui.hasher
import com.intellij.ui.scale.DerivedScaleType
import com.intellij.ui.scale.ScaleContext
import com.intellij.ui.seededHasher
import com.intellij.ui.svg.SvgAttributePatcher
import com.intellij.ui.svg.loadSvgAndCacheIfApplicable
import com.intellij.util.ArrayUtilRt
@@ -18,7 +20,6 @@ import com.intellij.util.io.URLUtil
import com.intellij.util.ui.StartupUiUtil
import org.jetbrains.annotations.ApiStatus
import org.jetbrains.annotations.TestOnly
import org.jetbrains.xxh3.Xxh3
import java.awt.Image
import java.awt.image.BufferedImage
import java.awt.image.ImageFilter
@@ -216,7 +217,7 @@ private fun loadByDescriptor(rawPathWithoutExt: String,
@Suppress("DEPRECATION")
deprecatedColorPatcher = colorPatcherProvider.forPath(path)
if (deprecatedColorPatcher != null) {
digest = deprecatedColorPatcher.digest()?.let { longArrayOf(Xxh3.hash(it), Xxh3.seededHash(it, 4849512324275L)) }
digest = deprecatedColorPatcher.digest()?.let { longArrayOf(hasher.hashBytesToLong(it), seededHasher.hashBytesToLong(it)) }
if (digest == null) {
tmpPatcher = true
}
@@ -6,12 +6,13 @@ package com.intellij.ui.svg
import com.intellij.diagnostic.StartUpMeasurer
import com.intellij.openapi.application.PathManager
import com.intellij.openapi.diagnostic.logger
import com.intellij.ui.hasher
import com.intellij.ui.icons.IconLoadMeasurer
import com.intellij.ui.seededHasher
import com.intellij.util.SVGLoader
import org.intellij.lang.annotations.Language
import org.jetbrains.annotations.ApiStatus
import org.jetbrains.sqlite.*
import org.jetbrains.xxh3.Xxh3
import sun.awt.image.SunWritableRaster
import java.awt.Point
import java.awt.Transparency
@@ -33,7 +34,7 @@ value class SvgCacheClassifier(internal val key: Int) {
constructor(scale: Float, size: Int) : this((scale + (10_000 + size)).toBits())
}
fun getSvgIconCacheFile(): Path = Path.of(PathManager.getSystemPath(), "icon-v8.db")
fun getSvgIconCacheFile(): Path = Path.of(PathManager.getSystemPath(), "icon-v9.db")
fun getSvgIconCacheInvalidMarkerFile(file: Path): Path = file.parent.resolve("${file.fileName}.invalidated")
@@ -128,8 +129,8 @@ class SvgCacheManager(dbFile: Path) {
val start = StartUpMeasurer.getCurrentTimeIfEnabled()
return selectStatementPool.use { statement, binder ->
val kind = compoundKey.key.toLong()
binder.bind(v1 = Xxh3.hash(imageBytes), v2 = Xxh3.seededHash(imageBytes, SEED), v3 = kind, v4 = themeKey)
binder.bind(v1 = hasher.hashBytesToLong(imageBytes), v2 = seededHasher.hashBytesToLong(imageBytes), v3 = kind, v4 = themeKey)
val result = readImage(statement)
IconLoadMeasurer.svgCacheRead.end(start)
result
@@ -143,8 +144,8 @@ class SvgCacheManager(dbFile: Path) {
image: BufferedImage) {
val data = writeImage(image)
if (precomputedCacheKey == 0) {
val key1 = Xxh3.hash(imageBytes)
val key2 = Xxh3.seededHash(imageBytes, SEED)
val key1 = hasher.hashBytesToLong(imageBytes)
val key2 = seededHasher.hashBytesToLong(imageBytes)
insertStatementPool.use { statement, binder ->
binder.bind(v1 = key1,
v2 = key2,
@@ -170,7 +171,7 @@ internal fun themeDigestToCacheKey(themeDigest: LongArray): Long {
return when (themeDigest.size) {
0 -> 0
1 -> themeDigest.first()
else -> Xxh3.hashLongs(themeDigest)
else -> hasher.hashStream().putLongArray(themeDigest).asLong
}
}
@@ -187,8 +188,6 @@ private val colorModel = ComponentColorModel(
private val ZERO_POINT = Point(0, 0)
private const val SEED = 4812324275L
private fun writeImage(image: BufferedImage): ByteArray {
val w = image.width
val h = image.height
@@ -6,6 +6,7 @@ package com.intellij.ui.svg
import com.intellij.diagnostic.StartUpMeasurer
import com.intellij.openapi.diagnostic.logger
import com.intellij.ui.ColorUtil
import com.intellij.ui.hasher
import com.intellij.ui.icons.IconLoadMeasurer
import com.intellij.ui.icons.getResourceData
import com.intellij.ui.scale.JBUIScale
@@ -15,7 +16,6 @@ import com.intellij.util.SVGLoader
import com.intellij.util.text.CharSequenceReader
import com.intellij.util.xml.dom.createXmlStreamReader
import org.jetbrains.annotations.ApiStatus
import org.jetbrains.xxh3.Xxh3
import java.awt.Image
import java.awt.geom.Rectangle2D
import java.awt.image.BufferedImage
@@ -79,7 +79,8 @@ internal fun loadSvg(path: String?,
}!!
}
fun newSvgPatcher(digest: LongArray?, newPalette: Map<String, String>, alphas: Map<String, Int>): SvgAttributePatcher? {
@ApiStatus.Internal
fun newSvgPatcher(digest: LongArray?, newPalette: Map<String, String>, alphaProvider: (String) -> Int?): SvgAttributePatcher? {
if (newPalette.isEmpty()) {
return null
}
@@ -114,7 +115,7 @@ fun newSvgPatcher(digest: LongArray?, newPalette: Map<String, String>, alphas: M
}
if (newColor != null) {
attributes.put(attributeName, newColor)
alphas.get(newColor)?.let {
alphaProvider(newColor)?.let {
attributes.put("$attributeName-opacity", (it.toFloat() / 255f).toString())
}
}
@@ -193,7 +194,7 @@ private inline fun loadAndCacheIfApplicable(path: String?,
colorPatcher: SvgAttributePatcher?,
deprecatedColorPatcher: SVGLoader.SvgElementColorPatcher?,
dataProvider: () -> ByteArray?): BufferedImage? {
val colorPatcherDigest = colorPatcher?.digest() ?: deprecatedColorPatcher?.digest()?.let { longArrayOf(Xxh3.hash(it)) }
val colorPatcherDigest = colorPatcher?.digest() ?: deprecatedColorPatcher?.digest()?.let { longArrayOf(hasher.hashBytesToLong(it)) }
val svgCache = svgCache
if (svgCache == null || !svgCache.isActive() ||
(colorPatcherDigest == null && (colorPatcher != null || deprecatedColorPatcher != null)) ||
@@ -3,139 +3,59 @@
package com.intellij.util
import com.dynatrace.hash4j.hashing.HashFunnel
import com.intellij.ui.hasher
import it.unimi.dsi.fastutil.longs.LongArrayList
import org.jetbrains.annotations.ApiStatus.Experimental
import org.jetbrains.xxh3.Access
import org.jetbrains.xxh3.Xxh3
import org.jetbrains.xxh3.Xxh3Impl
import java.util.*
private const val extraSeed = 1867412186L
// see https://github.com/Cyan4973/xxHash/wiki/Collision-ratio-comparison
@Experimental
class InsecureHashBuilder {
private val result = LongArrayList()
private val hashStream = hasher.hashStream()
fun build(): LongArray {
return result.toLongArray()
}
fun update(value: CharSequence) {
result.add(Xxh3.hashUnencodedChars(value))
fun update(value: CharSequence): InsecureHashBuilder {
hashStream.putChars(value)
return addAndReset()
}
private fun addAndReset(): InsecureHashBuilder {
result.add(hashStream.asLong)
hashStream.reset()
return this
}
fun update(value: LongArray): InsecureHashBuilder {
result.add(Xxh3.hashLongs(value))
return this
hashStream.putLongArray(value)
return addAndReset()
}
fun update(value: Int) {
result.add(Xxh3.hashInt(value))
fun update(value: Int): InsecureHashBuilder {
hashStream.putInt(value)
return addAndReset()
}
fun updateRaw(hash: Long) {
result.add(hash)
}
fun stringList(list: List<String>) {
val size = list.size
result.add(Xxh3.hashInt(size))
if (list.isEmpty()) {
return
}
val offset = result.size
result.ensureCapacity(size)
for (s in list) {
result.add(Xxh3.hashUnencodedChars(s))
}
hash128(offset)
fun stringList(list: List<String>): InsecureHashBuilder {
hashStream.putOrderedIterable(list, HashFunnel.forString())
return addAndReset()
}
fun stringMap(map: Map<String, String>): InsecureHashBuilder {
val size = map.size
result.add(Xxh3.hashInt(size))
hashStream.putUnorderedIterable(map.entries, HashFunnel.forEntry(
HashFunnel.forString(),
HashFunnel.forString()), hasher)
return addAndReset()
}
if (map.isEmpty()) {
return this
}
val offset = result.size
result.ensureCapacity(size * 2)
if (map is SortedMap || map is LinkedHashMap) {
for ((k, v) in map) {
result.add(Xxh3.hashUnencodedChars(k))
result.add(Xxh3.hashUnencodedChars(v))
}
}
else {
for (k in map.keys.sorted()) {
result.add(Xxh3.hashUnencodedChars(k))
val v = map.get(k)
result.add(if (v == null) 0 else Xxh3.hashUnencodedChars(v))
}
}
hash128(offset)
fun stringIntMap(map: Map<String, Int>): InsecureHashBuilder {
hashStream.putUnorderedIterable(map.entries, HashFunnel.forEntry(
HashFunnel.forString(),
HashFunnel { v, sink -> sink.putInt(v) }
), hasher)
addAndReset()
return this
}
fun stringIntMap(map: Map<String, Int?>): InsecureHashBuilder {
val size = map.size
result.add(Xxh3.hashInt(size))
if (map.isEmpty()) {
return this
}
val offset = result.size
result.ensureCapacity(size * 2)
if (map is SortedMap || map is LinkedHashMap) {
for ((k, v) in map) {
result.add(Xxh3.hashUnencodedChars(k))
result.add(if (v == null) 0 else Xxh3.hashInt(v, 0))
}
}
else {
for (k in map.keys.sorted()) {
result.add(Xxh3.hashUnencodedChars(k))
val v = map.get(k)
result.add(if (v == null) 0 else Xxh3.hashInt(v, 0))
}
}
hash128(offset)
return this
}
private fun hash128(offset: Int) {
val sizeInBytes = (result.size - offset) * Long.SIZE_BYTES
val offsetInBytes = offset * Long.SIZE_BYTES
val l1 = Xxh3Impl.hash(result, LongListAccessForLongs, offsetInBytes, sizeInBytes, 0)
// and with a custom seed to get a 128-bit hash
val l2 = Xxh3Impl.hash(result, LongListAccessForLongs, offsetInBytes, sizeInBytes, extraSeed)
result.size(offset)
result.add(l1)
result.add(l2)
}
}
// special implementation for hashing long array - it is guaranteed that only i64 will be called (as input is aligned)
object LongListAccessForLongs : Access<LongArrayList> {
override fun i64(input: LongArrayList, offset: Int): Long {
return input.getLong(offset shr 3)
}
override fun i32(input: LongArrayList, offset: Int): Int {
val v = input.getLong(offset shr 3)
return if (offset and 7 == 0) (v shr 32).toInt() else v.toInt()
}
override fun i8(input: LongArrayList, offset: Int): Int {
throw UnsupportedOperationException()
}
}
@@ -10,6 +10,7 @@ import com.intellij.ui.scale.DerivedScaleType
import com.intellij.ui.scale.ScaleContext
import com.intellij.ui.svg.*
import com.intellij.util.ui.ImageUtil
import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap
import org.jetbrains.annotations.ApiStatus
import org.w3c.dom.Element
import java.awt.*
@@ -101,20 +102,25 @@ object SVGLoader {
backgroundColors: List<String> = emptyList()): SvgElementColorPatcherProvider {
val fg = ColorUtil.toHtmlColor(resultColor)
val map = strokeColors.associateWith { fg }
val alpha = HashMap<String, Int>(map.size)
val alpha = Object2IntOpenHashMap<String>(map.size)
alpha.defaultReturnValue(Int.MIN_VALUE)
for (s in map.values) {
alpha.put(s, resultColor.alpha)
}
val hash = InsecureHashBuilder()
hash.stringList(strokeColors)
hash.stringList(backgroundColors)
hash.update(fg)
hash.update(resultColor.alpha)
val digest = hash.build()
val digest = InsecureHashBuilder()
.stringList(strokeColors)
.stringList(backgroundColors)
.update(fg)
.update(resultColor.alpha)
.build()
return object : SvgElementColorPatcherProvider {
override fun attributeForPath(path: String?): SvgAttributePatcher? {
return newSvgPatcher(digest = digest, newPalette = map + backgroundColors.associateWith { "#00000000" }, alphas = alpha)
return newSvgPatcher(digest = digest,
newPalette = map + backgroundColors.associateWith { "#00000000" },
alphaProvider = { color ->
alpha.getInt(color).takeIf { it != Int.MIN_VALUE }
})
}
override fun digest() = digest