From ae9fd63d674978487ab547aed0a9465fd78d07da Mon Sep 17 00:00:00 2001 From: Alexander Shparun Date: Tue, 2 Sep 2025 21:33:25 +0200 Subject: [PATCH] [fleet] filter out optional transient scalar attributes from db snapshot GitOrigin-RevId: c2e20bc0f84337de1c46d8f8ef3dd5c17ceb7e86 --- .../fleet/kernel/DurableSnapshot.kt | 53 +++++++++++-------- .../srcCommonMain/fleet/kernel/Storage.kt | 2 +- .../fleet/kernel/rebase/DbSerialization.kt | 12 +++-- .../com/jetbrains/rhizomedb/Attributes.kt | 43 +++++++-------- 4 files changed, 61 insertions(+), 49 deletions(-) diff --git a/fleet/kernel/srcCommonMain/fleet/kernel/DurableSnapshot.kt b/fleet/kernel/srcCommonMain/fleet/kernel/DurableSnapshot.kt index 06095834750c..3214de7b4d90 100644 --- a/fleet/kernel/srcCommonMain/fleet/kernel/DurableSnapshot.kt +++ b/fleet/kernel/srcCommonMain/fleet/kernel/DurableSnapshot.kt @@ -8,6 +8,7 @@ import com.jetbrains.rhizomedb.Schema.Companion.NothingMask import com.jetbrains.rhizomedb.Schema.Companion.RefMask import com.jetbrains.rhizomedb.Schema.Companion.RequiredMask import com.jetbrains.rhizomedb.Schema.Companion.UniqueMask +import com.jetbrains.rhizomedb.impl.attributeSerializer import com.jetbrains.rhizomedb.impl.generateSeed import fleet.kernel.rebase.deserialize import fleet.kernel.rebase.encodeDbValue @@ -120,33 +121,41 @@ fun DbContext.buildDurableSnapshot( } } }) { - datoms.filter { it.attr != Entity.EntityObject.attr }.forEach { datom -> - val (e, a, v, t) = datom - curDatom = datom - val uid = requireNotNull(getOne(e, uidAttribute)) { "datom is not durable: ${displayDatom(datom)}" } - entities.computeShim(uid) { _: UID, m: MutableMap? -> - val map = (m ?: hashMapOf()) - val attr = DurableSnapshot.Attr(attributeIdent(a)!!, a.schema.value) - when (a.schema.cardinality) { - Cardinality.One -> { - map[attr] = DurableSnapshot.OneOrMany.One(DurableSnapshot.VersionedValue(encodeDbValue(uidAttribute, a, v), t)) - } - Cardinality.Many -> { - map.computeShim(attr) { _, existingValue -> - existingValue as DurableSnapshot.OneOrMany.Many? - val value = encodeDbValue(uidAttribute, a, v) - if (existingValue != null) { - DurableSnapshot.OneOrMany.Many(existingValue.values + DurableSnapshot.VersionedValue(value, t)) - } - else { - DurableSnapshot.OneOrMany.Many(listOf(DurableSnapshot.VersionedValue(value, t))) + datoms + .filter { it.attr != Entity.EntityObject.attr } + .filterNot { + val optional = !it.attr.schema.required + val scalar = !it.attr.schema.isRef + val transient = attributeSerializer(it.attr) == null + scalar && optional && transient + } + .forEach { datom -> + val (e, a, v, t) = datom + curDatom = datom + val uid = requireNotNull(getOne(e, uidAttribute)) { "datom is not durable: ${displayDatom(datom)}" } + entities.computeShim(uid) { _: UID, m: MutableMap? -> + val map = (m ?: hashMapOf()) + val attr = DurableSnapshot.Attr(attributeIdent(a)!!, a.schema.value) + when (a.schema.cardinality) { + Cardinality.One -> { + map[attr] = DurableSnapshot.OneOrMany.One(DurableSnapshot.VersionedValue(encodeDbValue(uidAttribute, a, v), t)) + } + Cardinality.Many -> { + map.computeShim(attr) { _, existingValue -> + existingValue as DurableSnapshot.OneOrMany.Many? + val value = encodeDbValue(uidAttribute, a, v) + if (existingValue != null) { + DurableSnapshot.OneOrMany.Many(existingValue.values + DurableSnapshot.VersionedValue(value, t)) + } + else { + DurableSnapshot.OneOrMany.Many(listOf(DurableSnapshot.VersionedValue(value, t))) + } } } } + map } - map } - } } return DurableSnapshot(entities = entities.map { DurableSnapshot.DurableEntity(it.key, it.value) }) } diff --git a/fleet/kernel/srcCommonMain/fleet/kernel/Storage.kt b/fleet/kernel/srcCommonMain/fleet/kernel/Storage.kt index 3c610e86fc4a..f53e3d038b0c 100644 --- a/fleet/kernel/srcCommonMain/fleet/kernel/Storage.kt +++ b/fleet/kernel/srcCommonMain/fleet/kernel/Storage.kt @@ -105,7 +105,7 @@ suspend fun withStorage( val entitiesCount = snapshot.snapshot.entities.size Storage.logger.debug { "snapshot for $storageKey built with $entitiesCount entities, took $snapshotBuildDuration" } val savingDuration = measureTime { - saveSnapshot(snapshot) + coroutineScope { saveSnapshot(snapshot) } } Storage.logger.debug { "successfully saved snapshot for $storageKey, written in $savingDuration" } } diff --git a/fleet/kernel/srcCommonMain/fleet/kernel/rebase/DbSerialization.kt b/fleet/kernel/srcCommonMain/fleet/kernel/rebase/DbSerialization.kt index 0a08c4ac9d1c..89f5d4184d76 100644 --- a/fleet/kernel/srcCommonMain/fleet/kernel/rebase/DbSerialization.kt +++ b/fleet/kernel/srcCommonMain/fleet/kernel/rebase/DbSerialization.kt @@ -16,9 +16,11 @@ fun DbContext.sharedId(eid: EID, uidAttribute: Attribute): UID? = else -> null } -internal fun DbContext.encodeDbValue(uidAttribute: Attribute, - a: Attribute<*>, - v: Any): DurableDbValue = +internal fun DbContext.encodeDbValue( + uidAttribute: Attribute, + a: Attribute<*>, + v: Any, +): DurableDbValue = when { a.schema.isRef -> { when (val typeIdent = getOne(v as EID, EntityType.Ident.attr as Attribute)) { @@ -52,7 +54,7 @@ internal fun DbContext.deserialize( internal fun DbContext.serialize1( eidToUid: DbContext.(EID) -> UID?, a: Attribute<*>, - v: Any + v: Any, ): DurableDbValue? = when { a.schema.isRef -> @@ -65,7 +67,7 @@ internal fun DbContext.serialize1( private fun DbContext.serializeScalar( attribute: Attribute<*>, - value: Any + value: Any, ): DurableDbValue.Scalar = when (value) { is JsonElement -> DurableDbValue.Scalar(lazyOf(value)) diff --git a/fleet/rhizomedb/srcCommonMain/com/jetbrains/rhizomedb/Attributes.kt b/fleet/rhizomedb/srcCommonMain/com/jetbrains/rhizomedb/Attributes.kt index 822bfd58ac3f..1c0736c300c1 100644 --- a/fleet/rhizomedb/srcCommonMain/com/jetbrains/rhizomedb/Attributes.kt +++ b/fleet/rhizomedb/srcCommonMain/com/jetbrains/rhizomedb/Attributes.kt @@ -21,7 +21,7 @@ fun attr(ident: String, schema: Schema): Attribute<*> = sealed class Attributes( val namespace: String, val module: String, - initial: Map> + initial: Map>, ) { private val mutableAttrInfos: MutableMap> = HashMap(initial) @@ -45,7 +45,7 @@ sealed class Attributes( ident: String, attr: Attribute<*>, serializer: Lazy>?, - defaultValue: DefaultValue? + defaultValue: DefaultValue?, ) : EntityAttribute(ident, attr, serializer, defaultValue) { operator fun invoke(entity: E): V = entity[this] operator fun getValue(entity: E, property: KProperty<*>): V = entity[this] @@ -55,7 +55,7 @@ sealed class Attributes( ident: String, attr: Attribute<*>, serializer: Lazy>?, - defaultValue: DefaultValue? + defaultValue: DefaultValue?, ) : EntityAttribute(ident, attr, serializer, defaultValue) { operator fun invoke(entity: E): V? = entity[this] operator fun getValue(entity: E, property: KProperty<*>): V? = entity[this] @@ -65,7 +65,7 @@ sealed class Attributes( ident: String, attr: Attribute<*>, serializer: Lazy>?, - defaultValue: DefaultValue? + defaultValue: DefaultValue?, ) : EntityAttribute(ident, attr, serializer, defaultValue) { operator fun invoke(entity: E): Set = entity[this] operator fun getValue(entity: E, property: KProperty<*>): Set = entity[this] @@ -82,7 +82,7 @@ sealed class Attributes( name: String, serializer: KSerializer, valueFlags: Indexing = Indexing.NOT_INDEXED, - defaultValueProvider: DefaultValue? = null + defaultValueProvider: DefaultValue? = null, ): Required = addAttr(Required( ident = "$namespace/$name", @@ -110,7 +110,7 @@ sealed class Attributes( name: String, serializer: KSerializer, valueFlags: Indexing = Indexing.NOT_INDEXED, - defaultValueProvider: DefaultValue? = null + defaultValueProvider: DefaultValue? = null, ): Optional = addAttr(Optional( ident = "$namespace/$name", @@ -165,7 +165,7 @@ sealed class Attributes( protected fun requiredTransient( name: String, valueFlags: Indexing = Indexing.NOT_INDEXED, - defaultValueProvider: DefaultValue? = null + defaultValueProvider: DefaultValue? = null, ): Required = addAttr(Required( ident = "$namespace/$name", @@ -192,19 +192,20 @@ sealed class Attributes( protected fun optionalTransient( name: String, valueFlags: Indexing = Indexing.NOT_INDEXED, - defaultValueProvider: DefaultValue? = null + defaultValueProvider: DefaultValue? = null, ): Optional = addAttr(Optional( ident = "$namespace/$name", - attr("$namespace/$name", schema = Schema( - cardinality = Cardinality.One, - isRef = false, - indexed = Indexing.INDEXED == valueFlags, - unique = Indexing.UNIQUE == valueFlags, - cascadeDelete = false, - cascadeDeleteBy = false, - required = false - )), + attr = attr(ident = "$namespace/$name", + schema = Schema( + cardinality = Cardinality.One, + isRef = false, + indexed = Indexing.INDEXED == valueFlags, + unique = Indexing.UNIQUE == valueFlags, + cascadeDelete = false, + cascadeDeleteBy = false, + required = false + )), serializer = null, defaultValue = defaultValueProvider )) @@ -247,7 +248,7 @@ sealed class Attributes( * */ protected fun requiredRef( name: String, - vararg refFlags: RefFlags + vararg refFlags: RefFlags, ): Required = addAttr(Required( ident = "$namespace/$name", @@ -272,7 +273,7 @@ sealed class Attributes( * */ protected fun optionalRef( name: String, - vararg refFlags: RefFlags + vararg refFlags: RefFlags, ): Optional = addAttr(Optional( ident = "$namespace/$name", @@ -296,7 +297,7 @@ sealed class Attributes( * */ protected fun manyRef( name: String, - vararg refFlags: RefFlags + vararg refFlags: RefFlags, ): Many = addAttr(Many( ident = "$namespace/$name", @@ -352,7 +353,7 @@ enum class RefFlags { CASCADE_DELETE_BY } -internal fun merge(attrs: List>): Map> = +internal fun merge(attrs: List>): Map> = buildMap { attrs.forEach { m -> m.entityAttributes.forEach { (k, v) ->