update readme and teal defs (REALLOC/COMPMOVE fragments)

README.md

@@ -61,6 +61,7 @@
     - [Chunk](#chunk)
     - [Builder](#builder)
 - [Changelog](#changelog)
+  - [vX.Y.Z](#vxyz)
   - [v1.7.0](#v170)
   - [v1.6.0](#v160)
   - [v1.5.0](#v150)
@@ -1171,6 +1172,9 @@ storage :: component[]
 default :: component
 duplicate :: {component -> component}
 
+realloc :: {storage?, integer, integer -> storage}
+compmove :: {storage, integer, integer, integer, storage}
+
 execute :: {chunk, entity[], integer, any...}
 prologue :: {any...}
 epilogue :: {any...}
@@ -1200,6 +1204,9 @@ INTERNAL :: fragment
 DEFAULT :: fragment
 DUPLICATE :: fragment
 
+REALLOC :: fragment
+COMPMOVE :: fragment
+
 PREFAB :: fragment
 DISABLED :: fragment
 
@@ -1335,6 +1342,9 @@ builder_mt:internal :: builder
 builder_mt:default :: component -> builder
 builder_mt:duplicate :: {component -> component} -> builder
 
+builder_mt:realloc :: {storage?, integer, integer -> storage} -> builder
+builder_mt:compmove :: {storage, integer, integer, integer, storage} -> builder
+
 builder_mt:prefab :: builder
 builder_mt:disabled :: builder
 
@@ -1361,6 +1371,10 @@ builder_mt:destruction_policy :: id -> builder
 
 ## Changelog
 
+### vX.Y.Z
+
+- Added the new [`evolved.REALLOC`](#evolvedrealloc) and [`evolved.COMPMOVE`](#evolvedcompmove) fragment traits that allow customizing component storages
+
 ### v1.7.0
 
 - Added the new [`evolved.VARIANTS`](#evolvedvariants) query fragment that allows specifying any of multiple fragments in queries
@@ -1430,6 +1444,10 @@ builder_mt:destruction_policy :: id -> builder
 
 ### `evolved.DUPLICATE`
 
+### `evolved.REALLOC`
+
+### `evolved.COMPMOVE`
+
 ### `evolved.PREFAB`
 
 ### `evolved.DISABLED`
@@ -2047,6 +2065,22 @@ function evolved.builder_mt:default(default) end
 function evolved.builder_mt:duplicate(duplicate) end
 ```
 
+#### `evolved.builder_mt:realloc`
+
+```lua
+---@param realloc evolved.realloc
+---@return evolved.builder builder
+function evolved.builder_mt:realloc(realloc) end
+```
+
+#### `evolved.builder_mt:compmove`
+
+```lua
+---@param compmove evolved.compmove
+---@return evolved.builder builder
+function evolved.builder_mt:compmove(compmove) end
+```
+
 #### `evolved.builder_mt:prefab`
 
 ```lua

develop/ROADMAP.md

@@ -9,6 +9,7 @@
 
 ## Thoughts
 
+- We can create component storages on-demand rather than in advance
 - We should have a way to not copy components on deferred spawn/clone
 
 ## Known Issues

develop/all.lua

@@ -8,6 +8,7 @@ require 'develop.testing.main_tests'
 require 'develop.testing.multi_spawn_tests'
 require 'develop.testing.name_tests'
 require 'develop.testing.process_with_tests'
+require 'develop.testing.realloc_tests'
 require 'develop.testing.requires_fragment_tests'
 require 'develop.testing.spawn_tests'
 require 'develop.testing.system_as_query_tests'
@@ -35,3 +36,5 @@ print '----------------------------------------'
 basics.describe_fuzz 'develop.fuzzing.requires_fuzz'
 print '----------------------------------------'
 basics.describe_fuzz 'develop.fuzzing.unique_fuzz'
+
+print 'All tests passed.'

develop/testing/realloc_tests.lua

@@ -0,0 +1,401 @@
+local evo = require 'evolved'
+
+---@type ffilib?
+local ffi = (function()
+    local ffi_loader = package and package.preload and package.preload['ffi']
+    local ffi = ffi_loader and ffi_loader()
+    return ffi
+end)()
+
+if not ffi then
+    return
+end
+
+local FLOAT_TYPEOF = ffi.typeof('float')
+local FLOAT_SIZEOF = ffi.sizeof(FLOAT_TYPEOF)
+local FLOAT_STORAGE_TYPEOF = ffi.typeof('$[?]', FLOAT_TYPEOF)
+
+local DOUBLE_TYPEOF = ffi.typeof('double')
+local DOUBLE_SIZEOF = ffi.sizeof(DOUBLE_TYPEOF)
+local DOUBLE_STORAGE_TYPEOF = ffi.typeof('$[?]', DOUBLE_TYPEOF)
+
+local STORAGE_SIZES = {}
+
+---@type evolved.realloc
+local function float_realloc(old_storage, old_size, new_size)
+    if old_storage then
+        assert(STORAGE_SIZES[old_storage] == old_size)
+    end
+
+    local new_storage = ffi.new(FLOAT_STORAGE_TYPEOF, new_size + 1)
+
+    STORAGE_SIZES[new_storage] = new_size
+
+    if old_storage then
+        ffi.copy(new_storage + 1, old_storage + 1, math.min(old_size, new_size) * FLOAT_SIZEOF)
+    end
+
+    return new_storage
+end
+
+---@type evolved.realloc
+local function double_realloc(old_storage, old_size, new_size)
+    if old_storage then
+        assert(STORAGE_SIZES[old_storage] == old_size)
+    end
+
+    local new_storage = ffi.new(DOUBLE_STORAGE_TYPEOF, new_size + 1)
+
+    STORAGE_SIZES[new_storage] = new_size
+
+    if old_storage then
+        ffi.copy(new_storage + 1, old_storage + 1, math.min(old_size, new_size) * DOUBLE_SIZEOF)
+    end
+
+    return new_storage
+end
+
+---@type evolved.compmove
+local function double_compmove(src, f, e, t, dst)
+    ffi.copy(dst + t, src + f, (e - f + 1) * DOUBLE_SIZEOF)
+end
+
+do
+    local f1 = evo.builder():realloc(double_realloc):build()
+
+    local e1 = evo.builder():set(f1, 21):build()
+    assert(evo.has(e1, f1) and evo.get(e1, f1) == 21)
+
+    local e2 = evo.builder():set(f1, 42):build()
+    assert(evo.has(e1, f1) and evo.get(e1, f1) == 21)
+    assert(evo.has(e2, f1) and evo.get(e2, f1) == 42)
+
+    local e3 = evo.builder():set(f1, 84):build()
+    assert(evo.has(e1, f1) and evo.get(e1, f1) == 21)
+    assert(evo.has(e2, f1) and evo.get(e2, f1) == 42)
+    assert(evo.has(e3, f1) and evo.get(e3, f1) == 84)
+
+    evo.destroy(e1)
+    assert(not evo.has(e1, f1))
+    assert(evo.has(e2, f1) and evo.get(e2, f1) == 42)
+    assert(evo.has(e3, f1) and evo.get(e3, f1) == 84)
+
+    evo.destroy(e3)
+    assert(not evo.has(e1, f1))
+    assert(evo.has(e2, f1) and evo.get(e2, f1) == 42)
+    assert(not evo.has(e3, f1))
+
+    evo.destroy(e2)
+    assert(not evo.has(e1, f1))
+    assert(not evo.has(e2, f1))
+    assert(not evo.has(e3, f1))
+end
+
+do
+    local f1 = evo.builder():realloc(double_realloc):build()
+    local q1 = evo.builder():include(f1):build()
+
+    do
+        local es, ec = {}, 10
+        for i = 1, ec do es[i] = evo.spawn({ [f1] = i }) end
+        for i = 1, ec do assert(evo.has(es[i], f1) and evo.get(es[i], f1) == i) end
+    end
+
+    do
+        local p = evo.builder():set(f1, 42):build()
+        local es, ec = {}, 10
+        for i = 1, ec do es[i] = evo.clone(p) end
+        for i = 1, ec do assert(evo.has(es[i], f1) and evo.get(es[i], f1) == 42) end
+    end
+
+    do
+        local es1, ec1 = evo.multi_spawn(10, { [f1] = 42 })
+        for i = 1, ec1 do assert(evo.has(es1[i], f1) and evo.get(es1[i], f1) == 42) end
+
+        local es2, ec2 = evo.multi_spawn(20, { [f1] = 84 })
+        for i = 1, ec1 do assert(evo.has(es1[i], f1) and evo.get(es1[i], f1) == 42) end
+        for i = 1, ec2 do assert(evo.has(es2[i], f1) and evo.get(es2[i], f1) == 84) end
+    end
+
+    do
+        local p = evo.builder():set(f1, 21):build()
+
+        local es1, ec1 = evo.multi_clone(10, p)
+        for i = 1, ec1 do assert(evo.has(es1[i], f1) and evo.get(es1[i], f1) == 21) end
+
+        local es2, ec2 = evo.multi_clone(20, p)
+        for i = 1, ec1 do assert(evo.has(es1[i], f1) and evo.get(es1[i], f1) == 21) end
+        for i = 1, ec2 do assert(evo.has(es2[i], f1) and evo.get(es2[i], f1) == 21) end
+    end
+
+    evo.batch_destroy(q1)
+end
+
+do
+    local f1 = evo.builder():realloc(double_realloc):build()
+    local f2 = evo.builder():realloc(double_realloc):build()
+
+    local q1 = evo.builder():include(f1):build()
+    local q2 = evo.builder():include(f2):build()
+
+    do
+        local e = evo.builder():set(f1, 21):set(f2, 42):build()
+        assert(evo.has(e, f1) and evo.get(e, f1) == 21)
+        assert(evo.has(e, f2) and evo.get(e, f2) == 42)
+
+        evo.remove(e, f1)
+
+        assert(not evo.has(e, f1))
+        assert(evo.has(e, f2) and evo.get(e, f2) == 42)
+    end
+
+    do
+        local e = evo.builder():set(f1, 21):set(f2, 42):build()
+        assert(evo.has(e, f1) and evo.get(e, f1) == 21)
+        assert(evo.has(e, f2) and evo.get(e, f2) == 42)
+
+        evo.clear(e)
+
+        assert(not evo.has(e, f1))
+        assert(not evo.has(e, f2))
+    end
+
+    do
+        local es, ec = evo.multi_spawn(10, { [f1] = 21, [f2] = 42 })
+
+        for i = 1, ec do
+            assert(evo.has(es[i], f1) and evo.get(es[i], f1) == 21)
+            assert(evo.has(es[i], f2) and evo.get(es[i], f2) == 42)
+        end
+
+        evo.batch_remove(q1, f1)
+
+        local e12 = evo.builder():set(f1, 1):set(f2, 2):build()
+        assert(evo.has(e12, f1) and evo.get(e12, f1) == 1)
+        assert(evo.has(e12, f2) and evo.get(e12, f2) == 2)
+
+        for i = 1, ec do
+            assert(not evo.has(es[i], f1))
+            assert(evo.has(es[i], f2) and evo.get(es[i], f2) == 42)
+        end
+
+        evo.batch_set(q2, f1, 84)
+
+        assert(evo.has(e12, f1) and evo.get(e12, f1) == 84)
+        assert(evo.has(e12, f2) and evo.get(e12, f2) == 2)
+
+        for i = 1, ec do
+            assert(evo.has(es[i], f1) and evo.get(es[i], f1) == 84)
+            assert(evo.has(es[i], f2) and evo.get(es[i], f2) == 42)
+        end
+
+        evo.batch_set(q2, f1, 21)
+
+        assert(evo.has(e12, f1) and evo.get(e12, f1) == 21)
+        assert(evo.has(e12, f2) and evo.get(e12, f2) == 2)
+
+        for i = 1, ec do
+            assert(evo.has(es[i], f1) and evo.get(es[i], f1) == 21)
+            assert(evo.has(es[i], f2) and evo.get(es[i], f2) == 42)
+        end
+    end
+end
+
+do
+    local f1 = evo.builder():realloc(double_realloc):compmove(double_compmove):build()
+    local f2 = evo.builder():realloc(double_realloc):compmove(double_compmove):build()
+
+    local q1 = evo.builder():include(f1):build()
+    local q2 = evo.builder():include(f2):build()
+
+    do
+        local es1, ec1 = evo.multi_spawn(10, { [f1] = 1, [f2] = 2 })
+
+        for i = 1, ec1 do
+            assert(evo.has(es1[i], f1) and evo.get(es1[i], f1) == 1)
+            assert(evo.has(es1[i], f2) and evo.get(es1[i], f2) == 2)
+        end
+
+        local es2, ec2 = evo.multi_spawn(20, { [f1] = 3, [f2] = 4 })
+
+        for i = 1, ec1 do
+            assert(evo.has(es1[i], f1) and evo.get(es1[i], f1) == 1)
+            assert(evo.has(es1[i], f2) and evo.get(es1[i], f2) == 2)
+        end
+
+        for i = 1, ec2 do
+            assert(evo.has(es2[i], f1) and evo.get(es2[i], f1) == 3)
+            assert(evo.has(es2[i], f2) and evo.get(es2[i], f2) == 4)
+        end
+
+        local e2 = evo.builder():set(f2, 42):build()
+        assert(evo.has(e2, f2) and evo.get(e2, f2) == 42)
+
+        evo.batch_remove(q1, f1)
+
+        assert(evo.has(e2, f2) and evo.get(e2, f2) == 42)
+
+        for i = 1, ec1 do
+            assert(not evo.has(es1[i], f1))
+            assert(evo.has(es1[i], f2) and evo.get(es1[i], f2) == 2)
+        end
+
+        for i = 1, ec2 do
+            assert(not evo.has(es2[i], f1))
+            assert(evo.has(es2[i], f2) and evo.get(es2[i], f2) == 4)
+        end
+
+        local e12 = evo.builder():set(f1, 21):set(f2, 42):build()
+
+        assert(evo.has(e2, f2) and evo.get(e2, f2) == 42)
+        assert(evo.has(e12, f1) and evo.get(e12, f1) == 21)
+        assert(evo.has(e12, f2) and evo.get(e12, f2) == 42)
+
+        evo.batch_set(q2, f1, 84)
+
+        assert(evo.has(e2, f2) and evo.get(e2, f2) == 42)
+        assert(evo.has(e12, f1) and evo.get(e12, f1) == 84)
+        assert(evo.has(e12, f2) and evo.get(e12, f2) == 42)
+
+        for i = 1, ec1 do
+            assert(evo.has(es1[i], f1) and evo.get(es1[i], f1) == 84)
+            assert(evo.has(es1[i], f2) and evo.get(es1[i], f2) == 2)
+        end
+
+        for i = 1, ec2 do
+            assert(evo.has(es2[i], f1) and evo.get(es2[i], f1) == 84)
+            assert(evo.has(es2[i], f2) and evo.get(es2[i], f2) == 4)
+        end
+    end
+end
+
+do
+    local f1 = evo.builder():default(42):build()
+
+    local es, ec = evo.multi_spawn(10, { [f1] = 21 })
+    for i = 1, ec do assert(evo.has(es[i], f1) and evo.get(es[i], f1) == 21) end
+
+    evo.set(f1, evo.TAG)
+    for i = 1, ec do assert(evo.has(es[i], f1) and evo.get(es[i], f1) == nil) end
+
+    evo.remove(f1, evo.TAG)
+    for i = 1, ec do assert(evo.has(es[i], f1) and evo.get(es[i], f1) == 42) end
+end
+
+do
+    local f1 = evo.builder():realloc(float_realloc):build()
+
+    local e1 = evo.builder():set(f1, 3):build()
+    assert(evo.has(e1, f1) and evo.get(e1, f1) == 3)
+
+    evo.set(f1, evo.REALLOC, double_realloc)
+    assert(evo.has(e1, f1) and evo.get(e1, f1) == 3)
+
+    evo.remove(f1, evo.REALLOC)
+    assert(evo.has(e1, f1) and evo.get(e1, f1) == 3)
+
+    evo.set(f1, evo.REALLOC, double_realloc)
+    assert(evo.has(e1, f1) and evo.get(e1, f1) == 3)
+end
+
+do
+    local f1 = evo.builder():realloc(double_realloc):build()
+
+    local es, ec = evo.multi_spawn(20, { [f1] = 42 })
+
+    for i = 1, ec / 2 do
+        evo.destroy(es[ec - i + 1])
+    end
+
+    evo.collect_garbage()
+
+    for i = 1, ec / 2 do
+        assert(evo.has(es[i], f1) and evo.get(es[i], f1) == 42)
+    end
+end
+
+do
+    evo.collect_garbage()
+
+    local f1 = evo.builder():name('f1'):realloc(double_realloc):compmove(double_compmove):build()
+    local f2 = evo.builder():name('f2'):realloc(double_realloc):compmove(double_compmove):build()
+
+    local q1 = evo.builder():include(f1):build()
+    local q2 = evo.builder():include(f2):build()
+
+    do
+        local es, ec = evo.multi_spawn(40, { [f2] = 2 })
+        for i = 1, ec do
+            assert(not evo.has(es[i], f1))
+            assert(evo.has(es[i], f2) and evo.get(es[i], f2) == 2)
+        end
+        evo.batch_destroy(q2)
+    end
+
+    do
+        local es, ec = evo.multi_spawn(50, { [f1] = 1, [f2] = 2 })
+        for i = 1, ec do
+            assert(evo.has(es[i], f1) and evo.get(es[i], f1) == 1)
+            assert(evo.has(es[i], f2) and evo.get(es[i], f2) == 2)
+        end
+
+        evo.batch_remove(q1, f1)
+        for i = 1, ec do
+            assert(not evo.has(es[i], f1))
+            assert(evo.has(es[i], f2) and evo.get(es[i], f2) == 2)
+        end
+
+        evo.batch_destroy(q1, q2)
+    end
+
+    do
+        evo.spawn({ [f1] = 1 })
+        evo.spawn({ [f2] = 2 })
+        evo.spawn({ [f1] = 1, [f2] = 2 })
+    end
+
+    evo.collect_garbage()
+end
+
+do
+    evo.collect_garbage()
+
+    local f1 = evo.builder():name('f1'):realloc(double_realloc):compmove(double_compmove):build()
+    local f2 = evo.builder():name('f2'):realloc(double_realloc):compmove(double_compmove):build()
+
+    local q1 = evo.builder():include(f1):build()
+    local q2 = evo.builder():include(f2):build()
+
+    do
+        local es, ec = evo.multi_spawn(40, { [f1] = 1, [f2] = 2 })
+        for i = 1, ec do
+            assert(evo.has(es[i], f1) and evo.get(es[i], f1) == 1)
+            assert(evo.has(es[i], f2) and evo.get(es[i], f2) == 2)
+        end
+        evo.batch_destroy(q2)
+    end
+
+    do
+        local es, ec = evo.multi_spawn(50, { [f1] = 1 })
+        for i = 1, ec do
+            assert(evo.has(es[i], f1) and evo.get(es[i], f1) == 1)
+            assert(not evo.has(es[i], f2))
+        end
+
+        evo.batch_set(q1, f2, 2)
+        for i = 1, ec do
+            assert(evo.has(es[i], f1) and evo.get(es[i], f1) == 1)
+            assert(evo.has(es[i], f2) and evo.get(es[i], f2) == 2)
+        end
+
+        evo.batch_destroy(q1, q2)
+    end
+
+    do
+        evo.spawn({ [f1] = 1 })
+        evo.spawn({ [f2] = 2 })
+        evo.spawn({ [f1] = 1, [f2] = 2 })
+    end
+
+    evo.collect_garbage()
+end

evolved.d.tl

@@ -64,6 +64,9 @@
         default: function<Component>(self: Builder, default: Component): Builder
         duplicate: function<Component>(self: Builder, duplicate: function(Component): Component): Builder
 
+        realloc: function<Component>(self: Builder, realloc: function({ Component } | nil, integer, integer): { Component }): Builder
+        compmove: function<Component>(self: Builder, compmove: function({ Component }, integer, integer, integer, { Component })): Builder
+
         prefab: function(self: Builder): Builder
         disabled: function(self: Builder): Builder
 
@@ -98,6 +101,9 @@
     DEFAULT: Fragment
     DUPLICATE: Fragment
 
+    REALLOC: Fragment
+    COMPMOVE: Fragment
+
     PREFAB: Fragment
     DISABLED: Fragment
 

evolved.lua

@@ -40,6 +40,9 @@ local evolved = {
 ---@alias evolved.default evolved.component
 ---@alias evolved.duplicate fun(component: evolved.component): evolved.component
 
+---@alias evolved.realloc fun(src?: evolved.storage, old_size: integer, new_size: integer): evolved.storage
+---@alias evolved.compmove fun(src: evolved.storage, f: integer, e: integer, t: integer, dst: evolved.storage)
+
 ---@alias evolved.execute fun(
 ---  chunk: evolved.chunk,
 ---  entity_list: evolved.entity[],
@@ -158,6 +161,7 @@ local __structural_changes = 0 ---@type integer
 ---@field package __child_count integer
 ---@field package __entity_list evolved.entity[]
 ---@field package __entity_count integer
+---@field package __entity_capacity integer
 ---@field package __fragment evolved.fragment
 ---@field package __fragment_set table<evolved.fragment, integer>
 ---@field package __fragment_list evolved.fragment[]
@@ -168,6 +172,8 @@ local __structural_changes = 0 ---@type integer
 ---@field package __component_fragments evolved.fragment[]
 ---@field package __component_defaults evolved.default[]
 ---@field package __component_duplicates evolved.duplicate[]
+---@field package __component_reallocs evolved.realloc[]
+---@field package __component_compmoves evolved.compmove[]
 ---@field package __with_fragment_edges table<evolved.fragment, evolved.chunk>
 ---@field package __without_fragment_edges table<evolved.fragment, evolved.chunk>
 ---@field package __with_required_fragments? evolved.chunk
@@ -186,6 +192,8 @@ local __structural_changes = 0 ---@type integer
 ---@field package __has_internal_major boolean
 ---@field package __has_internal_minors boolean
 ---@field package __has_internal_fragments boolean
+---@field package __has_storage_reallocs boolean
+---@field package __has_storage_compmoves boolean
 ---@field package __has_required_fragments boolean
 local __chunk_mt = {}
 __chunk_mt.__index = __chunk_mt
@@ -971,6 +979,9 @@ local __INTERNAL = __acquire_id()
 local __DEFAULT = __acquire_id()
 local __DUPLICATE = __acquire_id()
 
+local __REALLOC = __acquire_id()
+local __COMPMOVE = __acquire_id()
+
 local __PREFAB = __acquire_id()
 local __DISABLED = __acquire_id()
 
@@ -1144,6 +1155,8 @@ local __clone_entity
 local __multi_clone_entity
 
 local __purge_chunk
+local __expand_chunk
+local __shrink_chunk
 local __clear_chunk_list
 local __destroy_entity_list
 local __destroy_fragment_list
@@ -1221,6 +1234,7 @@ function __new_chunk(chunk_parent, chunk_fragment)
         __child_count = 0,
         __entity_list = {},
         __entity_count = 0,
+        __entity_capacity = 0,
         __fragment = chunk_fragment,
         __fragment_set = chunk_fragment_set,
         __fragment_list = chunk_fragment_list,
@@ -1231,6 +1245,8 @@ function __new_chunk(chunk_parent, chunk_fragment)
         __component_fragments = {},
         __component_defaults = {},
         __component_duplicates = {},
+        __component_reallocs = {},
+        __component_compmoves = {},
         __with_fragment_edges = {},
         __without_fragment_edges = {},
         __with_required_fragments = nil,
@@ -1249,6 +1265,8 @@ function __new_chunk(chunk_parent, chunk_fragment)
         __has_internal_major = false,
         __has_internal_minors = false,
         __has_internal_fragments = false,
+        __has_storage_reallocs = false,
+        __has_storage_compmoves = false,
         __has_required_fragments = false,
     }, __chunk_mt)
 
@@ -1337,6 +1355,12 @@ function __update_chunk_caches(chunk)
     local has_internal_minors = chunk_parent ~= nil and chunk_parent.__has_internal_fragments
     local has_internal_fragments = has_internal_major or has_internal_minors
 
+    local has_storage_reallocs = chunk_parent ~= nil and chunk_parent.__has_storage_reallocs
+        or __evolved_has(chunk_fragment, __REALLOC)
+
+    local has_storage_compmoves = chunk_parent ~= nil and chunk_parent.__has_storage_compmoves
+        or __evolved_has(chunk_fragment, __COMPMOVE)
+
     local has_required_fragments = false
 
     for chunk_fragment_index = 1, chunk_fragment_count do
@@ -1377,6 +1401,9 @@ function __update_chunk_caches(chunk)
     chunk.__has_internal_minors = has_internal_minors
     chunk.__has_internal_fragments = has_internal_fragments
 
+    chunk.__has_storage_reallocs = has_storage_reallocs
+    chunk.__has_storage_compmoves = has_storage_compmoves
+
     chunk.__has_required_fragments = has_required_fragments
 
     if has_required_fragments then
@@ -1415,6 +1442,7 @@ end
 ---@param chunk evolved.chunk
 function __update_chunk_storages(chunk)
     local entity_count = chunk.__entity_count
+    local entity_capacity = chunk.__entity_capacity
 
     local fragment_list = chunk.__fragment_list
     local fragment_count = chunk.__fragment_count
@@ -1425,14 +1453,18 @@ function __update_chunk_storages(chunk)
     local component_fragments = chunk.__component_fragments
     local component_defaults = chunk.__component_defaults
     local component_duplicates = chunk.__component_duplicates
+    local component_reallocs = chunk.__component_reallocs
+    local component_compmoves = chunk.__component_compmoves
 
     for fragment_index = 1, fragment_count do
         local fragment = fragment_list[fragment_index]
-        local component_index = component_indices[fragment]
 
-        ---@type evolved.default?, evolved.duplicate?
-        local fragment_default, fragment_duplicate =
-            __evolved_get(fragment, __DEFAULT, __DUPLICATE)
+        local component_index = component_indices[fragment]
+        local component_realloc = component_index and component_reallocs[component_index]
+
+        ---@type evolved.default?, evolved.duplicate?, evolved.realloc?, evolved.compmove?
+        local fragment_default, fragment_duplicate, fragment_realloc, fragment_compmove =
+            __evolved_get(fragment, __DEFAULT, __DUPLICATE, __REALLOC, __COMPMOVE)
 
         local is_fragment_tag = __evolved_has(fragment, __TAG)
 
@@ -1442,12 +1474,16 @@ function __update_chunk_storages(chunk)
                 local last_component_fragment = component_fragments[component_count]
                 local last_component_default = component_defaults[component_count]
                 local last_component_duplicate = component_duplicates[component_count]
+                local last_component_realloc = component_reallocs[component_count]
+                local last_component_compmove = component_compmoves[component_count]
 
                 component_indices[last_component_fragment] = component_index
                 component_storages[component_index] = last_component_storage
                 component_fragments[component_index] = last_component_fragment
                 component_defaults[component_index] = last_component_default
                 component_duplicates[component_index] = last_component_duplicate
+                component_reallocs[component_index] = last_component_realloc
+                component_compmoves[component_index] = last_component_compmove
             end
 
             component_indices[fragment] = nil
@@ -1455,6 +1491,8 @@ function __update_chunk_storages(chunk)
             component_fragments[component_count] = nil
             component_defaults[component_count] = nil
             component_duplicates[component_count] = nil
+            component_reallocs[component_count] = nil
+            component_compmoves[component_count] = nil
 
             component_count = component_count - 1
             chunk.__component_count = component_count
@@ -1462,7 +1500,9 @@ function __update_chunk_storages(chunk)
             component_count = component_count + 1
             chunk.__component_count = component_count
 
-            local component_storage = __lua_table_new(entity_count)
+            local component_storage = fragment_realloc
+                and fragment_realloc(nil, 0, entity_capacity)
+                or __lua_table_new(entity_capacity)
             local component_storage_index = component_count
 
             component_indices[fragment] = component_storage_index
@@ -1470,6 +1510,8 @@ function __update_chunk_storages(chunk)
             component_fragments[component_storage_index] = fragment
             component_defaults[component_storage_index] = fragment_default
             component_duplicates[component_storage_index] = fragment_duplicate
+            component_reallocs[component_storage_index] = fragment_realloc
+            component_compmoves[component_storage_index] = fragment_compmove
 
             if fragment_duplicate then
                 for place = 1, entity_count do
@@ -1486,8 +1528,37 @@ function __update_chunk_storages(chunk)
                 end
             end
         elseif component_index then
+            if component_realloc ~= fragment_realloc then
+                local old_component_storage = component_storages[component_index]
+
+                local new_component_storage = fragment_realloc
+                    and fragment_realloc(nil, 0, entity_capacity)
+                    or __lua_table_new(entity_capacity)
+
+                if fragment_duplicate then
+                    for place = 1, entity_count do
+                        local new_component = old_component_storage[place]
+                        if new_component == nil then new_component = fragment_default end
+                        if new_component ~= nil then new_component = fragment_duplicate(new_component) end
+                        if new_component == nil then new_component = true end
+                        new_component_storage[place] = new_component
+                    end
+                else
+                    for place = 1, entity_count do
+                        local new_component = old_component_storage[place]
+                        if new_component == nil then new_component = fragment_default end
+                        if new_component == nil then new_component = true end
+                        new_component_storage[place] = new_component
+                    end
+                end
+
+                component_storages[component_index] = new_component_storage
+            end
+
             component_defaults[component_index] = fragment_default
             component_duplicates[component_index] = fragment_duplicate
+            component_reallocs[component_index] = fragment_realloc
+            component_compmoves[component_index] = fragment_compmove
         end
     end
 end
@@ -2159,27 +2230,16 @@ function __detach_entity(chunk, place)
     local component_count = chunk.__component_count
     local component_storages = chunk.__component_storages
 
-    if place == entity_count then
-        entity_list[entity_count] = nil
-
-        for component_index = 1, component_count do
-            local component_storage = component_storages[component_index]
-            component_storage[entity_count] = nil
-        end
-    else
+    if place ~= entity_count then
         local last_entity = entity_list[entity_count]
-        local last_entity_primary = last_entity % 2 ^ 20
+        entity_list[place] = last_entity
 
+        local last_entity_primary = last_entity % 2 ^ 20
         __entity_places[last_entity_primary] = place
 
-        entity_list[place] = last_entity
-        entity_list[entity_count] = nil
-
         for component_index = 1, component_count do
             local component_storage = component_storages[component_index]
-            local last_entity_component = component_storage[entity_count]
-            component_storage[place] = last_entity_component
-            component_storage[entity_count] = nil
+            component_storage[place] = component_storage[entity_count]
         end
     end
 
@@ -2188,18 +2248,6 @@ end
 
 ---@param chunk evolved.chunk
 function __detach_all_entities(chunk)
-    local entity_list = chunk.__entity_list
-
-    local component_count = chunk.__component_count
-    local component_storages = chunk.__component_storages
-
-    __lua_table_clear(entity_list)
-
-    for component_index = 1, component_count do
-        local component_storage = component_storages[component_index]
-        __lua_table_clear(component_storage)
-    end
-
     chunk.__entity_count = 0
 end
 
@@ -2230,9 +2278,6 @@ function __spawn_entity(chunk, entity, components)
         return
     end
 
-    local chunk_entity_list = chunk.__entity_list
-    local chunk_entity_count = chunk.__entity_count
-
     local chunk_component_count = chunk.__component_count
     local chunk_component_indices = chunk.__component_indices
     local chunk_component_storages = chunk.__component_storages
@@ -2240,7 +2285,13 @@ function __spawn_entity(chunk, entity, components)
     local chunk_component_defaults = chunk.__component_defaults
     local chunk_component_duplicates = chunk.__component_duplicates
 
-    local place = chunk_entity_count + 1
+    local place = chunk.__entity_count + 1
+
+    if place > chunk.__entity_capacity then
+        __expand_chunk(chunk, place)
+    end
+
+    local chunk_entity_list = chunk.__entity_list
 
     do
         chunk.__entity_count = place
@@ -2349,9 +2400,6 @@ function __multi_spawn_entity(chunk, entity_list, entity_count, components)
         return
     end
 
-    local chunk_entity_list = chunk.__entity_list
-    local chunk_entity_count = chunk.__entity_count
-
     local chunk_component_count = chunk.__component_count
     local chunk_component_indices = chunk.__component_indices
     local chunk_component_storages = chunk.__component_storages
@@ -2359,8 +2407,14 @@ function __multi_spawn_entity(chunk, entity_list, entity_count, components)
     local chunk_component_defaults = chunk.__component_defaults
     local chunk_component_duplicates = chunk.__component_duplicates
 
-    local b_place = chunk_entity_count + 1
-    local e_place = chunk_entity_count + entity_count
+    local b_place = chunk.__entity_count + 1
+    local e_place = b_place + entity_count - 1
+
+    if e_place > chunk.__entity_capacity then
+        __expand_chunk(chunk, e_place)
+    end
+
+    local chunk_entity_list = chunk.__entity_list
 
     do
         chunk.__entity_count = e_place
@@ -2500,9 +2554,6 @@ function __clone_entity(prefab, entity, components)
         return
     end
 
-    local chunk_entity_list = chunk.__entity_list
-    local chunk_entity_count = chunk.__entity_count
-
     local chunk_component_count = chunk.__component_count
     local chunk_component_indices = chunk.__component_indices
     local chunk_component_storages = chunk.__component_storages
@@ -2513,7 +2564,13 @@ function __clone_entity(prefab, entity, components)
     local prefab_component_indices = prefab_chunk.__component_indices
     local prefab_component_storages = prefab_chunk.__component_storages
 
-    local place = chunk_entity_count + 1
+    local place = chunk.__entity_count + 1
+
+    if place > chunk.__entity_capacity then
+        __expand_chunk(chunk, place)
+    end
+
+    local chunk_entity_list = chunk.__entity_list
 
     do
         chunk.__entity_count = place
@@ -2646,9 +2703,6 @@ function __multi_clone_entity(prefab, entity_list, entity_count, components)
         return
     end
 
-    local chunk_entity_list = chunk.__entity_list
-    local chunk_entity_count = chunk.__entity_count
-
     local chunk_component_count = chunk.__component_count
     local chunk_component_indices = chunk.__component_indices
     local chunk_component_storages = chunk.__component_storages
@@ -2659,8 +2713,14 @@ function __multi_clone_entity(prefab, entity_list, entity_count, components)
     local prefab_component_indices = prefab_chunk.__component_indices
     local prefab_component_storages = prefab_chunk.__component_storages
 
-    local b_place = chunk_entity_count + 1
-    local e_place = chunk_entity_count + entity_count
+    local b_place = chunk.__entity_count + 1
+    local e_place = b_place + entity_count - 1
+
+    if e_place > chunk.__entity_capacity then
+        __expand_chunk(chunk, e_place)
+    end
+
+    local chunk_entity_list = chunk.__entity_list
 
     do
         chunk.__entity_count = e_place
@@ -2833,6 +2893,108 @@ function __purge_chunk(chunk)
     chunk.__unreachable_or_collected = true
 end
 
+---@param chunk evolved.chunk
+---@param min_capacity integer
+function __expand_chunk(chunk, min_capacity)
+    if __defer_depth <= 0 then
+        __error_fmt('this operation should be deferred')
+    end
+
+    local entity_count = chunk.__entity_count
+
+    if min_capacity < entity_count then
+        min_capacity = entity_count
+    end
+
+    local old_capacity = chunk.__entity_capacity
+    if old_capacity >= min_capacity then
+        -- no need to expand, the chunk is already large enough
+        return
+    end
+
+    local new_capacity = old_capacity * 2
+
+    if new_capacity < min_capacity then
+        new_capacity = min_capacity
+    end
+
+    if new_capacity < 4 then
+        new_capacity = 4
+    end
+
+    if chunk.__has_storage_reallocs then
+        local component_count = chunk.__component_count
+        local component_storages = chunk.__component_storages
+        local component_reallocs = chunk.__component_reallocs
+
+        for component_index = 1, component_count do
+            local component_realloc = component_reallocs[component_index]
+
+            if component_realloc then
+                local old_component_storage = component_storages[component_index]
+
+                local new_component_storage = component_realloc(
+                    old_component_storage, old_capacity, new_capacity)
+
+                component_storages[component_index] = new_component_storage
+            end
+        end
+    end
+
+    chunk.__entity_capacity = new_capacity
+end
+
+---@param chunk evolved.chunk
+---@param min_capacity integer
+function __shrink_chunk(chunk, min_capacity)
+    if __defer_depth <= 0 then
+        __error_fmt('this operation should be deferred')
+    end
+
+    local entity_count = chunk.__entity_count
+
+    if min_capacity < entity_count then
+        min_capacity = entity_count
+    end
+
+    local old_capacity = chunk.__entity_capacity
+    if old_capacity <= min_capacity then
+        -- no need to shrink, the chunk is already small enough
+        return
+    end
+
+    do
+        local entity_list = __lua_table_new(min_capacity)
+        __lua_table_move(chunk.__entity_list, 1, entity_count, 1, entity_list)
+        chunk.__entity_list = entity_list
+    end
+
+    do
+        local component_count = chunk.__component_count
+        local component_storages = chunk.__component_storages
+        local component_reallocs = chunk.__component_reallocs
+
+        for component_index = 1, component_count do
+            local component_realloc = component_reallocs[component_index]
+
+            local old_component_storage = component_storages[component_index]
+            local new_component_storage ---@type evolved.storage?
+
+            if component_realloc then
+                new_component_storage = component_realloc(
+                    old_component_storage, old_capacity, min_capacity)
+            else
+                new_component_storage = __lua_table_new(min_capacity)
+                __lua_table_move(old_component_storage, 1, entity_count, 1, new_component_storage)
+            end
+
+            component_storages[component_index] = new_component_storage
+        end
+    end
+
+    chunk.__entity_capacity = min_capacity
+end
+
 ---@param chunk_list evolved.chunk[]
 ---@param chunk_count integer
 function __clear_chunk_list(chunk_list, chunk_count)
@@ -3152,11 +3314,10 @@ function __chunk_set(old_chunk, fragment, component)
             end
         end
 
-        local new_entity_list = new_chunk.__entity_list
-        local new_entity_count = new_chunk.__entity_count
-
         local new_component_indices = new_chunk.__component_indices
         local new_component_storages = new_chunk.__component_storages
+        local new_component_reallocs = new_chunk.__component_reallocs
+        local new_component_compmoves = new_chunk.__component_compmoves
 
         local new_chunk_has_setup_hooks = new_chunk.__has_setup_hooks
         local new_chunk_has_insert_hooks = new_chunk.__has_insert_hooks
@@ -3169,7 +3330,14 @@ function __chunk_set(old_chunk, fragment, component)
                 __evolved_get(fragment, __DEFAULT, __DUPLICATE, __ON_SET, __ON_INSERT)
         end
 
-        local sum_entity_count = old_entity_count + new_entity_count
+        local sum_entity_count = old_entity_count + new_chunk.__entity_count
+
+        if sum_entity_count > new_chunk.__entity_capacity then
+            __expand_chunk(new_chunk, sum_entity_count)
+        end
+
+        local new_entity_list = new_chunk.__entity_list
+        local new_entity_count = new_chunk.__entity_count
 
         if new_entity_count == 0 then
             old_chunk.__entity_list, new_chunk.__entity_list =
@@ -3180,10 +3348,28 @@ function __chunk_set(old_chunk, fragment, component)
 
             for old_ci = 1, old_component_count do
                 local old_f = old_component_fragments[old_ci]
-                local new_ci = new_component_indices[old_f]
 
-                old_component_storages[old_ci], new_component_storages[new_ci] =
-                    new_component_storages[new_ci], old_component_storages[old_ci]
+                local new_ci = new_component_indices[old_f]
+                local new_cr = new_component_reallocs[new_ci]
+
+                if new_cr then
+                    local old_cs = old_component_storages[old_ci]
+
+                    local new_cs = new_component_storages[new_ci]
+                    local new_cm = new_component_compmoves[new_ci]
+
+                    if new_cm then
+                        new_cm(old_cs, 1, old_entity_count, new_entity_count + 1, new_cs)
+                    else
+                        for old_place = 1, old_entity_count do
+                            local new_place = new_entity_count + old_place
+                            new_cs[new_place] = old_cs[old_place]
+                        end
+                    end
+                else
+                    old_component_storages[old_ci], new_component_storages[new_ci] =
+                        new_component_storages[new_ci], old_component_storages[old_ci]
+                end
             end
 
             new_chunk.__entity_count = sum_entity_count
@@ -3194,10 +3380,22 @@ function __chunk_set(old_chunk, fragment, component)
 
                 local new_ci = new_component_indices[old_f]
                 local new_cs = new_component_storages[new_ci]
+                local new_cr = new_component_reallocs[new_ci]
 
-                __lua_table_move(
-                    old_cs, 1, old_entity_count,
-                    new_entity_count + 1, new_cs)
+                if new_cr then
+                    local new_cm = new_component_compmoves[new_ci]
+
+                    if new_cm then
+                        new_cm(old_cs, 1, old_entity_count, new_entity_count + 1, new_cs)
+                    else
+                        for old_place = 1, old_entity_count do
+                            local new_place = new_entity_count + old_place
+                            new_cs[new_place] = old_cs[old_place]
+                        end
+                    end
+                else
+                    __lua_table_move(old_cs, 1, old_entity_count, new_entity_count + 1, new_cs)
+                end
             end
 
             __lua_table_move(
@@ -3475,14 +3673,20 @@ function __chunk_remove(old_chunk, ...)
     end
 
     if new_chunk then
-        local new_entity_list = new_chunk.__entity_list
-        local new_entity_count = new_chunk.__entity_count
-
         local new_component_count = new_chunk.__component_count
         local new_component_storages = new_chunk.__component_storages
         local new_component_fragments = new_chunk.__component_fragments
+        local new_component_reallocs = new_chunk.__component_reallocs
+        local new_component_compmoves = new_chunk.__component_compmoves
 
-        local sum_entity_count = old_entity_count + new_entity_count
+        local sum_entity_count = old_entity_count + new_chunk.__entity_count
+
+        if sum_entity_count > new_chunk.__entity_capacity then
+            __expand_chunk(new_chunk, sum_entity_count)
+        end
+
+        local new_entity_list = new_chunk.__entity_list
+        local new_entity_count = new_chunk.__entity_count
 
         if new_entity_count == 0 then
             old_chunk.__entity_list, new_chunk.__entity_list =
@@ -3493,10 +3697,28 @@ function __chunk_remove(old_chunk, ...)
 
             for new_ci = 1, new_component_count do
                 local new_f = new_component_fragments[new_ci]
+                local new_cr = new_component_reallocs[new_ci]
+
                 local old_ci = old_component_indices[new_f]
 
-                old_component_storages[old_ci], new_component_storages[new_ci] =
-                    new_component_storages[new_ci], old_component_storages[old_ci]
+                if new_cr then
+                    local new_cs = new_component_storages[new_ci]
+                    local new_cm = new_component_compmoves[new_ci]
+
+                    local old_cs = old_component_storages[old_ci]
+
+                    if new_cm then
+                        new_cm(old_cs, 1, old_entity_count, new_entity_count + 1, new_cs)
+                    else
+                        for old_place = 1, old_entity_count do
+                            local new_place = new_entity_count + old_place
+                            new_cs[new_place] = old_cs[old_place]
+                        end
+                    end
+                else
+                    old_component_storages[old_ci], new_component_storages[new_ci] =
+                        new_component_storages[new_ci], old_component_storages[old_ci]
+                end
             end
 
             new_chunk.__entity_count = sum_entity_count
@@ -3504,13 +3726,25 @@ function __chunk_remove(old_chunk, ...)
             for new_ci = 1, new_component_count do
                 local new_f = new_component_fragments[new_ci]
                 local new_cs = new_component_storages[new_ci]
+                local new_cr = new_component_reallocs[new_ci]
 
                 local old_ci = old_component_indices[new_f]
                 local old_cs = old_component_storages[old_ci]
 
-                __lua_table_move(
-                    old_cs, 1, old_entity_count,
-                    new_entity_count + 1, new_cs)
+                if new_cr then
+                    local new_cm = new_component_compmoves[new_ci]
+
+                    if new_cm then
+                        new_cm(old_cs, 1, old_entity_count, new_entity_count + 1, new_cs)
+                    else
+                        for old_place = 1, old_entity_count do
+                            local new_place = new_entity_count + old_place
+                            new_cs[new_place] = old_cs[old_place]
+                        end
+                    end
+                else
+                    __lua_table_move(old_cs, 1, old_entity_count, new_entity_count + 1, new_cs)
+                end
             end
 
             __lua_table_move(
@@ -4649,9 +4883,6 @@ function __evolved_set(entity, fragment, component)
             end
         end
 
-        local new_entity_list = new_chunk.__entity_list
-        local new_entity_count = new_chunk.__entity_count
-
         local new_component_indices = new_chunk.__component_indices
         local new_component_storages = new_chunk.__component_storages
 
@@ -4666,10 +4897,16 @@ function __evolved_set(entity, fragment, component)
                 __evolved_get(fragment, __DEFAULT, __DUPLICATE, __ON_SET, __ON_INSERT)
         end
 
-        local new_place = new_entity_count + 1
-        new_chunk.__entity_count = new_place
+        local new_place = new_chunk.__entity_count + 1
+
+        if new_place > new_chunk.__entity_capacity then
+            __expand_chunk(new_chunk, new_place)
+        end
+
+        local new_entity_list = new_chunk.__entity_list
 
         new_entity_list[new_place] = entity
+        new_chunk.__entity_count = new_place
 
         if old_chunk then
             local old_component_count = old_chunk.__component_count
@@ -4855,17 +5092,20 @@ function __evolved_remove(entity, ...)
         end
 
         if new_chunk then
-            local new_entity_list = new_chunk.__entity_list
-            local new_entity_count = new_chunk.__entity_count
-
             local new_component_count = new_chunk.__component_count
             local new_component_storages = new_chunk.__component_storages
             local new_component_fragments = new_chunk.__component_fragments
 
-            local new_place = new_entity_count + 1
-            new_chunk.__entity_count = new_place
+            local new_place = new_chunk.__entity_count + 1
+
+            if new_place > new_chunk.__entity_capacity then
+                __expand_chunk(new_chunk, new_place)
+            end
+
+            local new_entity_list = new_chunk.__entity_list
 
             new_entity_list[new_place] = entity
+            new_chunk.__entity_count = new_place
 
             for new_ci = 1, new_component_count do
                 local new_f = new_component_fragments[new_ci]
@@ -5469,6 +5709,8 @@ function __evolved_collect_garbage()
 
             if should_be_purged then
                 __purge_chunk(postorder_chunk)
+            else
+                __shrink_chunk(postorder_chunk, 0)
             end
         end
 
@@ -6051,6 +6293,18 @@ function __builder_mt:duplicate(duplicate)
     return self:set(__DUPLICATE, duplicate)
 end
 
+---@param realloc evolved.realloc
+---@return evolved.builder builder
+function __builder_mt:realloc(realloc)
+    return self:set(__REALLOC, realloc)
+end
+
+---@param compmove evolved.compmove
+---@return evolved.builder builder
+function __builder_mt:compmove(compmove)
+    return self:set(__COMPMOVE, compmove)
+end
+
 ---@return evolved.builder builder
 function __builder_mt:prefab()
     return self:set(__PREFAB)
@@ -6263,6 +6517,12 @@ __evolved_set(__DEFAULT, __ON_REMOVE, __update_major_chunks)
 __evolved_set(__DUPLICATE, __ON_INSERT, __update_major_chunks)
 __evolved_set(__DUPLICATE, __ON_REMOVE, __update_major_chunks)
 
+__evolved_set(__REALLOC, __ON_SET, __update_major_chunks)
+__evolved_set(__REALLOC, __ON_REMOVE, __update_major_chunks)
+
+__evolved_set(__COMPMOVE, __ON_SET, __update_major_chunks)
+__evolved_set(__COMPMOVE, __ON_REMOVE, __update_major_chunks)
+
 ---
 ---
 ---
@@ -6279,6 +6539,9 @@ __evolved_set(__INTERNAL, __NAME, 'INTERNAL')
 __evolved_set(__DEFAULT, __NAME, 'DEFAULT')
 __evolved_set(__DUPLICATE, __NAME, 'DUPLICATE')
 
+__evolved_set(__REALLOC, __NAME, 'REALLOC')
+__evolved_set(__COMPMOVE, __NAME, 'COMPMOVE')
+
 __evolved_set(__PREFAB, __NAME, 'PREFAB')
 __evolved_set(__DISABLED, __NAME, 'DISABLED')
 
@@ -6320,6 +6583,9 @@ __evolved_set(__INTERNAL, __INTERNAL)
 __evolved_set(__DEFAULT, __INTERNAL)
 __evolved_set(__DUPLICATE, __INTERNAL)
 
+__evolved_set(__REALLOC, __INTERNAL)
+__evolved_set(__COMPMOVE, __INTERNAL)
+
 __evolved_set(__PREFAB, __INTERNAL)
 __evolved_set(__DISABLED, __INTERNAL)
 
@@ -6683,6 +6949,9 @@ evolved.INTERNAL = __INTERNAL
 evolved.DEFAULT = __DEFAULT
 evolved.DUPLICATE = __DUPLICATE
 
+evolved.REALLOC = __REALLOC
+evolved.COMPMOVE = __COMPMOVE
+
 evolved.PREFAB = __PREFAB
 evolved.DISABLED = __DISABLED