Feature/schema array map presizing

sdk/include/vtx/common/readers/frame_reader/binary_loader.h

@@ -106,12 +106,12 @@ namespace VTX {
             return &prop_it->second;
         }
 
-        const std::vector<int32_t>* GetTypeMaxIndices(int32_t entity_type_id) const {
+        const StructSchemaCache* GetStructSizing(int32_t entity_type_id) const {
             auto struct_it = cache_->structs.find(entity_type_id);
             if (struct_it == cache_->structs.end()) {
                 return nullptr;
             }
-            return &struct_it->second.type_max_indices;
+            return &struct_it->second;
         }
 
         /**

sdk/include/vtx/common/readers/frame_reader/flatbuffer_loader.h

@@ -46,11 +46,11 @@ namespace VTX {
             return &prop_it->second;
         }
 
-        const std::vector<int32_t>* GetTypeMaxIndices(int32_t entity_type_id) const {
+        const StructSchemaCache* GetStructSizing(int32_t entity_type_id) const {
             auto struct_it = cache_->structs.find(entity_type_id);
             if (struct_it == cache_->structs.end())
                 return nullptr;
-            return &struct_it->second.type_max_indices;
+            return &struct_it->second;
         }
 
         /**

sdk/include/vtx/common/readers/frame_reader/loader_base.h

@@ -154,15 +154,16 @@ namespace VTX {
     protected:
         /**
          *The Derived loader MUST provide:
-         *const std::vector<int32_t>* GetTypeMaxIndices(int32_t entity_type_id) const;
+         *const StructSchemaCache* GetStructSizing(int32_t entity_type_id) const;
          */
         void PrepareContainer(PropertyContainer& dest) {
             if (dest.entity_type_id < 0) {
                 return;
             }
-            const std::vector<int32_t>* type_max_indices = AsDerived().GetTypeMaxIndices(dest.entity_type_id);
-            if (type_max_indices != nullptr) {
-                Helpers::ResizeContainerToMaxIndices(dest, *type_max_indices);
+            const StructSchemaCache* sizing = AsDerived().GetStructSizing(dest.entity_type_id);
+            if (sizing != nullptr) {
+                Helpers::ResizeContainerToMaxIndices(dest, sizing->type_max_indices, sizing->array_max_indices,
+                                                     sizing->map_max_index);
             }
         }
 

sdk/include/vtx/common/readers/frame_reader/native_loader.h

@@ -54,11 +54,11 @@ namespace VTX {
             return &prop_it->second;
         }
 
-        const std::vector<int32_t>* GetTypeMaxIndices(int32_t entity_type_id) const {
+        const StructSchemaCache* GetStructSizing(int32_t entity_type_id) const {
             auto struct_it = cache_->structs.find(entity_type_id);
             if (struct_it == cache_->structs.end())
                 return nullptr;
-            return &struct_it->second.type_max_indices;
+            return &struct_it->second;
         }
 
         /**

sdk/include/vtx/common/readers/frame_reader/protobuff_loader.h

@@ -47,11 +47,11 @@ namespace VTX {
             return &prop_it->second;
         }
 
-        const std::vector<int32_t>* GetTypeMaxIndices(int32_t entity_type_id) const {
+        const StructSchemaCache* GetStructSizing(int32_t entity_type_id) const {
             auto struct_it = cache_->structs.find(entity_type_id);
             if (struct_it == cache_->structs.end())
                 return nullptr;
-            return &struct_it->second.type_max_indices;
+            return &struct_it->second;
         }
 
         /**

sdk/include/vtx/common/readers/schema_reader/game_schema_types.h

@@ -79,6 +79,13 @@ namespace VTX {
          */
         std::vector<int32_t> type_max_indices;
 
+        /** @brief Max Array-container index (+1) per FieldType, i.e. how many array fields of each
+         * type the struct declares. Used to pre-create empty subarrays in the matching FlatArray. */
+        std::vector<int32_t> array_max_indices;
+
+        /** @brief Number of Map-container fields declared (all Struct-valued); pre-sizes map_properties. */
+        int32_t map_max_index = 0;
+
         /// @brief Fast lookup map: Property Name -> Field Pointer.
         std::unordered_map<std::string, const SchemaField*> field_map;
 

sdk/include/vtx/common/vtx_property_cache.h

@@ -85,6 +85,8 @@ namespace VTX {
         std::unordered_map<uint64_t, std::string> names_by_lookup_key;
         std::vector<std::string> property_order;
         std::vector<int32_t> type_max_indices;
+        std::vector<int32_t> array_max_indices; ///< Max Array-container index (+1) per FieldType.
+        int32_t map_max_index = 0;              ///< Number of Map-container fields (pre-sizes map_properties).
 
         std::vector<OrderedPropertyView> GetPropertiesInOrder() const {
             std::vector<OrderedPropertyView> ordered_properties;

sdk/include/vtx/common/vtx_types.h

@@ -127,6 +127,12 @@ namespace VTX {
         /** Creates an empty subarray at the end (i.e., a new marker pointing to current Bucket.size()). */
         void CreateEmptySubArray() { offsets.push_back(data.size()); }
 
+        /** Ensures at least @p Count subarrays exist, appending empty ones as needed (never shrinks). */
+        void EnsureSubArrayCount(size_t Count) {
+            while (offsets.size() < Count)
+                offsets.push_back(data.size());
+        }
+
         /** Appends a subarray with given elements at the end. */
         void AppendSubArray(std::span<const T> Items) {
             offsets.push_back(data.size());

sdk/include/vtx/common/vtx_types_helpers.h

@@ -18,13 +18,52 @@ using int64 = std::int64_t;
 namespace VTX {
     namespace Helpers {
 
+        // Grow-only: ensures each FlatArray holds at least the schema-declared number of
+        // (possibly empty) subarrays. Never shrinks; leaves scalars and maps untouched.
+        // Reused by ResizeContainerToMaxIndices (write/ingest) and by the reader to
+        // restore declared-but-empty array fields that were not serialized.
+        inline void EnsureDeclaredArrays(PropertyContainer& container, const std::vector<int32_t>& array_max_indices) {
+            auto GetNeeded = [&](FieldType type) -> int32_t {
+                size_t typeIdx = static_cast<size_t>(type);
+                return (typeIdx < array_max_indices.size()) ? array_max_indices[typeIdx] : 0;
+            };
+
+            if (int32_t n = GetNeeded(FieldType::Bool))
+                container.bool_arrays.EnsureSubArrayCount(n);
+            if (int32_t n = GetNeeded(FieldType::Int32))
+                container.int32_arrays.EnsureSubArrayCount(n);
+            if (int32_t n = GetNeeded(FieldType::Int64))
+                container.int64_arrays.EnsureSubArrayCount(n);
+            if (int32_t n = GetNeeded(FieldType::Float))
+                container.float_arrays.EnsureSubArrayCount(n);
+            if (int32_t n = GetNeeded(FieldType::Double))
+                container.double_arrays.EnsureSubArrayCount(n);
+            if (int32_t n = GetNeeded(FieldType::String))
+                container.string_arrays.EnsureSubArrayCount(n);
+
+            if (int32_t n = GetNeeded(FieldType::Vector))
+                container.vector_arrays.EnsureSubArrayCount(n);
+            if (int32_t n = GetNeeded(FieldType::Quat))
+                container.quat_arrays.EnsureSubArrayCount(n);
+            if (int32_t n = GetNeeded(FieldType::Transform))
+                container.transform_arrays.EnsureSubArrayCount(n);
+            if (int32_t n = GetNeeded(FieldType::FloatRange))
+                container.range_arrays.EnsureSubArrayCount(n);
+
+            if (int32_t n = GetNeeded(FieldType::Struct))
+                container.any_struct_arrays.EnsureSubArrayCount(n);
+        }
+
         inline void ResizeContainerToMaxIndices(PropertyContainer& container,
-                                                const std::vector<int32_t>& type_max_indices) {
+                                                const std::vector<int32_t>& type_max_indices,
+                                                const std::vector<int32_t>& array_max_indices = {},
+                                                int32_t map_max_index = 0) {
             auto GetNeeded = [&](FieldType type) -> int32_t {
                 size_t typeIdx = static_cast<size_t>(type);
                 return (typeIdx < type_max_indices.size()) ? type_max_indices[typeIdx] : 0;
             };
 
+            // --- Scalars: default-initialized slot per declared field. ---
             if (int32_t n = GetNeeded(FieldType::Bool))
                 container.bool_properties.resize(n);
             if (int32_t n = GetNeeded(FieldType::Int32))
@@ -49,10 +88,18 @@ namespace VTX {
 
             if (int32_t n = GetNeeded(FieldType::Struct))
                 container.any_struct_properties.resize(n);
+
+            // --- Arrays: one empty subarray per declared array field of each type. ---
+            EnsureDeclaredArrays(container, array_max_indices);
+
+            // --- Maps: all Struct-valued, a single contiguous index space. ---
+            if (map_max_index > 0)
+                container.map_properties.resize(static_cast<size_t>(map_max_index));
         }
 
         inline void PreparePropertyContainer(PropertyContainer& container, const SchemaStruct& schema) {
-            ResizeContainerToMaxIndices(container, schema.type_max_indices);
+            ResizeContainerToMaxIndices(container, schema.type_max_indices, schema.array_max_indices,
+                                        schema.map_max_index);
         }
 
         inline uint64_t CalculateContainerHash(const PropertyContainer& container) {

sdk/include/vtx/reader/core/vtx_reader.h

@@ -19,6 +19,7 @@
 #include "vtx_deserializer_service.h"
 #include "vtx/common/vtx_diagnostics.h"
 #include "vtx/common/vtx_types.h"
+#include "vtx/common/vtx_types_helpers.h"
 #include "vtx/common/vtx_concepts.h"
 #include "vtx/reader/core/vtx_schema_adapter.h"
 #include "vtx/common/vtx_frame_accessor.h"

sdk/src/vtx_common/src/vtx/common/readers/schema_reader/schema_registry.cpp

@@ -138,15 +138,22 @@ bool VTX::SchemaRegistry::LoadFromRawString(const std::string& raw_json) {
         for (const auto& field : current_struct.fields) {
             current_struct.field_map[field.name] = &field;
 
-            if (field.container_type == VTX::FieldContainerType::None) {
-                size_t typeIdx = static_cast<size_t>(field.type_id);
+            const size_t typeIdx = static_cast<size_t>(field.type_id);
 
+            if (field.container_type == VTX::FieldContainerType::None) {
                 if (typeIdx >= current_struct.type_max_indices.size()) {
                     current_struct.type_max_indices.resize(typeIdx + 1, 0);
                 }
-
                 current_struct.type_max_indices[typeIdx] =
                     std::max(current_struct.type_max_indices[typeIdx], field.index + 1);
+            } else if (field.container_type == VTX::FieldContainerType::Array) {
+                if (typeIdx >= current_struct.array_max_indices.size()) {
+                    current_struct.array_max_indices.resize(typeIdx + 1, 0);
+                }
+                current_struct.array_max_indices[typeIdx] =
+                    std::max(current_struct.array_max_indices[typeIdx], field.index + 1);
+            } else if (field.container_type == VTX::FieldContainerType::Map) {
+                current_struct.map_max_index = std::max(current_struct.map_max_index, field.index + 1);
             }
         }
     }
@@ -165,6 +172,8 @@ bool VTX::SchemaRegistry::LoadFromRawString(const std::string& raw_json) {
         auto& struct_cache = property_cache_.structs[type_id];
         struct_cache.name = struct_name;
         struct_cache.type_max_indices = struct_def.type_max_indices;
+        struct_cache.array_max_indices = struct_def.array_max_indices;
+        struct_cache.map_max_index = struct_def.map_max_index;
 
         for (const auto& field : struct_def.fields) {
             if (field.type_id != VTX::FieldType::None) {

tests/common/test_native_loader.cpp

@@ -11,7 +11,10 @@
 #include <gtest/gtest.h>
 
 #include "vtx/common/adapters/native/struct_mapping.h"
+#include "vtx/common/readers/frame_reader/binary_loader.h"
+#include "vtx/common/readers/frame_reader/flatbuffer_loader.h"
 #include "vtx/common/readers/frame_reader/native_loader.h"
+#include "vtx/common/readers/frame_reader/protobuff_loader.h"
 #include "vtx/common/readers/schema_reader/schema_registry.h"
 #include "vtx/common/vtx_property_cache.h"
 #include "vtx/common/vtx_types.h"
@@ -88,6 +91,21 @@ namespace vtx_native_loader_test {
         // Rotation (Quat[0]) and IsAlive (Bool[0]) are not mapped at all.
     };
 
+    // -----------------------------------------------------------------
+    //  Setup 4: a struct that maps ONLY scalar fields of a schema that
+    //  also declares array + map fields. Used to prove the loader
+    //  pre-creates the declared-but-unpopulated array subarrays and map
+    //  slots (symmetric with scalar pre-sizing) via PrepareContainer ->
+    //  GetStructSizing -> ResizeContainerToMaxIndices.
+    // -----------------------------------------------------------------
+
+    struct HeroScalarOnly {
+        std::string unique_id; // String[0]  (Name = String[1] left unmapped)
+        int score = 0;         // Int32[0]
+        // Cooldowns (float[]), Tags/Names (string[]), Inventory (struct[]) and
+        // AmmoByWeapon (struct map) are all declared in the schema but unmapped.
+    };
+
 } // namespace vtx_native_loader_test
 
 // StructMapping specializations live in namespace VTX (qualified form).
@@ -127,10 +145,45 @@ struct VTX::StructMapping<vtx_native_loader_test::PartialPlayer> {
     }
 };
 
+template <>
+struct VTX::StructMapping<vtx_native_loader_test::HeroScalarOnly> {
+    static constexpr auto GetFields() {
+        using P = vtx_native_loader_test::HeroScalarOnly;
+        return std::make_tuple(MakeStructField("UniqueID", &P::unique_id), MakeStructField("Score", &P::score));
+    }
+};
+
 namespace {
     std::string SchemaPath() {
         return VtxTest::FixturePath("test_schema.json");
     }
+
+    // "Hero" declares scalar fields plus one float array (Cooldowns), two string
+    // arrays (Tags, Names), one struct array (Inventory) and one struct map
+    // (AmmoByWeapon). HeroScalarOnly maps only the scalars, so the array/map
+    // fields exercise the loader's array/map pre-sizing path.
+    constexpr const char* kHeroSchema = R"({
+        "version": "1.0.0",
+        "buckets": ["entity"],
+        "property_mapping": [
+            { "struct": "Item", "values": [
+                {"name":"Id","typeId":"Int32","containerType":"None","keyId":"None","structType":"","meta":{"defaultValue":"0","fixedArrayDim":1}}
+            ]},
+            { "struct": "Ammo", "values": [
+                {"name":"Count","typeId":"Int32","containerType":"None","keyId":"None","structType":"","meta":{"defaultValue":"0","fixedArrayDim":1}}
+            ]},
+            { "struct": "Hero", "values": [
+                {"name":"UniqueID","typeId":"String","containerType":"None","keyId":"None","structType":"","meta":{"defaultValue":"","fixedArrayDim":1}},
+                {"name":"Name","typeId":"String","containerType":"None","keyId":"None","structType":"","meta":{"defaultValue":"","fixedArrayDim":1}},
+                {"name":"Score","typeId":"Int32","containerType":"None","keyId":"None","structType":"","meta":{"defaultValue":"0","fixedArrayDim":1}},
+                {"name":"Cooldowns","typeId":"Float","containerType":"Array","keyId":"None","structType":"","meta":{"defaultValue":"","fixedArrayDim":0}},
+                {"name":"Tags","typeId":"String","containerType":"Array","keyId":"None","structType":"","meta":{"defaultValue":"","fixedArrayDim":0}},
+                {"name":"Names","typeId":"String","containerType":"Array","keyId":"None","structType":"","meta":{"defaultValue":"","fixedArrayDim":0}},
+                {"name":"Inventory","typeId":"Struct","containerType":"Array","keyId":"None","structType":"Item","meta":{"defaultValue":"","fixedArrayDim":0}},
+                {"name":"AmmoByWeapon","typeId":"Struct","containerType":"Map","keyId":"String","structType":"Ammo","meta":{"defaultValue":"","fixedArrayDim":1}}
+            ]}
+        ]
+    })";
 } // namespace
 
 // ===================================================================
@@ -240,6 +293,81 @@ TEST(NativeLoader, PreSizesContainerToSchemaMaxOnPartialMapping) {
     EXPECT_NE(dest.content_hash, 0u);
 }
 
+// The loader pre-creates declared-but-unpopulated ARRAY and MAP fields as
+// empty slots, symmetric with scalar pre-sizing. Exercises the full loader glue:
+// PrepareContainer -> GetStructSizing -> ResizeContainerToMaxIndices(array/map).
+TEST(NativeLoader, PreSizesArraysAndMapsThroughLoader) {
+    VTX::SchemaRegistry schema;
+    ASSERT_TRUE(schema.LoadFromRawString(kHeroSchema));
+
+    VTX::GenericNativeLoader loader(schema.GetPropertyCache());
+
+    vtx_native_loader_test::HeroScalarOnly h {};
+    h.unique_id = "hero_1";
+    h.score = 7;
+
+    VTX::PropertyContainer dest;
+    loader.Load(h, dest, "Hero");
+
+    EXPECT_GE(dest.entity_type_id, 0);
+
+    // Scalars: mapped + pre-sized (UniqueID+Name -> 2 strings; Score -> 1 int32).
+    ASSERT_EQ(dest.string_properties.size(), 2u);
+    EXPECT_EQ(dest.string_properties[0], "hero_1");
+    ASSERT_EQ(dest.int32_properties.size(), 1u);
+    EXPECT_EQ(dest.int32_properties[0], 7);
+
+    // Arrays: declared but unpopulated -> present as empty subarrays.
+    ASSERT_EQ(dest.float_arrays.SubArrayCount(), 1u); // Cooldowns
+    EXPECT_TRUE(dest.float_arrays.GetSubArray(0).empty());
+    ASSERT_EQ(dest.string_arrays.SubArrayCount(), 2u); // Tags, Names
+    EXPECT_TRUE(dest.string_arrays.GetSubArray(0).empty());
+    EXPECT_TRUE(dest.string_arrays.GetSubArray(1).empty());
+    ASSERT_EQ(dest.any_struct_arrays.SubArrayCount(), 1u); // Inventory
+
+    // Array types with no declared field stay untouched.
+    EXPECT_EQ(dest.int32_arrays.SubArrayCount(), 0u);
+
+    // Map: declared but unpopulated -> present as one empty slot.
+    ASSERT_EQ(dest.map_properties.size(), 1u); // AmmoByWeapon
+    EXPECT_TRUE(dest.map_properties[0].keys.empty());
+}
+
+// All four frame loaders must expose the same array/map sizing to the shared
+// base PrepareContainer via GetStructSizing. PreSizesArraysAndMapsThroughLoader
+// (above) pins the base PrepareContainer -> ResizeContainerToMaxIndices path via
+// the native loader; this pins each loader's GetStructSizing hook so a break in
+// any single loader's hook is caught.
+TEST(FrameLoaders, GetStructSizingExposesArrayAndMapSizingForAllLoaders) {
+    VTX::SchemaRegistry schema;
+    ASSERT_TRUE(schema.LoadFromRawString(kHeroSchema));
+    const auto& cache = schema.GetPropertyCache();
+
+    const int32_t heroId = schema.GetStructTypeId("Hero");
+    ASSERT_GE(heroId, 0);
+
+    const auto floatIdx = static_cast<size_t>(VTX::FieldType::Float);
+    const auto stringIdx = static_cast<size_t>(VTX::FieldType::String);
+
+    auto expect_hero_sizing = [&](const VTX::StructSchemaCache* s) {
+        ASSERT_NE(s, nullptr);
+        ASSERT_GT(s->array_max_indices.size(), stringIdx);
+        EXPECT_EQ(s->array_max_indices[floatIdx], 1);  // Cooldowns
+        EXPECT_EQ(s->array_max_indices[stringIdx], 2); // Tags, Names
+        EXPECT_EQ(s->map_max_index, 1);                // AmmoByWeapon
+    };
+
+    VTX::GenericNativeLoader native(cache);
+    VTX::GenericFlatBufferLoader flatbuffer(cache);
+    VTX::GenericProtobufLoader protobuf(schema); // proto loader takes the registry, not the cache
+    VTX::GenericBinaryLoader binary(cache);
+
+    expect_hero_sizing(native.GetStructSizing(heroId));
+    expect_hero_sizing(flatbuffer.GetStructSizing(heroId));
+    expect_hero_sizing(protobuf.GetStructSizing(heroId));
+    expect_hero_sizing(binary.GetStructSizing(heroId));
+}
+
 TEST(NativeLoader, ADLConversionFromCustomMathType) {
     VTX::SchemaRegistry schema;
     ASSERT_TRUE(schema.LoadFromJson(SchemaPath()));