diff --git a/sdk/include/vtx/common/readers/frame_reader/binary_loader.h b/sdk/include/vtx/common/readers/frame_reader/binary_loader.h index 1af07ac..0790f16 100644 --- a/sdk/include/vtx/common/readers/frame_reader/binary_loader.h +++ b/sdk/include/vtx/common/readers/frame_reader/binary_loader.h @@ -106,12 +106,12 @@ namespace VTX { return &prop_it->second; } - const std::vector* GetTypeMaxIndices(int32_t entity_type_id) const { + const StructSchemaCache* GetStructSizing(int32_t entity_type_id) const { auto struct_it = cache_->structs.find(entity_type_id); if (struct_it == cache_->structs.end()) { return nullptr; } - return &struct_it->second.type_max_indices; + return &struct_it->second; } /** diff --git a/sdk/include/vtx/common/readers/frame_reader/flatbuffer_loader.h b/sdk/include/vtx/common/readers/frame_reader/flatbuffer_loader.h index db0ff4d..6c24f50 100644 --- a/sdk/include/vtx/common/readers/frame_reader/flatbuffer_loader.h +++ b/sdk/include/vtx/common/readers/frame_reader/flatbuffer_loader.h @@ -46,11 +46,11 @@ namespace VTX { return &prop_it->second; } - const std::vector* GetTypeMaxIndices(int32_t entity_type_id) const { + const StructSchemaCache* GetStructSizing(int32_t entity_type_id) const { auto struct_it = cache_->structs.find(entity_type_id); if (struct_it == cache_->structs.end()) return nullptr; - return &struct_it->second.type_max_indices; + return &struct_it->second; } /** diff --git a/sdk/include/vtx/common/readers/frame_reader/loader_base.h b/sdk/include/vtx/common/readers/frame_reader/loader_base.h index 5ea6ae2..8a48f59 100644 --- a/sdk/include/vtx/common/readers/frame_reader/loader_base.h +++ b/sdk/include/vtx/common/readers/frame_reader/loader_base.h @@ -154,15 +154,16 @@ namespace VTX { protected: /** *The Derived loader MUST provide: - *const std::vector* GetTypeMaxIndices(int32_t entity_type_id) const; + *const StructSchemaCache* GetStructSizing(int32_t entity_type_id) const; */ void PrepareContainer(PropertyContainer& dest) { if (dest.entity_type_id < 0) { return; } - const std::vector* type_max_indices = AsDerived().GetTypeMaxIndices(dest.entity_type_id); - if (type_max_indices != nullptr) { - Helpers::ResizeContainerToMaxIndices(dest, *type_max_indices); + const StructSchemaCache* sizing = AsDerived().GetStructSizing(dest.entity_type_id); + if (sizing != nullptr) { + Helpers::ResizeContainerToMaxIndices(dest, sizing->type_max_indices, sizing->array_max_indices, + sizing->map_max_index); } } diff --git a/sdk/include/vtx/common/readers/frame_reader/native_loader.h b/sdk/include/vtx/common/readers/frame_reader/native_loader.h index 46f3d4b..57e0afb 100644 --- a/sdk/include/vtx/common/readers/frame_reader/native_loader.h +++ b/sdk/include/vtx/common/readers/frame_reader/native_loader.h @@ -54,11 +54,11 @@ namespace VTX { return &prop_it->second; } - const std::vector* GetTypeMaxIndices(int32_t entity_type_id) const { + const StructSchemaCache* GetStructSizing(int32_t entity_type_id) const { auto struct_it = cache_->structs.find(entity_type_id); if (struct_it == cache_->structs.end()) return nullptr; - return &struct_it->second.type_max_indices; + return &struct_it->second; } /** diff --git a/sdk/include/vtx/common/readers/frame_reader/protobuff_loader.h b/sdk/include/vtx/common/readers/frame_reader/protobuff_loader.h index a2d4c75..08add49 100644 --- a/sdk/include/vtx/common/readers/frame_reader/protobuff_loader.h +++ b/sdk/include/vtx/common/readers/frame_reader/protobuff_loader.h @@ -47,11 +47,11 @@ namespace VTX { return &prop_it->second; } - const std::vector* GetTypeMaxIndices(int32_t entity_type_id) const { + const StructSchemaCache* GetStructSizing(int32_t entity_type_id) const { auto struct_it = cache_->structs.find(entity_type_id); if (struct_it == cache_->structs.end()) return nullptr; - return &struct_it->second.type_max_indices; + return &struct_it->second; } /** diff --git a/sdk/include/vtx/common/readers/schema_reader/game_schema_types.h b/sdk/include/vtx/common/readers/schema_reader/game_schema_types.h index 4c36337..027056d 100644 --- a/sdk/include/vtx/common/readers/schema_reader/game_schema_types.h +++ b/sdk/include/vtx/common/readers/schema_reader/game_schema_types.h @@ -79,6 +79,13 @@ namespace VTX { */ std::vector type_max_indices; + /** @brief Max Array-container index (+1) per FieldType, i.e. how many array fields of each + * type the struct declares. Used to pre-create empty subarrays in the matching FlatArray. */ + std::vector array_max_indices; + + /** @brief Number of Map-container fields declared (all Struct-valued); pre-sizes map_properties. */ + int32_t map_max_index = 0; + /// @brief Fast lookup map: Property Name -> Field Pointer. std::unordered_map field_map; diff --git a/sdk/include/vtx/common/vtx_property_cache.h b/sdk/include/vtx/common/vtx_property_cache.h index 6f4fd15..bd95cda 100644 --- a/sdk/include/vtx/common/vtx_property_cache.h +++ b/sdk/include/vtx/common/vtx_property_cache.h @@ -85,6 +85,8 @@ namespace VTX { std::unordered_map names_by_lookup_key; std::vector property_order; std::vector type_max_indices; + std::vector array_max_indices; ///< Max Array-container index (+1) per FieldType. + int32_t map_max_index = 0; ///< Number of Map-container fields (pre-sizes map_properties). std::vector GetPropertiesInOrder() const { std::vector ordered_properties; diff --git a/sdk/include/vtx/common/vtx_types.h b/sdk/include/vtx/common/vtx_types.h index 37ff7b0..a2ff5ee 100644 --- a/sdk/include/vtx/common/vtx_types.h +++ b/sdk/include/vtx/common/vtx_types.h @@ -127,6 +127,12 @@ namespace VTX { /** Creates an empty subarray at the end (i.e., a new marker pointing to current Bucket.size()). */ void CreateEmptySubArray() { offsets.push_back(data.size()); } + /** Ensures at least @p Count subarrays exist, appending empty ones as needed (never shrinks). */ + void EnsureSubArrayCount(size_t Count) { + while (offsets.size() < Count) + offsets.push_back(data.size()); + } + /** Appends a subarray with given elements at the end. */ void AppendSubArray(std::span Items) { offsets.push_back(data.size()); diff --git a/sdk/include/vtx/common/vtx_types_helpers.h b/sdk/include/vtx/common/vtx_types_helpers.h index e0e6a57..207888d 100644 --- a/sdk/include/vtx/common/vtx_types_helpers.h +++ b/sdk/include/vtx/common/vtx_types_helpers.h @@ -18,13 +18,52 @@ using int64 = std::int64_t; namespace VTX { namespace Helpers { + // Grow-only: ensures each FlatArray holds at least the schema-declared number of + // (possibly empty) subarrays. Never shrinks; leaves scalars and maps untouched. + // Reused by ResizeContainerToMaxIndices (write/ingest) and by the reader to + // restore declared-but-empty array fields that were not serialized. + inline void EnsureDeclaredArrays(PropertyContainer& container, const std::vector& array_max_indices) { + auto GetNeeded = [&](FieldType type) -> int32_t { + size_t typeIdx = static_cast(type); + return (typeIdx < array_max_indices.size()) ? array_max_indices[typeIdx] : 0; + }; + + if (int32_t n = GetNeeded(FieldType::Bool)) + container.bool_arrays.EnsureSubArrayCount(n); + if (int32_t n = GetNeeded(FieldType::Int32)) + container.int32_arrays.EnsureSubArrayCount(n); + if (int32_t n = GetNeeded(FieldType::Int64)) + container.int64_arrays.EnsureSubArrayCount(n); + if (int32_t n = GetNeeded(FieldType::Float)) + container.float_arrays.EnsureSubArrayCount(n); + if (int32_t n = GetNeeded(FieldType::Double)) + container.double_arrays.EnsureSubArrayCount(n); + if (int32_t n = GetNeeded(FieldType::String)) + container.string_arrays.EnsureSubArrayCount(n); + + if (int32_t n = GetNeeded(FieldType::Vector)) + container.vector_arrays.EnsureSubArrayCount(n); + if (int32_t n = GetNeeded(FieldType::Quat)) + container.quat_arrays.EnsureSubArrayCount(n); + if (int32_t n = GetNeeded(FieldType::Transform)) + container.transform_arrays.EnsureSubArrayCount(n); + if (int32_t n = GetNeeded(FieldType::FloatRange)) + container.range_arrays.EnsureSubArrayCount(n); + + if (int32_t n = GetNeeded(FieldType::Struct)) + container.any_struct_arrays.EnsureSubArrayCount(n); + } + inline void ResizeContainerToMaxIndices(PropertyContainer& container, - const std::vector& type_max_indices) { + const std::vector& type_max_indices, + const std::vector& array_max_indices = {}, + int32_t map_max_index = 0) { auto GetNeeded = [&](FieldType type) -> int32_t { size_t typeIdx = static_cast(type); return (typeIdx < type_max_indices.size()) ? type_max_indices[typeIdx] : 0; }; + // --- Scalars: default-initialized slot per declared field. --- if (int32_t n = GetNeeded(FieldType::Bool)) container.bool_properties.resize(n); if (int32_t n = GetNeeded(FieldType::Int32)) @@ -49,10 +88,18 @@ namespace VTX { if (int32_t n = GetNeeded(FieldType::Struct)) container.any_struct_properties.resize(n); + + // --- Arrays: one empty subarray per declared array field of each type. --- + EnsureDeclaredArrays(container, array_max_indices); + + // --- Maps: all Struct-valued, a single contiguous index space. --- + if (map_max_index > 0) + container.map_properties.resize(static_cast(map_max_index)); } inline void PreparePropertyContainer(PropertyContainer& container, const SchemaStruct& schema) { - ResizeContainerToMaxIndices(container, schema.type_max_indices); + ResizeContainerToMaxIndices(container, schema.type_max_indices, schema.array_max_indices, + schema.map_max_index); } inline uint64_t CalculateContainerHash(const PropertyContainer& container) { diff --git a/sdk/include/vtx/reader/core/vtx_reader.h b/sdk/include/vtx/reader/core/vtx_reader.h index 4de4d89..279121e 100644 --- a/sdk/include/vtx/reader/core/vtx_reader.h +++ b/sdk/include/vtx/reader/core/vtx_reader.h @@ -19,6 +19,7 @@ #include "vtx_deserializer_service.h" #include "vtx/common/vtx_diagnostics.h" #include "vtx/common/vtx_types.h" +#include "vtx/common/vtx_types_helpers.h" #include "vtx/common/vtx_concepts.h" #include "vtx/reader/core/vtx_schema_adapter.h" #include "vtx/common/vtx_frame_accessor.h" diff --git a/sdk/src/vtx_common/src/vtx/common/readers/schema_reader/schema_registry.cpp b/sdk/src/vtx_common/src/vtx/common/readers/schema_reader/schema_registry.cpp index 38dd094..b9b200d 100644 --- a/sdk/src/vtx_common/src/vtx/common/readers/schema_reader/schema_registry.cpp +++ b/sdk/src/vtx_common/src/vtx/common/readers/schema_reader/schema_registry.cpp @@ -138,15 +138,22 @@ bool VTX::SchemaRegistry::LoadFromRawString(const std::string& raw_json) { for (const auto& field : current_struct.fields) { current_struct.field_map[field.name] = &field; - if (field.container_type == VTX::FieldContainerType::None) { - size_t typeIdx = static_cast(field.type_id); + const size_t typeIdx = static_cast(field.type_id); + if (field.container_type == VTX::FieldContainerType::None) { if (typeIdx >= current_struct.type_max_indices.size()) { current_struct.type_max_indices.resize(typeIdx + 1, 0); } - current_struct.type_max_indices[typeIdx] = std::max(current_struct.type_max_indices[typeIdx], field.index + 1); + } else if (field.container_type == VTX::FieldContainerType::Array) { + if (typeIdx >= current_struct.array_max_indices.size()) { + current_struct.array_max_indices.resize(typeIdx + 1, 0); + } + current_struct.array_max_indices[typeIdx] = + std::max(current_struct.array_max_indices[typeIdx], field.index + 1); + } else if (field.container_type == VTX::FieldContainerType::Map) { + current_struct.map_max_index = std::max(current_struct.map_max_index, field.index + 1); } } } @@ -165,6 +172,8 @@ bool VTX::SchemaRegistry::LoadFromRawString(const std::string& raw_json) { auto& struct_cache = property_cache_.structs[type_id]; struct_cache.name = struct_name; struct_cache.type_max_indices = struct_def.type_max_indices; + struct_cache.array_max_indices = struct_def.array_max_indices; + struct_cache.map_max_index = struct_def.map_max_index; for (const auto& field : struct_def.fields) { if (field.type_id != VTX::FieldType::None) { diff --git a/tests/common/test_native_loader.cpp b/tests/common/test_native_loader.cpp index 17c5f8e..c6fb069 100644 --- a/tests/common/test_native_loader.cpp +++ b/tests/common/test_native_loader.cpp @@ -11,7 +11,10 @@ #include #include "vtx/common/adapters/native/struct_mapping.h" +#include "vtx/common/readers/frame_reader/binary_loader.h" +#include "vtx/common/readers/frame_reader/flatbuffer_loader.h" #include "vtx/common/readers/frame_reader/native_loader.h" +#include "vtx/common/readers/frame_reader/protobuff_loader.h" #include "vtx/common/readers/schema_reader/schema_registry.h" #include "vtx/common/vtx_property_cache.h" #include "vtx/common/vtx_types.h" @@ -88,6 +91,21 @@ namespace vtx_native_loader_test { // Rotation (Quat[0]) and IsAlive (Bool[0]) are not mapped at all. }; + // ----------------------------------------------------------------- + // Setup 4: a struct that maps ONLY scalar fields of a schema that + // also declares array + map fields. Used to prove the loader + // pre-creates the declared-but-unpopulated array subarrays and map + // slots (symmetric with scalar pre-sizing) via PrepareContainer -> + // GetStructSizing -> ResizeContainerToMaxIndices. + // ----------------------------------------------------------------- + + struct HeroScalarOnly { + std::string unique_id; // String[0] (Name = String[1] left unmapped) + int score = 0; // Int32[0] + // Cooldowns (float[]), Tags/Names (string[]), Inventory (struct[]) and + // AmmoByWeapon (struct map) are all declared in the schema but unmapped. + }; + } // namespace vtx_native_loader_test // StructMapping specializations live in namespace VTX (qualified form). @@ -127,10 +145,45 @@ struct VTX::StructMapping { } }; +template <> +struct VTX::StructMapping { + static constexpr auto GetFields() { + using P = vtx_native_loader_test::HeroScalarOnly; + return std::make_tuple(MakeStructField("UniqueID", &P::unique_id), MakeStructField("Score", &P::score)); + } +}; + namespace { std::string SchemaPath() { return VtxTest::FixturePath("test_schema.json"); } + + // "Hero" declares scalar fields plus one float array (Cooldowns), two string + // arrays (Tags, Names), one struct array (Inventory) and one struct map + // (AmmoByWeapon). HeroScalarOnly maps only the scalars, so the array/map + // fields exercise the loader's array/map pre-sizing path. + constexpr const char* kHeroSchema = R"({ + "version": "1.0.0", + "buckets": ["entity"], + "property_mapping": [ + { "struct": "Item", "values": [ + {"name":"Id","typeId":"Int32","containerType":"None","keyId":"None","structType":"","meta":{"defaultValue":"0","fixedArrayDim":1}} + ]}, + { "struct": "Ammo", "values": [ + {"name":"Count","typeId":"Int32","containerType":"None","keyId":"None","structType":"","meta":{"defaultValue":"0","fixedArrayDim":1}} + ]}, + { "struct": "Hero", "values": [ + {"name":"UniqueID","typeId":"String","containerType":"None","keyId":"None","structType":"","meta":{"defaultValue":"","fixedArrayDim":1}}, + {"name":"Name","typeId":"String","containerType":"None","keyId":"None","structType":"","meta":{"defaultValue":"","fixedArrayDim":1}}, + {"name":"Score","typeId":"Int32","containerType":"None","keyId":"None","structType":"","meta":{"defaultValue":"0","fixedArrayDim":1}}, + {"name":"Cooldowns","typeId":"Float","containerType":"Array","keyId":"None","structType":"","meta":{"defaultValue":"","fixedArrayDim":0}}, + {"name":"Tags","typeId":"String","containerType":"Array","keyId":"None","structType":"","meta":{"defaultValue":"","fixedArrayDim":0}}, + {"name":"Names","typeId":"String","containerType":"Array","keyId":"None","structType":"","meta":{"defaultValue":"","fixedArrayDim":0}}, + {"name":"Inventory","typeId":"Struct","containerType":"Array","keyId":"None","structType":"Item","meta":{"defaultValue":"","fixedArrayDim":0}}, + {"name":"AmmoByWeapon","typeId":"Struct","containerType":"Map","keyId":"String","structType":"Ammo","meta":{"defaultValue":"","fixedArrayDim":1}} + ]} + ] + })"; } // namespace // =================================================================== @@ -240,6 +293,81 @@ TEST(NativeLoader, PreSizesContainerToSchemaMaxOnPartialMapping) { EXPECT_NE(dest.content_hash, 0u); } +// The loader pre-creates declared-but-unpopulated ARRAY and MAP fields as +// empty slots, symmetric with scalar pre-sizing. Exercises the full loader glue: +// PrepareContainer -> GetStructSizing -> ResizeContainerToMaxIndices(array/map). +TEST(NativeLoader, PreSizesArraysAndMapsThroughLoader) { + VTX::SchemaRegistry schema; + ASSERT_TRUE(schema.LoadFromRawString(kHeroSchema)); + + VTX::GenericNativeLoader loader(schema.GetPropertyCache()); + + vtx_native_loader_test::HeroScalarOnly h {}; + h.unique_id = "hero_1"; + h.score = 7; + + VTX::PropertyContainer dest; + loader.Load(h, dest, "Hero"); + + EXPECT_GE(dest.entity_type_id, 0); + + // Scalars: mapped + pre-sized (UniqueID+Name -> 2 strings; Score -> 1 int32). + ASSERT_EQ(dest.string_properties.size(), 2u); + EXPECT_EQ(dest.string_properties[0], "hero_1"); + ASSERT_EQ(dest.int32_properties.size(), 1u); + EXPECT_EQ(dest.int32_properties[0], 7); + + // Arrays: declared but unpopulated -> present as empty subarrays. + ASSERT_EQ(dest.float_arrays.SubArrayCount(), 1u); // Cooldowns + EXPECT_TRUE(dest.float_arrays.GetSubArray(0).empty()); + ASSERT_EQ(dest.string_arrays.SubArrayCount(), 2u); // Tags, Names + EXPECT_TRUE(dest.string_arrays.GetSubArray(0).empty()); + EXPECT_TRUE(dest.string_arrays.GetSubArray(1).empty()); + ASSERT_EQ(dest.any_struct_arrays.SubArrayCount(), 1u); // Inventory + + // Array types with no declared field stay untouched. + EXPECT_EQ(dest.int32_arrays.SubArrayCount(), 0u); + + // Map: declared but unpopulated -> present as one empty slot. + ASSERT_EQ(dest.map_properties.size(), 1u); // AmmoByWeapon + EXPECT_TRUE(dest.map_properties[0].keys.empty()); +} + +// All four frame loaders must expose the same array/map sizing to the shared +// base PrepareContainer via GetStructSizing. PreSizesArraysAndMapsThroughLoader +// (above) pins the base PrepareContainer -> ResizeContainerToMaxIndices path via +// the native loader; this pins each loader's GetStructSizing hook so a break in +// any single loader's hook is caught. +TEST(FrameLoaders, GetStructSizingExposesArrayAndMapSizingForAllLoaders) { + VTX::SchemaRegistry schema; + ASSERT_TRUE(schema.LoadFromRawString(kHeroSchema)); + const auto& cache = schema.GetPropertyCache(); + + const int32_t heroId = schema.GetStructTypeId("Hero"); + ASSERT_GE(heroId, 0); + + const auto floatIdx = static_cast(VTX::FieldType::Float); + const auto stringIdx = static_cast(VTX::FieldType::String); + + auto expect_hero_sizing = [&](const VTX::StructSchemaCache* s) { + ASSERT_NE(s, nullptr); + ASSERT_GT(s->array_max_indices.size(), stringIdx); + EXPECT_EQ(s->array_max_indices[floatIdx], 1); // Cooldowns + EXPECT_EQ(s->array_max_indices[stringIdx], 2); // Tags, Names + EXPECT_EQ(s->map_max_index, 1); // AmmoByWeapon + }; + + VTX::GenericNativeLoader native(cache); + VTX::GenericFlatBufferLoader flatbuffer(cache); + VTX::GenericProtobufLoader protobuf(schema); // proto loader takes the registry, not the cache + VTX::GenericBinaryLoader binary(cache); + + expect_hero_sizing(native.GetStructSizing(heroId)); + expect_hero_sizing(flatbuffer.GetStructSizing(heroId)); + expect_hero_sizing(protobuf.GetStructSizing(heroId)); + expect_hero_sizing(binary.GetStructSizing(heroId)); +} + TEST(NativeLoader, ADLConversionFromCustomMathType) { VTX::SchemaRegistry schema; ASSERT_TRUE(schema.LoadFromJson(SchemaPath())); diff --git a/tests/common/test_schema_registry.cpp b/tests/common/test_schema_registry.cpp index da60118..d60e4b5 100644 --- a/tests/common/test_schema_registry.cpp +++ b/tests/common/test_schema_registry.cpp @@ -6,6 +6,8 @@ #include #include "vtx/common/readers/schema_reader/schema_registry.h" +#include "vtx/common/vtx_types.h" +#include "vtx/common/vtx_types_helpers.h" #include "util/test_fixtures.h" diff --git a/tests/writer/test_roundtrip.cpp b/tests/writer/test_roundtrip.cpp index e75eaba..9c8170e 100644 --- a/tests/writer/test_roundtrip.cpp +++ b/tests/writer/test_roundtrip.cpp @@ -12,6 +12,7 @@ #include "vtx/writer/core/vtx_writer_facade.h" #include "vtx/reader/core/vtx_reader_facade.h" #include "vtx/common/vtx_types.h" +#include "vtx/common/readers/schema_reader/schema_registry.h" #include "util/test_fixtures.h"