From c816666f531bf87183f7ce59f437a480fa6e7cd4 Mon Sep 17 00:00:00 2001 From: Anton Borisov Date: Sun, 7 Jun 2026 17:02:33 +0100 Subject: [PATCH] [c++] Add MAP/ROW types, unify complex types under Value handle --- bindings/cpp/include/fluss.hpp | 379 ++-- bindings/cpp/src/admin.cpp | 33 +- bindings/cpp/src/ffi_converter.hpp | 31 +- bindings/cpp/src/lib.rs | 1744 ++++++++--------- bindings/cpp/src/table.cpp | 556 +++--- bindings/cpp/src/type_lowering.hpp | 144 ++ bindings/cpp/src/types.rs | 298 ++- bindings/cpp/test/test_kv_table.cpp | 814 +++++++- bindings/cpp/test/test_log_table.cpp | 328 +++- website/docs/user-guide/cpp/api-reference.md | 129 +- website/docs/user-guide/cpp/data-types.md | 124 +- .../user-guide/cpp/example/prefix-lookup.md | 2 + 12 files changed, 2847 insertions(+), 1735 deletions(-) create mode 100644 bindings/cpp/src/type_lowering.hpp diff --git a/bindings/cpp/include/fluss.hpp b/bindings/cpp/include/fluss.hpp index c27b039b..1b5b1ba0 100644 --- a/bindings/cpp/include/fluss.hpp +++ b/bindings/cpp/include/fluss.hpp @@ -34,6 +34,8 @@ // Forward declare Arrow classes to avoid including heavy Arrow headers in header namespace arrow { class RecordBatch; +class Schema; +class DataType; } namespace fluss { @@ -53,7 +55,8 @@ struct GenericRowInner; struct LookupResultInner; struct PrefixLookupResultInner; struct ArrayWriterInner; -struct ArrayViewInner; +struct MapWriterInner; +struct ValueInner; } // namespace ffi /// Named constants for Fluss API error codes. @@ -281,6 +284,8 @@ enum class TypeId { Char = 15, Binary = 16, Array = 17, + Map = 18, + Row = 19, }; class DataType { @@ -319,6 +324,10 @@ class DataType { dt.element_type_ = std::make_shared(std::move(element)); return dt; } + /// Constructs a `MAP` type. Either side may itself be complex. + static DataType Map(DataType key, DataType value); + /// Constructs a `ROW` type from `{name, type}` fields. + static DataType Row(std::vector> fields); TypeId id() const { return id_; } int32_t precision() const { return precision_; } @@ -328,11 +337,22 @@ class DataType { /// types. The returned pointer is valid as long as this DataType (or a /// copy holding the same shared element) is alive. const DataType* element_type() const { return element_type_.get(); } + /// MAP key / value types. Return `nullptr` for non-MAP types. + const DataType* key_type() const { return key_type_.get(); } + const DataType* value_type() const { return value_type_.get(); } + /// ROW fields (empty for non-ROW types). + size_t field_count() const { return row_field_types_.size(); } + const std::string& field_name(size_t i) const { return row_field_names_.at(i); } + const DataType* field_type(size_t i) const { return row_field_types_.at(i).get(); } /// Returns a copy of this DataType with nullable set to false. DataType NotNull() const { DataType dt(id_, precision_, scale_, false); dt.element_type_ = element_type_; + dt.key_type_ = key_type_; + dt.value_type_ = value_type_; + dt.row_field_names_ = row_field_names_; + dt.row_field_types_ = row_field_types_; return dt; } @@ -342,8 +362,30 @@ class DataType { int32_t scale_{0}; bool nullable_{true}; std::shared_ptr element_type_; + std::shared_ptr key_type_; + std::shared_ptr value_type_; + std::vector row_field_names_; + std::vector> row_field_types_; }; +inline DataType DataType::Map(DataType key, DataType value) { + DataType dt(TypeId::Map, 0, 0); + dt.key_type_ = std::make_shared(std::move(key)); + dt.value_type_ = std::make_shared(std::move(value)); + return dt; +} + +inline DataType DataType::Row(std::vector> fields) { + DataType dt(TypeId::Row, 0, 0); + dt.row_field_names_.reserve(fields.size()); + dt.row_field_types_.reserve(fields.size()); + for (auto& f : fields) { + dt.row_field_names_.push_back(std::move(f.first)); + dt.row_field_types_.push_back(std::make_shared(std::move(f.second))); + } + return dt; +} + constexpr int64_t EARLIEST_OFFSET = -2; enum class OffsetType { @@ -391,6 +433,10 @@ struct Column { struct Schema { std::vector columns; std::vector primary_keys; + /// When set (via FromArrow), the table's column types come from this Arrow + /// schema instead of `columns` — the only way to express nested MAP/ROW + /// columns. `columns` stays empty in that case. + std::shared_ptr arrow_schema; class Builder { public: @@ -412,6 +458,15 @@ struct Schema { }; static Builder NewBuilder() { return Builder(); } + + /// Build a Schema whose column types come from an Arrow schema. Use this + /// for tables with nested MAP/ROW columns (`arrow::map()`, `arrow::struct_()`); + /// `Admin::CreateTable` routes Arrow-backed schemas through the C Data + /// Interface automatically. + static Schema FromArrow(std::shared_ptr arrow_schema, + std::vector primary_keys = {}) { + return Schema{{}, std::move(primary_keys), std::move(arrow_schema)}; + } }; struct TableDescriptor { @@ -521,10 +576,11 @@ inline size_t ResolveColumn(const ColumnMap& map, const std::string& name) { return it->second.index; } -// Forward declaration so NamedGetters can declare GetArrayView(...) even -// though the concrete class is defined further down. } // namespace detail -class ArrayView; +class Value; +class GenericRow; +class ArrayWriter; +class MapWriter; namespace detail { /// CRTP mixin that adds name-based getters to any class with index-based getters. @@ -552,51 +608,6 @@ struct NamedGetters { std::string GetDecimalString(const std::string& n) const { return Self().GetDecimalString(Self().Resolve(n)); } - size_t GetArraySize(const std::string& n) const { - return Self().GetArraySize(Self().Resolve(n)); - } - TypeId GetArrayElementType(const std::string& n) const { - return Self().GetArrayElementType(Self().Resolve(n)); - } - bool IsArrayElementNull(const std::string& n, size_t element) const { - return Self().IsArrayElementNull(Self().Resolve(n), element); - } - bool GetArrayBool(const std::string& n, size_t element) const { - return Self().GetArrayBool(Self().Resolve(n), element); - } - int32_t GetArrayInt32(const std::string& n, size_t element) const { - return Self().GetArrayInt32(Self().Resolve(n), element); - } - int64_t GetArrayInt64(const std::string& n, size_t element) const { - return Self().GetArrayInt64(Self().Resolve(n), element); - } - float GetArrayFloat32(const std::string& n, size_t element) const { - return Self().GetArrayFloat32(Self().Resolve(n), element); - } - double GetArrayFloat64(const std::string& n, size_t element) const { - return Self().GetArrayFloat64(Self().Resolve(n), element); - } - std::string GetArrayString(const std::string& n, size_t element) const { - return Self().GetArrayString(Self().Resolve(n), element); - } - std::vector GetArrayBytes(const std::string& n, size_t element) const { - return Self().GetArrayBytes(Self().Resolve(n), element); - } - fluss::Date GetArrayDate(const std::string& n, size_t element) const { - return Self().GetArrayDate(Self().Resolve(n), element); - } - fluss::Time GetArrayTime(const std::string& n, size_t element) const { - return Self().GetArrayTime(Self().Resolve(n), element); - } - fluss::Timestamp GetArrayTimestamp(const std::string& n, size_t element) const { - return Self().GetArrayTimestamp(Self().Resolve(n), element); - } - std::string GetArrayDecimalString(const std::string& n, size_t element) const { - return Self().GetArrayDecimalString(Self().Resolve(n), element); - } - // Definition appears below the ArrayView class; return-by-value requires - // the complete type so we cannot inline the body here. - ArrayView GetArrayView(const std::string& n) const; private: const Derived& Self() const { return static_cast(*this); } @@ -627,60 +638,60 @@ struct PrefixData { }; } // namespace detail -/** - * @brief Read-only view over a FlussArray column value. - * - * Obtained from RowView::GetArrayView() / LookupResult::GetArrayView(), and - * recursively from ArrayView::GetArray() for nested `ARRAY>` - * columns. Owns an opaque Rust handle (FlussArray + element DataType) that - * is released on destruction. Move-only. - */ -class ArrayView { +/// One recursive handle for reading a complex (ARRAY / MAP / ROW) column value. +/// `Navigate` with At/KeyAt/ValueAt/Field — each returns a child `Value`; read a +/// leaf with the Get* methods (no index — the handle points at one value). +/// Obtained from LookupResult::GetValue() / RowView::GetValue(). Move-only; +/// owns an opaque Rust handle released on destruction. +class Value { public: - ~ArrayView() noexcept; - - ArrayView(const ArrayView&) = delete; - ArrayView& operator=(const ArrayView&) = delete; - ArrayView(ArrayView&& other) noexcept; - ArrayView& operator=(ArrayView&& other) noexcept; - - size_t Size() const noexcept; - TypeId ElementType() const noexcept; - bool IsNull(size_t element) const; - - bool GetBool(size_t element) const; - int32_t GetInt32(size_t element) const; - int64_t GetInt64(size_t element) const; - float GetFloat32(size_t element) const; - double GetFloat64(size_t element) const; - std::string GetString(size_t element) const; - std::vector GetBytes(size_t element) const; - fluss::Date GetDate(size_t element) const; - fluss::Time GetTime(size_t element) const; - fluss::Timestamp GetTimestampNtz(size_t element) const; - fluss::Timestamp GetTimestampLtz(size_t element) const; - std::string GetDecimalString(size_t element) const; - ArrayView GetArray(size_t element) const; + ~Value() noexcept; + Value(const Value&) = delete; + Value& operator=(const Value&) = delete; + Value(Value&& other) noexcept; + Value& operator=(Value&& other) noexcept; + + TypeId Type() const noexcept; + bool IsNull() const noexcept; + + // ── Leaf reads ── + bool GetBool() const; + int32_t GetInt32() const; + int64_t GetInt64() const; + float GetFloat32() const; + double GetFloat64() const; + std::string GetString() const; + std::vector GetBytes() const; + fluss::Date GetDate() const; + fluss::Time GetTime() const; + fluss::Timestamp GetTimestamp() const; + std::string GetDecimalString() const; + + // ── Navigation ── + size_t Size() const; // ARRAY / MAP entry count + size_t FieldCount() const; // ROW + Value At(size_t i) const; // ARRAY element + Value KeyAt(size_t i) const; // MAP key + Value ValueAt(size_t i) const; // MAP value + Value Field(size_t i) const; // ROW field + Value Field(const std::string& name) const; // ROW field by name private: - friend class RowView; friend class LookupResult; + friend class RowView; friend class PrefixRowView; - explicit ArrayView(ffi::ArrayViewInner* inner) : inner_(inner) {} + explicit Value(ffi::ValueInner* inner) : inner_(inner) {} void Destroy() noexcept; - ffi::ArrayViewInner* inner_{nullptr}; + ffi::ValueInner* inner_{nullptr}; }; -namespace detail { -template -inline ArrayView NamedGetters::GetArrayView(const std::string& n) const { - return Self().GetArrayView(Self().Resolve(n)); -} -} // namespace detail - class ArrayWriter { public: ArrayWriter(size_t size, DataType element_type); + /// Builds an array whose element is a ROW / MAP (which DataType can't + /// express). Pass the element type as an Arrow type, e.g. + /// `arrow::struct_({...})` or `arrow::map(...)`. + ArrayWriter(size_t size, std::shared_ptr element_type); ~ArrayWriter() noexcept; ArrayWriter(const ArrayWriter&) = delete; @@ -705,14 +716,81 @@ class ArrayWriter { void SetTimestampLtz(size_t idx, fluss::Timestamp ts); void SetDecimal(size_t idx, const std::string& value); void SetArray(size_t idx, ArrayWriter&& nested); + /// Sets a ROW / MAP element. The nested row/map is consumed (moved-from). + void SetRow(size_t idx, GenericRow&& row); + void SetMap(size_t idx, MapWriter&& map); private: friend class GenericRow; + friend class MapWriter; void Destroy() noexcept; ffi::ArrayWriterInner* inner_{nullptr}; DataType element_type_; }; +/// Builder for a MAP column value. Construct with the key/value types, then for +/// each entry set the key and value and call Commit(). Keys cannot be null. +/// Move-only; consumed by GenericRow::SetMap / Set(name, MapWriter&&). +class MapWriter { + public: + MapWriter(size_t capacity, DataType key_type, DataType value_type); + /// Builds a map whose key/value is a ROW / MAP / ARRAY (which DataType + /// can't express). Pass the key and value types as Arrow types. + MapWriter(size_t capacity, std::shared_ptr key_type, + std::shared_ptr value_type); + ~MapWriter() noexcept; + + MapWriter(const MapWriter&) = delete; + MapWriter& operator=(const MapWriter&) = delete; + MapWriter(MapWriter&& other) noexcept; + MapWriter& operator=(MapWriter&& other) noexcept; + + bool Available() const; + + // ── Key setters ────────────────────────────────────────────────────── + void SetKeyBool(bool k); + void SetKeyInt32(int32_t k); + void SetKeyInt64(int64_t k); + void SetKeyFloat32(float k); + void SetKeyFloat64(double k); + void SetKeyString(const std::string& k); + void SetKeyBytes(const std::vector& k); + void SetKeyDate(fluss::Date k); + void SetKeyTime(fluss::Time k); + /// NTZ vs LTZ is chosen from the map's declared key type. + void SetKeyTimestamp(fluss::Timestamp k); + void SetKeyDecimal(const std::string& k); + + // ── Value setters ────────────────────────────────────────────────────── + void SetValueNull(); + void SetValueBool(bool v); + void SetValueInt32(int32_t v); + void SetValueInt64(int64_t v); + void SetValueFloat32(float v); + void SetValueFloat64(double v); + void SetValueString(const std::string& v); + void SetValueBytes(const std::vector& v); + void SetValueDate(fluss::Date v); + void SetValueTime(fluss::Time v); + /// NTZ vs LTZ is chosen from the map's declared value type. + void SetValueTimestamp(fluss::Timestamp v); + void SetValueDecimal(const std::string& v); + // Compound values: the writer/row is consumed (moved-from). + void SetValueRow(GenericRow&& v); + void SetValueMap(MapWriter&& v); + void SetValueArray(ArrayWriter&& v); + + void Commit(); + + private: + friend class GenericRow; + friend class ArrayWriter; + void Destroy() noexcept; + ffi::MapWriterInner* inner_{nullptr}; + DataType key_type_; + DataType value_type_; +}; + class GenericRow { public: GenericRow(); @@ -742,6 +820,10 @@ class GenericRow { void SetTimestampLtz(size_t idx, fluss::Timestamp ts); void SetDecimal(size_t idx, const std::string& value); void SetArray(size_t idx, ArrayWriter&& writer); + void SetMap(size_t idx, MapWriter&& writer); + /// Sets a ROW-typed field from a nested row built with GenericRow. The + /// nested row is consumed (moved-from) by this call. + void SetRow(size_t idx, GenericRow&& nested); // ── Name-based setters (require schema — see Table::NewRow()) ─── void Set(const std::string& name, std::nullptr_t) { SetNull(Resolve(name)); } @@ -790,6 +872,8 @@ class GenericRow { } } void Set(const std::string& name, ArrayWriter&& writer) { SetArray(Resolve(name), std::move(writer)); } + void Set(const std::string& name, MapWriter&& writer) { SetMap(Resolve(name), std::move(writer)); } + void Set(const std::string& name, GenericRow&& nested) { SetRow(Resolve(name), std::move(nested)); } private: friend class Table; @@ -797,6 +881,8 @@ class GenericRow { friend class UpsertWriter; friend class Lookuper; friend class PrefixLookuper; + friend class ArrayWriter; + friend class MapWriter; using ColumnInfo = detail::ColumnInfo; using ColumnMap = detail::ColumnMap; @@ -850,25 +936,9 @@ class RowView : public detail::NamedGetters { bool IsDecimal(size_t idx) const; std::string GetDecimalString(size_t idx) const; - // ── Array getters ──────────────────────────────────────────────── - size_t GetArraySize(size_t idx) const; - TypeId GetArrayElementType(size_t idx) const; - bool IsArrayElementNull(size_t idx, size_t element) const; - bool GetArrayBool(size_t idx, size_t element) const; - int32_t GetArrayInt32(size_t idx, size_t element) const; - int64_t GetArrayInt64(size_t idx, size_t element) const; - float GetArrayFloat32(size_t idx, size_t element) const; - double GetArrayFloat64(size_t idx, size_t element) const; - std::string GetArrayString(size_t idx, size_t element) const; - std::vector GetArrayBytes(size_t idx, size_t element) const; - fluss::Date GetArrayDate(size_t idx, size_t element) const; - fluss::Time GetArrayTime(size_t idx, size_t element) const; - fluss::Timestamp GetArrayTimestamp(size_t idx, size_t element) const; - std::string GetArrayDecimalString(size_t idx, size_t element) const; - /// Returns an owning ArrayView over the array column at `idx`. ArrayView - /// supports nested arrays via ArrayView::GetArray(). Parity with Python's - /// recursive list return from `row.get_array(i)`. - ArrayView GetArrayView(size_t idx) const; + /// One recursive handle for any complex (ARRAY/MAP/ROW) column. + Value GetValue(size_t idx) const; + Value GetValue(const std::string& name) const; // Name-based getters inherited from detail::NamedGetters using detail::NamedGetters::IsNull; @@ -883,21 +953,6 @@ class RowView : public detail::NamedGetters { using detail::NamedGetters::GetTime; using detail::NamedGetters::GetTimestamp; using detail::NamedGetters::GetDecimalString; - using detail::NamedGetters::GetArraySize; - using detail::NamedGetters::GetArrayElementType; - using detail::NamedGetters::IsArrayElementNull; - using detail::NamedGetters::GetArrayBool; - using detail::NamedGetters::GetArrayInt32; - using detail::NamedGetters::GetArrayInt64; - using detail::NamedGetters::GetArrayFloat32; - using detail::NamedGetters::GetArrayFloat64; - using detail::NamedGetters::GetArrayString; - using detail::NamedGetters::GetArrayBytes; - using detail::NamedGetters::GetArrayDate; - using detail::NamedGetters::GetArrayTime; - using detail::NamedGetters::GetArrayTimestamp; - using detail::NamedGetters::GetArrayDecimalString; - using detail::NamedGetters::GetArrayView; private: size_t Resolve(const std::string& name) const { @@ -940,22 +995,9 @@ class PrefixRowView : public detail::NamedGetters { bool IsDecimal(size_t idx) const; std::string GetDecimalString(size_t idx) const; - // ── Array getters ──────────────────────────────────────────────── - size_t GetArraySize(size_t idx) const; - TypeId GetArrayElementType(size_t idx) const; - bool IsArrayElementNull(size_t idx, size_t element) const; - bool GetArrayBool(size_t idx, size_t element) const; - int32_t GetArrayInt32(size_t idx, size_t element) const; - int64_t GetArrayInt64(size_t idx, size_t element) const; - float GetArrayFloat32(size_t idx, size_t element) const; - double GetArrayFloat64(size_t idx, size_t element) const; - std::string GetArrayString(size_t idx, size_t element) const; - std::vector GetArrayBytes(size_t idx, size_t element) const; - fluss::Date GetArrayDate(size_t idx, size_t element) const; - fluss::Time GetArrayTime(size_t idx, size_t element) const; - fluss::Timestamp GetArrayTimestamp(size_t idx, size_t element) const; - std::string GetArrayDecimalString(size_t idx, size_t element) const; - ArrayView GetArrayView(size_t idx) const; + /// One recursive handle for any complex (ARRAY/MAP/ROW) column, by index or name. + Value GetValue(size_t idx) const; + Value GetValue(const std::string& name) const; // Name-based getters inherited from detail::NamedGetters using detail::NamedGetters::IsNull; @@ -970,21 +1012,6 @@ class PrefixRowView : public detail::NamedGetters { using detail::NamedGetters::GetTime; using detail::NamedGetters::GetTimestamp; using detail::NamedGetters::GetDecimalString; - using detail::NamedGetters::GetArraySize; - using detail::NamedGetters::GetArrayElementType; - using detail::NamedGetters::IsArrayElementNull; - using detail::NamedGetters::GetArrayBool; - using detail::NamedGetters::GetArrayInt32; - using detail::NamedGetters::GetArrayInt64; - using detail::NamedGetters::GetArrayFloat32; - using detail::NamedGetters::GetArrayFloat64; - using detail::NamedGetters::GetArrayString; - using detail::NamedGetters::GetArrayBytes; - using detail::NamedGetters::GetArrayDate; - using detail::NamedGetters::GetArrayTime; - using detail::NamedGetters::GetArrayTimestamp; - using detail::NamedGetters::GetArrayDecimalString; - using detail::NamedGetters::GetArrayView; private: size_t Resolve(const std::string& name) const { @@ -1279,23 +1306,9 @@ class LookupResult : public detail::NamedGetters { bool IsDecimal(size_t idx) const; std::string GetDecimalString(size_t idx) const; - // ── Array getters ──────────────────────────────────────────────── - size_t GetArraySize(size_t idx) const; - TypeId GetArrayElementType(size_t idx) const; - bool IsArrayElementNull(size_t idx, size_t element) const; - bool GetArrayBool(size_t idx, size_t element) const; - int32_t GetArrayInt32(size_t idx, size_t element) const; - int64_t GetArrayInt64(size_t idx, size_t element) const; - float GetArrayFloat32(size_t idx, size_t element) const; - double GetArrayFloat64(size_t idx, size_t element) const; - std::string GetArrayString(size_t idx, size_t element) const; - std::vector GetArrayBytes(size_t idx, size_t element) const; - fluss::Date GetArrayDate(size_t idx, size_t element) const; - fluss::Time GetArrayTime(size_t idx, size_t element) const; - fluss::Timestamp GetArrayTimestamp(size_t idx, size_t element) const; - std::string GetArrayDecimalString(size_t idx, size_t element) const; - /// See RowView::GetArrayView for semantics. Supports nested arrays. - ArrayView GetArrayView(size_t idx) const; + /// One recursive handle for any complex (ARRAY/MAP/ROW) column, by index or name. + Value GetValue(size_t idx) const; + Value GetValue(const std::string& name) const; // Name-based getters inherited from detail::NamedGetters using detail::NamedGetters::IsNull; @@ -1310,21 +1323,6 @@ class LookupResult : public detail::NamedGetters { using detail::NamedGetters::GetTime; using detail::NamedGetters::GetTimestamp; using detail::NamedGetters::GetDecimalString; - using detail::NamedGetters::GetArraySize; - using detail::NamedGetters::GetArrayElementType; - using detail::NamedGetters::IsArrayElementNull; - using detail::NamedGetters::GetArrayBool; - using detail::NamedGetters::GetArrayInt32; - using detail::NamedGetters::GetArrayInt64; - using detail::NamedGetters::GetArrayFloat32; - using detail::NamedGetters::GetArrayFloat64; - using detail::NamedGetters::GetArrayString; - using detail::NamedGetters::GetArrayBytes; - using detail::NamedGetters::GetArrayDate; - using detail::NamedGetters::GetArrayTime; - using detail::NamedGetters::GetArrayTimestamp; - using detail::NamedGetters::GetArrayDecimalString; - using detail::NamedGetters::GetArrayView; private: friend class Lookuper; @@ -1451,6 +1449,9 @@ class Admin { bool Available() const; + /// Creates a table. For nested MAP/ROW columns, build `descriptor`'s schema + /// via `Schema::FromArrow(...)` — `CreateTable` routes Arrow-backed schemas + /// through the C Data Interface automatically. Result CreateTable(const TablePath& table_path, const TableDescriptor& descriptor, bool ignore_if_exists = false); diff --git a/bindings/cpp/src/admin.cpp b/bindings/cpp/src/admin.cpp index a689c614..4ebc6f45 100644 --- a/bindings/cpp/src/admin.cpp +++ b/bindings/cpp/src/admin.cpp @@ -21,6 +21,8 @@ #include "fluss.hpp" #include "lib.rs.h" #include "rust/cxx.h" +#include "type_lowering.hpp" +#include #include namespace fluss { @@ -58,8 +60,37 @@ Result Admin::CreateTable(const TablePath& table_path, const TableDescriptor& de } auto ffi_path = utils::to_ffi_table_path(table_path); - auto ffi_desc = utils::to_ffi_table_descriptor(descriptor); + // A MAP/ROW column can't go through the flat FFI encoding, so the schema is + // sent over Arrow instead (explicit via FromArrow, or lowered from native + // columns). Rust derives the columns from it, so the flat columns are dropped + // here; primary keys / metadata still come from the descriptor. + std::shared_ptr arrow_schema = descriptor.schema.arrow_schema; + if (!arrow_schema) { + for (const auto& col : descriptor.schema.columns) { + if (detail::is_compound(col.data_type)) { + arrow_schema = detail::columns_to_arrow_schema(descriptor.schema.columns); + break; + } + } + } + + if (arrow_schema) { + TableDescriptor arrow_desc = descriptor; + arrow_desc.schema.columns.clear(); + auto ffi_desc = utils::to_ffi_table_descriptor(arrow_desc); + size_t schema_ptr = 0; + try { + schema_ptr = detail::export_arrow_schema(*arrow_schema); + } catch (const std::exception& e) { + return utils::make_client_error(e.what()); + } + auto ffi_result = + admin_->create_table_arrow(ffi_path, ffi_desc, schema_ptr, ignore_if_exists); + return utils::from_ffi_result(ffi_result); + } + + auto ffi_desc = utils::to_ffi_table_descriptor(descriptor); auto ffi_result = admin_->create_table(ffi_path, ffi_desc, ignore_if_exists); return utils::from_ffi_result(ffi_result); } diff --git a/bindings/cpp/src/ffi_converter.hpp b/bindings/cpp/src/ffi_converter.hpp index 47453d99..89a70182 100644 --- a/bindings/cpp/src/ffi_converter.hpp +++ b/bindings/cpp/src/ffi_converter.hpp @@ -253,18 +253,6 @@ inline ffi::FfiTableDescriptor to_ffi_table_descriptor(const TableDescriptor& de inline Column from_ffi_column(const ffi::FfiColumn& ffi_col) { auto type_id = static_cast(ffi_col.data_type); if (type_id == TypeId::Array) { - if (ffi_col.element_data_type == 0) { - throw std::runtime_error("Malformed ARRAY column '" + std::string(ffi_col.name) + - "': missing element_data_type"); - } - if (ffi_col.array_nesting < 0) { - throw std::runtime_error("Malformed ARRAY column '" + std::string(ffi_col.name) + - "': array_nesting must be non-negative"); - } - if (ffi_col.element_data_type == static_cast(TypeId::Array)) { - throw std::runtime_error("Malformed ARRAY column '" + std::string(ffi_col.name) + - "': leaf element_data_type cannot be ARRAY"); - } auto is_supported_leaf_type = [](int32_t leaf_type) { switch (static_cast(leaf_type)) { case TypeId::Boolean: @@ -288,6 +276,25 @@ inline Column from_ffi_column(const ffi::FfiColumn& ffi_col) { return false; } }; + // ROW/MAP element schema can't pass through the flat FFI column; give the + // array a non-null element of the right kind so element_type() is safe to deref. + auto element_id = static_cast(ffi_col.element_data_type); + if (element_id == TypeId::Map || element_id == TypeId::Row) { + return Column{std::string(ffi_col.name), DataType::Array(DataType(element_id)), + std::string(ffi_col.comment)}; + } + if (ffi_col.element_data_type == 0) { + throw std::runtime_error("Malformed ARRAY column '" + std::string(ffi_col.name) + + "': missing element_data_type"); + } + if (ffi_col.array_nesting < 0) { + throw std::runtime_error("Malformed ARRAY column '" + std::string(ffi_col.name) + + "': array_nesting must be non-negative"); + } + if (ffi_col.element_data_type == static_cast(TypeId::Array)) { + throw std::runtime_error("Malformed ARRAY column '" + std::string(ffi_col.name) + + "': leaf element_data_type cannot be ARRAY"); + } if (!is_supported_leaf_type(ffi_col.element_data_type)) { throw std::runtime_error("Malformed ARRAY column '" + std::string(ffi_col.name) + "': unsupported leaf element_data_type " + diff --git a/bindings/cpp/src/lib.rs b/bindings/cpp/src/lib.rs index b5417ae4..e5aac2a1 100644 --- a/bindings/cpp/src/lib.rs +++ b/bindings/cpp/src/lib.rs @@ -17,17 +17,20 @@ mod types; +use std::collections::HashMap; use std::str::FromStr; use std::sync::{Arc, LazyLock}; use std::time::Duration; +use arrow::ffi::{FFI_ArrowArray, FFI_ArrowSchema}; use fluss as fcore; use fluss::PartitionId; use fluss::client::PrefixKeyLookuper; use fluss::error::Error; -use fluss::metadata::{Column, TableInfo}; +use fluss::metadata::{Column, DataType, TableInfo}; use fluss::row::{Datum, GenericRow}; use fluss::rpc::FlussError as CoreFlussError; +use fluss::rpc::message::OffsetSpec; static RUNTIME: LazyLock = LazyLock::new(|| { tokio::runtime::Builder::new_multi_thread() @@ -296,7 +299,8 @@ mod ffi { type LookupResultInner; type PrefixLookupResultInner; type ArrayWriterInner; - type ArrayViewInner; + type MapWriterInner; + type ValueInner; // Connection fn new_connection(config: &FfiConfig) -> FfiPtrResult; @@ -312,6 +316,17 @@ mod ffi { descriptor: &FfiTableDescriptor, ignore_if_exists: bool, ) -> FfiResult; + // Create a table whose columns come from an Arrow schema (C Data + // Interface pointer), supporting nested MAP/ROW columns. `descriptor` + // supplies primary keys + table-level metadata; its `schema.columns` + // are ignored in favour of `arrow_schema_ptr`. + fn create_table_arrow( + self: &Admin, + table_path: &FfiTablePath, + descriptor: &FfiTableDescriptor, + arrow_schema_ptr: usize, + ignore_if_exists: bool, + ) -> FfiResult; fn drop_table( self: &Admin, table_path: &FfiTablePath, @@ -407,6 +422,16 @@ mod ffi { idx: usize, writer: &mut ArrayWriterInner, ) -> Result<()>; + fn gr_set_map( + self: &mut GenericRowInner, + idx: usize, + writer: &mut MapWriterInner, + ) -> Result<()>; + fn gr_set_row( + self: &mut GenericRowInner, + idx: usize, + row: &mut GenericRowInner, + ) -> Result<()>; // ArrayWriterInner — opaque array builder for writes fn new_array_writer( @@ -416,6 +441,11 @@ mod ffi { scale: u32, array_nesting: u32, ) -> Result>; + // ROW/MAP element types: a one-field Arrow schema (the flat encoding can't carry them). + fn new_array_writer_arrow( + size: usize, + element_schema_ptr: usize, + ) -> Result>; fn aw_size(self: &ArrayWriterInner) -> usize; fn aw_set_null(self: &mut ArrayWriterInner, idx: usize) -> Result<()>; fn aw_set_bool(self: &mut ArrayWriterInner, idx: usize, val: bool) -> Result<()>; @@ -445,6 +475,61 @@ mod ffi { idx: usize, nested: &mut ArrayWriterInner, ) -> Result<()>; + fn aw_set_row( + self: &mut ArrayWriterInner, + idx: usize, + row: &mut GenericRowInner, + ) -> Result<()>; + fn aw_set_map( + self: &mut ArrayWriterInner, + idx: usize, + map: &mut MapWriterInner, + ) -> Result<()>; + + // MapWriterInner — opaque map builder for writes. + #[allow(clippy::too_many_arguments)] + fn new_map_writer( + capacity: usize, + key_leaf_type_id: i32, + key_precision: u32, + key_scale: u32, + value_leaf_type_id: i32, + value_precision: u32, + value_scale: u32, + value_array_nesting: u32, + ) -> Result>; + // ROW/MAP key/value types: a [key, value] Arrow schema (the flat encoding can't). + fn new_map_writer_arrow( + capacity: usize, + kv_schema_ptr: usize, + ) -> Result>; + fn mw_key_bool(self: &mut MapWriterInner, val: bool) -> Result<()>; + fn mw_key_i32(self: &mut MapWriterInner, val: i32) -> Result<()>; + fn mw_key_i64(self: &mut MapWriterInner, val: i64) -> Result<()>; + fn mw_key_f32(self: &mut MapWriterInner, val: f32) -> Result<()>; + fn mw_key_f64(self: &mut MapWriterInner, val: f64) -> Result<()>; + fn mw_key_str(self: &mut MapWriterInner, val: &str) -> Result<()>; + fn mw_key_bytes(self: &mut MapWriterInner, val: &[u8]) -> Result<()>; + fn mw_key_date(self: &mut MapWriterInner, days: i32) -> Result<()>; + fn mw_key_time(self: &mut MapWriterInner, millis: i32) -> Result<()>; + fn mw_key_timestamp(self: &mut MapWriterInner, millis: i64, nanos: i32) -> Result<()>; + fn mw_key_decimal_str(self: &mut MapWriterInner, val: &str) -> Result<()>; + fn mw_value_null(self: &mut MapWriterInner) -> Result<()>; + fn mw_value_bool(self: &mut MapWriterInner, val: bool) -> Result<()>; + fn mw_value_i32(self: &mut MapWriterInner, val: i32) -> Result<()>; + fn mw_value_i64(self: &mut MapWriterInner, val: i64) -> Result<()>; + fn mw_value_f32(self: &mut MapWriterInner, val: f32) -> Result<()>; + fn mw_value_f64(self: &mut MapWriterInner, val: f64) -> Result<()>; + fn mw_value_str(self: &mut MapWriterInner, val: &str) -> Result<()>; + fn mw_value_bytes(self: &mut MapWriterInner, val: &[u8]) -> Result<()>; + fn mw_value_date(self: &mut MapWriterInner, days: i32) -> Result<()>; + fn mw_value_time(self: &mut MapWriterInner, millis: i32) -> Result<()>; + fn mw_value_timestamp(self: &mut MapWriterInner, millis: i64, nanos: i32) -> Result<()>; + fn mw_value_decimal_str(self: &mut MapWriterInner, val: &str) -> Result<()>; + fn mw_value_row(self: &mut MapWriterInner, row: &mut GenericRowInner) -> Result<()>; + fn mw_value_map(self: &mut MapWriterInner, map: &mut MapWriterInner) -> Result<()>; + fn mw_value_array(self: &mut MapWriterInner, array: &mut ArrayWriterInner) -> Result<()>; + fn mw_commit(self: &mut MapWriterInner) -> Result<()>; // AppendWriter unsafe fn delete_append_writer(writer: *mut AppendWriter); @@ -493,59 +578,35 @@ mod ffi { fn lv_is_ts_ltz(self: &LookupResultInner, field: usize) -> Result; fn lv_get_decimal_str(self: &LookupResultInner, field: usize) -> Result; - fn lv_get_array_size(self: &LookupResultInner, field: usize) -> Result; - fn lv_get_array_is_null( - self: &LookupResultInner, - field: usize, - element: usize, - ) -> Result; - fn lv_get_array_bool( - self: &LookupResultInner, - field: usize, - element: usize, - ) -> Result; - fn lv_get_array_i32(self: &LookupResultInner, field: usize, element: usize) -> Result; - fn lv_get_array_i64(self: &LookupResultInner, field: usize, element: usize) -> Result; - fn lv_get_array_f32(self: &LookupResultInner, field: usize, element: usize) -> Result; - fn lv_get_array_f64(self: &LookupResultInner, field: usize, element: usize) -> Result; - fn lv_get_array_str( - self: &LookupResultInner, - field: usize, - element: usize, - ) -> Result; - fn lv_get_array_bytes( - self: &LookupResultInner, - field: usize, - element: usize, - ) -> Result>; - fn lv_get_array_date_days( - self: &LookupResultInner, - field: usize, - element: usize, - ) -> Result; - fn lv_get_array_time_millis( - self: &LookupResultInner, - field: usize, - element: usize, - ) -> Result; - fn lv_get_array_ts_millis( - self: &LookupResultInner, - field: usize, - element: usize, - ) -> Result; - fn lv_get_array_ts_nanos( - self: &LookupResultInner, - field: usize, - element: usize, - ) -> Result; - fn lv_get_array_decimal_str( - self: &LookupResultInner, + // ValueInner — recursive value handle for complex column reads. + fn lv_get_value(self: &LookupResultInner, field: usize) -> Result>; + fn sv_get_value( + self: &ScanResultInner, + bucket: usize, + rec: usize, field: usize, - element: usize, - ) -> Result; - fn lv_get_array_element_type(self: &LookupResultInner, field: usize) -> Result; - fn lv_get_array_view(self: &LookupResultInner, field: usize) - -> Result>; + ) -> Result>; + fn v_type(self: &ValueInner) -> i32; + fn v_is_null(self: &ValueInner) -> bool; + fn v_get_bool(self: &ValueInner) -> Result; + fn v_get_i32(self: &ValueInner) -> Result; + fn v_get_i64(self: &ValueInner) -> Result; + fn v_get_f32(self: &ValueInner) -> Result; + fn v_get_f64(self: &ValueInner) -> Result; + fn v_get_str(self: &ValueInner) -> Result; + fn v_get_bytes(self: &ValueInner) -> Result>; + fn v_get_date_days(self: &ValueInner) -> Result; + fn v_get_time_millis(self: &ValueInner) -> Result; + fn v_get_ts_millis(self: &ValueInner) -> Result; + fn v_get_ts_nanos(self: &ValueInner) -> Result; + fn v_get_decimal_str(self: &ValueInner) -> Result; + fn v_size(self: &ValueInner) -> Result; + fn v_field_count(self: &ValueInner) -> Result; + fn v_at(self: &ValueInner, i: usize) -> Result>; + fn v_key_at(self: &ValueInner, i: usize) -> Result>; + fn v_value_at(self: &ValueInner, i: usize) -> Result>; + fn v_field(self: &ValueInner, i: usize) -> Result>; + fn v_field_by_name(self: &ValueInner, name: &str) -> Result>; // PrefixLookuper unsafe fn delete_prefix_lookuper(lookuper: *mut PrefixLookuper); @@ -599,117 +660,11 @@ mod ffi { field: usize, ) -> Result; - fn plv_get_array_size( - self: &PrefixLookupResultInner, - rec: usize, - field: usize, - ) -> Result; - fn plv_get_array_is_null( - self: &PrefixLookupResultInner, - rec: usize, - field: usize, - element: usize, - ) -> Result; - fn plv_get_array_bool( - self: &PrefixLookupResultInner, - rec: usize, - field: usize, - element: usize, - ) -> Result; - fn plv_get_array_i32( - self: &PrefixLookupResultInner, - rec: usize, - field: usize, - element: usize, - ) -> Result; - fn plv_get_array_i64( - self: &PrefixLookupResultInner, - rec: usize, - field: usize, - element: usize, - ) -> Result; - fn plv_get_array_f32( - self: &PrefixLookupResultInner, - rec: usize, - field: usize, - element: usize, - ) -> Result; - fn plv_get_array_f64( - self: &PrefixLookupResultInner, - rec: usize, - field: usize, - element: usize, - ) -> Result; - fn plv_get_array_str( - self: &PrefixLookupResultInner, - rec: usize, - field: usize, - element: usize, - ) -> Result; - fn plv_get_array_bytes( - self: &PrefixLookupResultInner, - rec: usize, - field: usize, - element: usize, - ) -> Result>; - fn plv_get_array_date_days( - self: &PrefixLookupResultInner, - rec: usize, - field: usize, - element: usize, - ) -> Result; - fn plv_get_array_time_millis( - self: &PrefixLookupResultInner, - rec: usize, - field: usize, - element: usize, - ) -> Result; - fn plv_get_array_ts_millis( - self: &PrefixLookupResultInner, - rec: usize, - field: usize, - element: usize, - ) -> Result; - fn plv_get_array_ts_nanos( - self: &PrefixLookupResultInner, - rec: usize, - field: usize, - element: usize, - ) -> Result; - fn plv_get_array_decimal_str( - self: &PrefixLookupResultInner, - rec: usize, - field: usize, - element: usize, - ) -> Result; - fn plv_get_array_element_type( - self: &PrefixLookupResultInner, - rec: usize, - field: usize, - ) -> Result; - fn plv_get_array_view( + fn plv_get_value( self: &PrefixLookupResultInner, rec: usize, field: usize, - ) -> Result>; - - // ArrayViewInner — opaque recursive array reader for C++ bindings - fn av_size(self: &ArrayViewInner) -> usize; - fn av_element_type_id(self: &ArrayViewInner) -> i32; - fn av_is_null(self: &ArrayViewInner, element: usize) -> Result; - fn av_get_bool(self: &ArrayViewInner, element: usize) -> Result; - fn av_get_i32(self: &ArrayViewInner, element: usize) -> Result; - fn av_get_i64(self: &ArrayViewInner, element: usize) -> Result; - fn av_get_f32(self: &ArrayViewInner, element: usize) -> Result; - fn av_get_f64(self: &ArrayViewInner, element: usize) -> Result; - fn av_get_str(self: &ArrayViewInner, element: usize) -> Result; - fn av_get_bytes(self: &ArrayViewInner, element: usize) -> Result>; - fn av_get_date_days(self: &ArrayViewInner, element: usize) -> Result; - fn av_get_time_millis(self: &ArrayViewInner, element: usize) -> Result; - fn av_get_ts_millis(self: &ArrayViewInner, element: usize) -> Result; - fn av_get_ts_nanos(self: &ArrayViewInner, element: usize) -> Result; - fn av_get_decimal_str(self: &ArrayViewInner, element: usize) -> Result; - fn av_get_nested(self: &ArrayViewInner, element: usize) -> Result>; + ) -> Result>; // LogScanner unsafe fn delete_log_scanner(scanner: *mut LogScanner); @@ -832,111 +787,6 @@ mod ffi { field: usize, ) -> Result; - fn sv_get_array_size( - self: &ScanResultInner, - bucket: usize, - rec: usize, - field: usize, - ) -> Result; - fn sv_get_array_is_null( - self: &ScanResultInner, - bucket: usize, - rec: usize, - field: usize, - element: usize, - ) -> Result; - fn sv_get_array_bool( - self: &ScanResultInner, - bucket: usize, - rec: usize, - field: usize, - element: usize, - ) -> Result; - fn sv_get_array_i32( - self: &ScanResultInner, - bucket: usize, - rec: usize, - field: usize, - element: usize, - ) -> Result; - fn sv_get_array_i64( - self: &ScanResultInner, - bucket: usize, - rec: usize, - field: usize, - element: usize, - ) -> Result; - fn sv_get_array_f32( - self: &ScanResultInner, - bucket: usize, - rec: usize, - field: usize, - element: usize, - ) -> Result; - fn sv_get_array_f64( - self: &ScanResultInner, - bucket: usize, - rec: usize, - field: usize, - element: usize, - ) -> Result; - fn sv_get_array_str( - self: &ScanResultInner, - bucket: usize, - rec: usize, - field: usize, - element: usize, - ) -> Result; - fn sv_get_array_bytes( - self: &ScanResultInner, - bucket: usize, - rec: usize, - field: usize, - element: usize, - ) -> Result>; - fn sv_get_array_date_days( - self: &ScanResultInner, - bucket: usize, - rec: usize, - field: usize, - element: usize, - ) -> Result; - fn sv_get_array_time_millis( - self: &ScanResultInner, - bucket: usize, - rec: usize, - field: usize, - element: usize, - ) -> Result; - fn sv_get_array_ts_millis( - self: &ScanResultInner, - bucket: usize, - rec: usize, - field: usize, - element: usize, - ) -> Result; - fn sv_get_array_ts_nanos( - self: &ScanResultInner, - bucket: usize, - rec: usize, - field: usize, - element: usize, - ) -> Result; - fn sv_get_array_decimal_str( - self: &ScanResultInner, - bucket: usize, - rec: usize, - field: usize, - element: usize, - ) -> Result; - fn sv_get_array_element_type(self: &ScanResultInner, field: usize) -> Result; - fn sv_get_array_view( - self: &ScanResultInner, - bucket: usize, - rec: usize, - field: usize, - ) -> Result>; - fn sv_bucket_infos(self: &ScanResultInner) -> &Vec; } } @@ -1190,6 +1040,38 @@ impl Admin { } } + fn create_table_arrow( + &self, + table_path: &ffi::FfiTablePath, + descriptor: &ffi::FfiTableDescriptor, + arrow_schema_ptr: usize, + ignore_if_exists: bool, + ) -> ffi::FfiResult { + let path = fcore::metadata::TablePath::new( + table_path.database_name.clone(), + table_path.table_name.clone(), + ); + + // Safety: C++ exports the schema via `arrow::ExportSchema` into a heap + // `FFI_ArrowSchema` whose pointer is passed here; ownership transfers. + let core_descriptor = + match unsafe { types::arrow_ffi_to_core_descriptor(arrow_schema_ptr, descriptor) } { + Ok(d) => d, + Err(e) => return client_err(e.to_string()), + }; + + let result = RUNTIME.block_on(async { + self.inner + .create_table(&path, &core_descriptor, ignore_if_exists) + .await + }); + + match result { + Ok(_) => ok_result(), + Err(e) => err_from_core_error(&e), + } + } + fn drop_table( &self, table_path: &ffi::FfiTablePath, @@ -1263,8 +1145,6 @@ impl Admin { bucket_ids: Vec, offset_query: &ffi::FfiOffsetQuery, ) -> ffi::FfiListOffsetsResult { - use fcore::rpc::message::OffsetSpec; - let path = fcore::metadata::TablePath::new( table_path.database_name.clone(), table_path.table_name.clone(), @@ -1817,8 +1697,6 @@ impl AppendWriter { } fn append_arrow_batch(&mut self, array_ptr: usize, schema_ptr: usize) -> ffi::FfiPtrResult { - use arrow::ffi::{FFI_ArrowArray, FFI_ArrowSchema}; - // Safety: C++ allocates these via `new ArrowArray/ArrowSchema` after a // successful `ExportRecordBatch`, so both pointers are valid heap // allocations that we take ownership of here. @@ -2118,7 +1996,6 @@ unsafe fn delete_log_scanner(scanner: *mut LogScanner) { // Helper function to free the Arrow FFI structures separately (for use after ImportRecordBatch) pub extern "C" fn free_arrow_ffi_structures(array_ptr: usize, schema_ptr: usize) { - use arrow::ffi::{FFI_ArrowArray, FFI_ArrowSchema}; if array_ptr != 0 { let _array = unsafe { Box::from_raw(array_ptr as *mut FFI_ArrowArray) }; } @@ -2149,7 +2026,6 @@ impl LogScanner { } fn subscribe_buckets(&self, subscriptions: Vec) -> ffi::FfiResult { - use std::collections::HashMap; let bucket_offsets: HashMap = subscriptions .into_iter() .map(|s| (s.bucket_id, s.offset)) @@ -2173,7 +2049,6 @@ impl LogScanner { &self, subscriptions: Vec, ) -> ffi::FfiResult { - use std::collections::HashMap; let offsets: HashMap<(PartitionId, i32), i64> = subscriptions .into_iter() .map(|s| ((s.partition_id, s.bucket_id), s.offset)) @@ -2380,6 +2255,25 @@ impl GenericRowInner { Ok(()) } + fn gr_set_map(&mut self, idx: usize, writer: &mut MapWriterInner) -> Result<(), String> { + self.ensure_size(idx); + writer.complete_if_needed()?; + let map = writer.completed.take().ok_or_else(|| { + "MapWriter invariant violation: completed map missing after finalize".to_string() + })?; + self.row.set_field(idx, fcore::row::Datum::Map(map)); + Ok(()) + } + + fn gr_set_row(&mut self, idx: usize, row: &mut GenericRowInner) -> Result<(), String> { + self.ensure_size(idx); + // Move the nested row out; the C++ side discards its writer afterwards. + let nested = std::mem::replace(&mut row.row, fcore::row::GenericRow::new(0)); + self.row + .set_field(idx, fcore::row::Datum::Row(Box::new(nested))); + Ok(()) + } + fn ensure_size(&mut self, idx: usize) { if self.row.values.len() <= idx { self.row.values.resize(idx + 1, fcore::row::Datum::Null); @@ -2392,7 +2286,6 @@ impl GenericRowInner { // ============================================================================ mod row_reader { - use super::array_reader; use fcore::row::InternalRow; use fluss as fcore; @@ -2661,560 +2554,102 @@ mod row_reader { dt => Err(format!("get_decimal_str: unexpected type {dt}")), } } +} - fn get_fluss_array( - row: &dyn InternalRow, - columns: &[fcore::metadata::Column], - field: usize, - ) -> Result { - validate(row, columns, field, "get_array", |dt| { - matches!(dt, fcore::metadata::DataType::Array(_)) - })?; - row.get_array(field) - .map_err(|e| e.to_string())? - .try_into_binary() - .map_err(|e| e.to_string()) - } - - pub fn get_array_element_type( - columns: &[fcore::metadata::Column], - field: usize, - ) -> Result<&fcore::metadata::DataType, String> { - let col = get_column(columns, field)?; - match col.data_type() { - fcore::metadata::DataType::Array(at) => Ok(at.get_element_type()), - dt => Err(format!("get_array: column {field} is not Array, got {dt}")), - } - } +// ============================================================================ +// array_reader — low-level accessors over an already-resolved FlussArray. +// +// `get_datum` is the single dispatch that turns any array/map element into an +// owned `Datum` for the recursive `ValueInner` handle (bounds-checked, null +// and type validated, recursing into nested ROW/MAP/ARRAY). +// ============================================================================ - pub fn get_array_size( - row: &dyn InternalRow, - columns: &[fcore::metadata::Column], - field: usize, - ) -> Result { - let arr = get_fluss_array(row, columns, field)?; - Ok(arr.size()) - } +mod array_reader { + use super::fcore; + use fcore::metadata::DataType as DT; + use fcore::row::Datum; - pub fn get_array_and_elem_type<'a>( - row: &dyn InternalRow, - columns: &'a [fcore::metadata::Column], - field: usize, - ) -> Result< - ( - fcore::row::binary_array::FlussArray, - &'a fcore::metadata::DataType, - ), - String, - > { - let arr = get_fluss_array(row, columns, field)?; - let elem = get_array_element_type(columns, field)?; - Ok((arr, elem)) - } - - pub fn get_array_is_null( - row: &dyn InternalRow, - columns: &[fcore::metadata::Column], - field: usize, + fn validate_index( + arr: &fcore::row::binary_array::FlussArray, element: usize, - ) -> Result { - let arr = get_fluss_array(row, columns, field)?; - array_reader::is_null(&arr, element) + op: &str, + ) -> Result<(), String> { + if element < arr.size() { + Ok(()) + } else { + Err(format!( + "{op}: element index out of bounds: element={element}, size={}", + arr.size() + )) + } } - pub fn get_array_bool( - row: &dyn InternalRow, - columns: &[fcore::metadata::Column], - field: usize, + pub fn is_null( + arr: &fcore::row::binary_array::FlussArray, element: usize, ) -> Result { - let (arr, elem) = get_array_and_elem_type(row, columns, field)?; - array_reader::get_bool(&arr, elem, element) + validate_index(arr, element, "array_is_null")?; + Ok(arr.is_null_at(element)) } - pub fn get_array_i32( - row: &dyn InternalRow, - columns: &[fcore::metadata::Column], - field: usize, - element: usize, - ) -> Result { - let (arr, elem) = get_array_and_elem_type(row, columns, field)?; - array_reader::get_i32(&arr, elem, element) - } - - pub fn get_array_i64( - row: &dyn InternalRow, - columns: &[fcore::metadata::Column], - field: usize, - element: usize, - ) -> Result { - let (arr, elem) = get_array_and_elem_type(row, columns, field)?; - array_reader::get_i64(&arr, elem, element) - } - - pub fn get_array_f32( - row: &dyn InternalRow, - columns: &[fcore::metadata::Column], - field: usize, - element: usize, - ) -> Result { - let (arr, elem) = get_array_and_elem_type(row, columns, field)?; - array_reader::get_f32(&arr, elem, element) - } - - pub fn get_array_f64( - row: &dyn InternalRow, - columns: &[fcore::metadata::Column], - field: usize, - element: usize, - ) -> Result { - let (arr, elem) = get_array_and_elem_type(row, columns, field)?; - array_reader::get_f64(&arr, elem, element) - } - - pub fn get_array_str( - row: &dyn InternalRow, - columns: &[fcore::metadata::Column], - field: usize, - element: usize, - ) -> Result { - let (arr, elem) = get_array_and_elem_type(row, columns, field)?; - array_reader::get_str(&arr, elem, element) - } - - pub fn get_array_bytes( - row: &dyn InternalRow, - columns: &[fcore::metadata::Column], - field: usize, - element: usize, - ) -> Result, String> { - let (arr, elem) = get_array_and_elem_type(row, columns, field)?; - array_reader::get_bytes(&arr, elem, element) - } - - pub fn get_array_date_days( - row: &dyn InternalRow, - columns: &[fcore::metadata::Column], - field: usize, - element: usize, - ) -> Result { - let (arr, elem) = get_array_and_elem_type(row, columns, field)?; - array_reader::get_date_days(&arr, elem, element) - } - - pub fn get_array_time_millis( - row: &dyn InternalRow, - columns: &[fcore::metadata::Column], - field: usize, - element: usize, - ) -> Result { - let (arr, elem) = get_array_and_elem_type(row, columns, field)?; - array_reader::get_time_millis(&arr, elem, element) - } - - pub fn get_array_ts_millis( - row: &dyn InternalRow, - columns: &[fcore::metadata::Column], - field: usize, - element: usize, - ) -> Result { - let (arr, elem) = get_array_and_elem_type(row, columns, field)?; - array_reader::get_ts_millis(&arr, elem, element) - } - - pub fn get_array_ts_nanos( - row: &dyn InternalRow, - columns: &[fcore::metadata::Column], - field: usize, - element: usize, - ) -> Result { - let (arr, elem) = get_array_and_elem_type(row, columns, field)?; - array_reader::get_ts_nanos(&arr, elem, element) - } - - pub fn get_array_decimal_str( - row: &dyn InternalRow, - columns: &[fcore::metadata::Column], - field: usize, - element: usize, - ) -> Result { - let (arr, elem) = get_array_and_elem_type(row, columns, field)?; - array_reader::get_decimal_str(&arr, elem, element) - } - - pub fn get_array_element_type_id( - columns: &[fcore::metadata::Column], - field: usize, - ) -> Result { - let elem_type = get_array_element_type(columns, field)?; - Ok(crate::types::core_data_type_to_ffi(elem_type)) - } -} - -// ============================================================================ -// array_reader — low-level accessors over an already-resolved FlussArray -// -// Shared by the top-level `row_reader::get_array_*` wrappers and by -// `ArrayViewInner` (which exposes recursive/nested access to C++). Keeping -// one implementation here guarantees identical bounds-checking, null -// validation, type checking, and type dispatch across flat and nested reads. -// ============================================================================ - -mod array_reader { - use super::fcore; - - fn validate_index( - arr: &fcore::row::binary_array::FlussArray, - element: usize, - op: &str, - ) -> Result<(), String> { - if element < arr.size() { - Ok(()) - } else { - Err(format!( - "{op}: element index out of bounds: element={element}, size={}", - arr.size() - )) - } - } - - fn ensure_non_null( - arr: &fcore::row::binary_array::FlussArray, - element: usize, - op: &str, - ) -> Result<(), String> { - if arr.is_null_at(element) { - Err(format!( - "{op}: element at index {element} is null; call array_is_null first" - )) - } else { - Ok(()) - } - } - - fn ensure_type( - elem_type: &fcore::metadata::DataType, - op: &str, - expected: &str, - allowed: impl FnOnce(&fcore::metadata::DataType) -> bool, - ) -> Result<(), String> { - if allowed(elem_type) { - Ok(()) - } else { - Err(format!( - "{op}: element type is {elem_type}, expected {expected}" - )) - } - } - - fn ensure_readable( - arr: &fcore::row::binary_array::FlussArray, - elem_type: &fcore::metadata::DataType, - element: usize, - op: &str, - expected: &str, - allowed: impl FnOnce(&fcore::metadata::DataType) -> bool, - ) -> Result<(), String> { - validate_index(arr, element, op)?; - ensure_type(elem_type, op, expected, allowed)?; - ensure_non_null(arr, element, op) - } - - pub fn is_null( - arr: &fcore::row::binary_array::FlussArray, - element: usize, - ) -> Result { - validate_index(arr, element, "array_is_null")?; - Ok(arr.is_null_at(element)) - } - - pub fn get_bool( - arr: &fcore::row::binary_array::FlussArray, - elem_type: &fcore::metadata::DataType, - element: usize, - ) -> Result { - ensure_readable(arr, elem_type, element, "array_bool", "BOOLEAN", |dt| { - matches!(dt, fcore::metadata::DataType::Boolean(_)) - })?; - arr.get_boolean(element).map_err(|e| e.to_string()) - } - - pub fn get_i32( - arr: &fcore::row::binary_array::FlussArray, - elem_type: &fcore::metadata::DataType, - element: usize, - ) -> Result { - ensure_readable( - arr, - elem_type, - element, - "array_i32", - "TINYINT/SMALLINT/INT", - |dt| { - matches!( - dt, - fcore::metadata::DataType::TinyInt(_) - | fcore::metadata::DataType::SmallInt(_) - | fcore::metadata::DataType::Int(_) - ) - }, - )?; - match elem_type { - fcore::metadata::DataType::TinyInt(_) => arr - .get_byte(element) - .map(|v| v as i32) - .map_err(|e| e.to_string()), - fcore::metadata::DataType::SmallInt(_) => arr - .get_short(element) - .map(|v| v as i32) - .map_err(|e| e.to_string()), - fcore::metadata::DataType::Int(_) => arr.get_int(element).map_err(|e| e.to_string()), - _ => unreachable!("type validated by ensure_readable"), - } - } - - pub fn get_i64( - arr: &fcore::row::binary_array::FlussArray, - elem_type: &fcore::metadata::DataType, - element: usize, - ) -> Result { - ensure_readable(arr, elem_type, element, "array_i64", "BIGINT", |dt| { - matches!(dt, fcore::metadata::DataType::BigInt(_)) - })?; - arr.get_long(element).map_err(|e| e.to_string()) - } - - pub fn get_f32( - arr: &fcore::row::binary_array::FlussArray, - elem_type: &fcore::metadata::DataType, - element: usize, - ) -> Result { - ensure_readable(arr, elem_type, element, "array_f32", "FLOAT", |dt| { - matches!(dt, fcore::metadata::DataType::Float(_)) - })?; - arr.get_float(element).map_err(|e| e.to_string()) - } - - pub fn get_f64( - arr: &fcore::row::binary_array::FlussArray, - elem_type: &fcore::metadata::DataType, - element: usize, - ) -> Result { - ensure_readable(arr, elem_type, element, "array_f64", "DOUBLE", |dt| { - matches!(dt, fcore::metadata::DataType::Double(_)) - })?; - arr.get_double(element).map_err(|e| e.to_string()) - } - - pub fn get_str( - arr: &fcore::row::binary_array::FlussArray, - elem_type: &fcore::metadata::DataType, - element: usize, - ) -> Result { - ensure_readable(arr, elem_type, element, "array_str", "STRING/CHAR", |dt| { - matches!( - dt, - fcore::metadata::DataType::String(_) | fcore::metadata::DataType::Char(_) - ) - })?; - arr.get_string(element) - .map(|s| s.to_string()) - .map_err(|e| e.to_string()) - } - - pub fn get_bytes( - arr: &fcore::row::binary_array::FlussArray, - elem_type: &fcore::metadata::DataType, - element: usize, - ) -> Result, String> { - ensure_readable( - arr, - elem_type, - element, - "array_bytes", - "BYTES/BINARY", - |dt| { - matches!( - dt, - fcore::metadata::DataType::Bytes(_) | fcore::metadata::DataType::Binary(_) - ) - }, - )?; - arr.get_binary(element) - .map(|b| b.to_vec()) - .map_err(|e| e.to_string()) - } - - pub fn get_date_days( + pub fn get_datum( arr: &fcore::row::binary_array::FlussArray, elem_type: &fcore::metadata::DataType, element: usize, - ) -> Result { - ensure_readable(arr, elem_type, element, "array_date", "DATE", |dt| { - matches!(dt, fcore::metadata::DataType::Date(_)) - })?; - arr.get_date(element) - .map(|d| d.get_inner()) - .map_err(|e| e.to_string()) - } - - pub fn get_time_millis( - arr: &fcore::row::binary_array::FlussArray, - elem_type: &fcore::metadata::DataType, - element: usize, - ) -> Result { - ensure_readable(arr, elem_type, element, "array_time", "TIME", |dt| { - matches!(dt, fcore::metadata::DataType::Time(_)) - })?; - arr.get_time(element) - .map(|t| t.get_inner()) - .map_err(|e| e.to_string()) - } - - pub fn get_ts_millis( - arr: &fcore::row::binary_array::FlussArray, - elem_type: &fcore::metadata::DataType, - element: usize, - ) -> Result { - ensure_readable( - arr, - elem_type, - element, - "array_ts_millis", - "TIMESTAMP/TIMESTAMP_LTZ", - |dt| { - matches!( - dt, - fcore::metadata::DataType::Timestamp(_) - | fcore::metadata::DataType::TimestampLTz(_) - ) - }, - )?; - match elem_type { - fcore::metadata::DataType::TimestampLTz(ts) => arr - .get_timestamp_ltz(element, ts.precision()) - .map(|v| v.get_epoch_millisecond()) - .map_err(|e| e.to_string()), - fcore::metadata::DataType::Timestamp(ts) => arr - .get_timestamp_ntz(element, ts.precision()) - .map(|v| v.get_millisecond()) - .map_err(|e| e.to_string()), - _ => unreachable!("type validated by ensure_readable"), - } - } - - pub fn get_ts_nanos( - arr: &fcore::row::binary_array::FlussArray, - elem_type: &fcore::metadata::DataType, - element: usize, - ) -> Result { - ensure_readable( - arr, - elem_type, - element, - "array_ts_nanos", - "TIMESTAMP/TIMESTAMP_LTZ", - |dt| { - matches!( - dt, - fcore::metadata::DataType::Timestamp(_) - | fcore::metadata::DataType::TimestampLTz(_) - ) - }, - )?; - match elem_type { - fcore::metadata::DataType::TimestampLTz(ts) => arr - .get_timestamp_ltz(element, ts.precision()) - .map(|v| v.get_nano_of_millisecond()) - .map_err(|e| e.to_string()), - fcore::metadata::DataType::Timestamp(ts) => arr - .get_timestamp_ntz(element, ts.precision()) - .map(|v| v.get_nano_of_millisecond()) - .map_err(|e| e.to_string()), - _ => unreachable!("type validated by ensure_readable"), - } - } - - pub fn get_decimal_str( - arr: &fcore::row::binary_array::FlussArray, - elem_type: &fcore::metadata::DataType, - element: usize, - ) -> Result { - ensure_readable(arr, elem_type, element, "array_decimal", "DECIMAL", |dt| { - matches!(dt, fcore::metadata::DataType::Decimal(_)) - })?; - match elem_type { - fcore::metadata::DataType::Decimal(dd) => { - let decimal = arr - .get_decimal(element, dd.precision(), dd.scale()) - .map_err(|e| e.to_string())?; - Ok(decimal.to_big_decimal().to_string()) - } - _ => unreachable!("type validated by ensure_readable"), - } - } - - pub fn get_nested_array( - arr: &fcore::row::binary_array::FlussArray, - elem_type: &fcore::metadata::DataType, - element: usize, - ) -> Result< - ( - fcore::row::binary_array::FlussArray, - fcore::metadata::DataType, - ), - String, - > { - ensure_readable(arr, elem_type, element, "array_nested", "ARRAY", |dt| { - matches!(dt, fcore::metadata::DataType::Array(_)) - })?; - match elem_type { - fcore::metadata::DataType::Array(at) => { - let nested = arr.get_array(element).map_err(|e| e.to_string())?; - Ok((nested, at.get_element_type().clone())) + ) -> Result, String> { + if is_null(arr, element)? { + return Ok(Datum::Null); + } + let map_err = |e: fcore::error::Error| e.to_string(); + Ok(match elem_type { + DT::Boolean(_) => Datum::Bool(arr.get_boolean(element).map_err(map_err)?), + DT::TinyInt(_) => Datum::Int8(arr.get_byte(element).map_err(map_err)?), + DT::SmallInt(_) => Datum::Int16(arr.get_short(element).map_err(map_err)?), + DT::Int(_) => Datum::Int32(arr.get_int(element).map_err(map_err)?), + DT::BigInt(_) => Datum::Int64(arr.get_long(element).map_err(map_err)?), + DT::Float(_) => Datum::Float32(arr.get_float(element).map_err(map_err)?.into()), + DT::Double(_) => Datum::Float64(arr.get_double(element).map_err(map_err)?.into()), + DT::String(_) | DT::Char(_) => Datum::String(std::borrow::Cow::Owned( + arr.get_string(element).map_err(map_err)?.to_string(), + )), + DT::Bytes(_) | DT::Binary(_) => Datum::Blob(std::borrow::Cow::Owned( + arr.get_binary(element).map_err(map_err)?.to_vec(), + )), + DT::Decimal(d) => Datum::Decimal( + arr.get_decimal(element, d.precision(), d.scale()) + .map_err(map_err)?, + ), + DT::Date(_) => Datum::Date(arr.get_date(element).map_err(map_err)?), + DT::Time(_) => Datum::Time(arr.get_time(element).map_err(map_err)?), + DT::Timestamp(t) => Datum::TimestampNtz( + arr.get_timestamp_ntz(element, t.precision()) + .map_err(map_err)?, + ), + DT::TimestampLTz(t) => Datum::TimestampLtz( + arr.get_timestamp_ltz(element, t.precision()) + .map_err(map_err)?, + ), + DT::Array(_) => Datum::Array(arr.get_array(element).map_err(map_err)?), + DT::Map(mt) => Datum::Map( + arr.get_map(element, mt.key_type(), mt.value_type()) + .map_err(map_err)?, + ), + DT::Row(rt) => { + let cr = arr.get_row(element, rt).map_err(map_err)?; + let cols: Vec = rt + .fields() + .iter() + .map(|f| fcore::metadata::Column::new(f.name(), f.data_type().clone())) + .collect(); + Datum::Row(Box::new( + crate::types::internal_row_to_owned_generic(&cr, &cols) + .map_err(|e| e.to_string())?, + )) } - _ => unreachable!("type validated by ensure_readable"), - } + }) } } -// ============================================================================ -// Macros that generate uniform sv_/lv_ array element getters (thin wrappers -// that only forward to `row_reader::get_array_*`). -// ============================================================================ - -macro_rules! sv_array_element_getters { - ($( $method:ident, $reader_fn:ident, $ret:ty; )+) => { - $( - fn $method( - &self, - bucket: usize, - rec: usize, - field: usize, - element: usize, - ) -> Result<$ret, String> { - row_reader::$reader_fn( - self.resolve(bucket, rec).row(), - &self.columns, - field, - element, - ) - } - )+ - }; -} - -macro_rules! lv_array_element_getters { - ($( $method:ident, $reader_fn:ident, $ret:ty; )+) => { - $( - fn $method(&self, field: usize, element: usize) -> Result<$ret, String> { - let r = self.lv_row()?; - row_reader::$reader_fn(r, &self.columns, field, element) - } - )+ - }; -} - // ============================================================================ // Opaque types: ScanResultInner (scan read path) // ============================================================================ @@ -3330,42 +2765,28 @@ impl ScanResultInner { row_reader::get_decimal_str(self.resolve(bucket, rec).row(), &self.columns, field) } - fn sv_get_array_size(&self, bucket: usize, rec: usize, field: usize) -> Result { - row_reader::get_array_size(self.resolve(bucket, rec).row(), &self.columns, field) - } - sv_array_element_getters! { - sv_get_array_is_null, get_array_is_null, bool; - sv_get_array_bool, get_array_bool, bool; - sv_get_array_i32, get_array_i32, i32; - sv_get_array_i64, get_array_i64, i64; - sv_get_array_f32, get_array_f32, f32; - sv_get_array_f64, get_array_f64, f64; - sv_get_array_str, get_array_str, String; - sv_get_array_bytes, get_array_bytes, Vec; - sv_get_array_date_days, get_array_date_days, i32; - sv_get_array_time_millis, get_array_time_millis, i32; - sv_get_array_ts_millis, get_array_ts_millis, i64; - sv_get_array_ts_nanos, get_array_ts_nanos, i32; - sv_get_array_decimal_str, get_array_decimal_str, String; - } - fn sv_get_array_element_type(&self, field: usize) -> Result { - row_reader::get_array_element_type_id(&self.columns, field) - } - fn sv_get_array_view( + fn sv_get_value( &self, bucket: usize, rec: usize, field: usize, - ) -> Result, String> { - let (arr, elem) = row_reader::get_array_and_elem_type( + ) -> Result, String> { + // Scan rows are borrowed Arrow cursors; materialize only the requested + // field's owned datum (not the whole row). Top-level scalar scan reads + // keep their zero-copy index fast-path. + let datum = crate::types::field_to_owned_datum( self.resolve(bucket, rec).row(), &self.columns, field, - )?; - Ok(Box::new(ArrayViewInner { - array: arr, - element_type: elem.clone(), - })) + ) + .map_err(|e| e.to_string())?; + let data_type = self + .columns + .get(field) + .ok_or_else(|| format!("field index {field} out of range"))? + .data_type() + .clone(); + Ok(Box::new(ValueInner { datum, data_type })) } fn sv_bucket_infos(&self) -> &Vec { @@ -3484,35 +2905,8 @@ impl LookupResultInner { let r = self.lv_row()?; row_reader::get_decimal_str(r, &self.columns, field) } - fn lv_get_array_size(&self, field: usize) -> Result { - let r = self.lv_row()?; - row_reader::get_array_size(r, &self.columns, field) - } - lv_array_element_getters! { - lv_get_array_is_null, get_array_is_null, bool; - lv_get_array_bool, get_array_bool, bool; - lv_get_array_i32, get_array_i32, i32; - lv_get_array_i64, get_array_i64, i64; - lv_get_array_f32, get_array_f32, f32; - lv_get_array_f64, get_array_f64, f64; - lv_get_array_str, get_array_str, String; - lv_get_array_bytes, get_array_bytes, Vec; - lv_get_array_date_days, get_array_date_days, i32; - lv_get_array_time_millis, get_array_time_millis, i32; - lv_get_array_ts_millis, get_array_ts_millis, i64; - lv_get_array_ts_nanos, get_array_ts_nanos, i32; - lv_get_array_decimal_str, get_array_decimal_str, String; - } - fn lv_get_array_element_type(&self, field: usize) -> Result { - row_reader::get_array_element_type_id(&self.columns, field) - } - fn lv_get_array_view(&self, field: usize) -> Result, String> { - let r = self.lv_row()?; - let (arr, elem) = row_reader::get_array_and_elem_type(r, &self.columns, field)?; - Ok(Box::new(ArrayViewInner { - array: arr, - element_type: elem.clone(), - })) + fn lv_get_value(&self, field: usize) -> Result, String> { + value_from_owned_row(self.lv_row()?, &self.columns, field) } } @@ -3528,17 +2922,6 @@ pub struct PrefixLookupResultInner { columns: Vec, } -macro_rules! plv_array_element_getters { - ($( $method:ident, $reader_fn:ident, $ret:ty; )+) => { - $( - fn $method(&self, rec: usize, field: usize, element: usize) -> Result<$ret, String> { - let r = self.plv_row(rec)?; - row_reader::$reader_fn(r, &self.columns, field, element) - } - )+ - }; -} - impl PrefixLookupResultInner { fn from_error(code: i32, msg: String) -> Self { Self { @@ -3625,118 +3008,223 @@ impl PrefixLookupResultInner { fn plv_get_decimal_str(&self, rec: usize, field: usize) -> Result { row_reader::get_decimal_str(self.plv_row(rec)?, &self.columns, field) } - fn plv_get_array_size(&self, rec: usize, field: usize) -> Result { - row_reader::get_array_size(self.plv_row(rec)?, &self.columns, field) - } - plv_array_element_getters! { - plv_get_array_is_null, get_array_is_null, bool; - plv_get_array_bool, get_array_bool, bool; - plv_get_array_i32, get_array_i32, i32; - plv_get_array_i64, get_array_i64, i64; - plv_get_array_f32, get_array_f32, f32; - plv_get_array_f64, get_array_f64, f64; - plv_get_array_str, get_array_str, String; - plv_get_array_bytes, get_array_bytes, Vec; - plv_get_array_date_days, get_array_date_days, i32; - plv_get_array_time_millis, get_array_time_millis, i32; - plv_get_array_ts_millis, get_array_ts_millis, i64; - plv_get_array_ts_nanos, get_array_ts_nanos, i32; - plv_get_array_decimal_str, get_array_decimal_str, String; - } - fn plv_get_array_element_type(&self, _rec: usize, field: usize) -> Result { - row_reader::get_array_element_type_id(&self.columns, field) - } - fn plv_get_array_view(&self, rec: usize, field: usize) -> Result, String> { - let r = self.plv_row(rec)?; - let (arr, elem) = row_reader::get_array_and_elem_type(r, &self.columns, field)?; - Ok(Box::new(ArrayViewInner { - array: arr, - element_type: elem.clone(), - })) + fn plv_get_value(&self, rec: usize, field: usize) -> Result, String> { + value_from_owned_row(self.plv_row(rec)?, &self.columns, field) } } // ============================================================================ -// Opaque types: ArrayViewInner (recursive array reader) -// -// Wraps an owned `FlussArray` plus its element `DataType` and exposes the -// same accessors as `row_reader::get_array_*`, delegating to the shared -// `array_reader` primitives. Enables C++ bindings to recurse into nested -// arrays without per-level FFI scaffolding. +// ValueInner — recursive value handle for complex reads. Holds an owned Datum +// + its DataType: leaf getters match on the datum, navigators (v_at/v_key_at/ +// v_value_at/v_field) descend into a child node. // ============================================================================ -pub struct ArrayViewInner { - array: fcore::row::binary_array::FlussArray, - element_type: fcore::metadata::DataType, +pub struct ValueInner { + datum: fcore::row::Datum<'static>, + data_type: fcore::metadata::DataType, } -impl ArrayViewInner { - fn av_size(&self) -> usize { - self.array.size() - } +/// Builds a `ValueInner` from an owned row; shared by the lookup and prefix-lookup paths. +fn value_from_owned_row( + row: &GenericRow<'static>, + columns: &[Column], + field: usize, +) -> Result, String> { + let datum = row + .values + .get(field) + .ok_or_else(|| format!("field index {field} out of range"))? + .clone(); + let data_type = columns + .get(field) + .ok_or_else(|| format!("field index {field} out of range"))? + .data_type() + .clone(); + Ok(Box::new(ValueInner { datum, data_type })) +} - fn av_element_type_id(&self) -> i32 { - crate::types::core_data_type_to_ffi(&self.element_type) +impl ValueInner { + fn type_err(&self, expected: &str) -> String { + format!("value of type {} is not {expected}", self.data_type) } - fn av_is_null(&self, element: usize) -> Result { - array_reader::is_null(&self.array, element) + fn v_type(&self) -> i32 { + crate::types::core_data_type_to_ffi(&self.data_type) } - - fn av_get_bool(&self, element: usize) -> Result { - array_reader::get_bool(&self.array, &self.element_type, element) + fn v_is_null(&self) -> bool { + matches!(self.datum, fcore::row::Datum::Null) } - fn av_get_i32(&self, element: usize) -> Result { - array_reader::get_i32(&self.array, &self.element_type, element) + fn v_get_bool(&self) -> Result { + match &self.datum { + fcore::row::Datum::Bool(v) => Ok(*v), + _ => Err(self.type_err("BOOLEAN")), + } } - - fn av_get_i64(&self, element: usize) -> Result { - array_reader::get_i64(&self.array, &self.element_type, element) + fn v_get_i32(&self) -> Result { + match &self.datum { + fcore::row::Datum::Int8(v) => Ok(i32::from(*v)), + fcore::row::Datum::Int16(v) => Ok(i32::from(*v)), + fcore::row::Datum::Int32(v) => Ok(*v), + _ => Err(self.type_err("INT")), + } } - - fn av_get_f32(&self, element: usize) -> Result { - array_reader::get_f32(&self.array, &self.element_type, element) + fn v_get_i64(&self) -> Result { + match &self.datum { + fcore::row::Datum::Int64(v) => Ok(*v), + _ => Err(self.type_err("BIGINT")), + } } - - fn av_get_f64(&self, element: usize) -> Result { - array_reader::get_f64(&self.array, &self.element_type, element) + fn v_get_f32(&self) -> Result { + match &self.datum { + fcore::row::Datum::Float32(v) => Ok(v.0), + _ => Err(self.type_err("FLOAT")), + } } - - fn av_get_str(&self, element: usize) -> Result { - array_reader::get_str(&self.array, &self.element_type, element) + fn v_get_f64(&self) -> Result { + match &self.datum { + fcore::row::Datum::Float64(v) => Ok(v.0), + _ => Err(self.type_err("DOUBLE")), + } } - - fn av_get_bytes(&self, element: usize) -> Result, String> { - array_reader::get_bytes(&self.array, &self.element_type, element) + fn v_get_str(&self) -> Result { + match &self.datum { + fcore::row::Datum::String(v) => Ok(v.to_string()), + _ => Err(self.type_err("STRING")), + } } - - fn av_get_date_days(&self, element: usize) -> Result { - array_reader::get_date_days(&self.array, &self.element_type, element) + fn v_get_bytes(&self) -> Result, String> { + match &self.datum { + fcore::row::Datum::Blob(v) => Ok(v.to_vec()), + _ => Err(self.type_err("BYTES")), + } } - - fn av_get_time_millis(&self, element: usize) -> Result { - array_reader::get_time_millis(&self.array, &self.element_type, element) + fn v_get_date_days(&self) -> Result { + match &self.datum { + fcore::row::Datum::Date(d) => Ok(d.get_inner()), + _ => Err(self.type_err("DATE")), + } } - - fn av_get_ts_millis(&self, element: usize) -> Result { - array_reader::get_ts_millis(&self.array, &self.element_type, element) + fn v_get_time_millis(&self) -> Result { + match &self.datum { + fcore::row::Datum::Time(t) => Ok(t.get_inner()), + _ => Err(self.type_err("TIME")), + } } - - fn av_get_ts_nanos(&self, element: usize) -> Result { - array_reader::get_ts_nanos(&self.array, &self.element_type, element) + fn v_get_ts_millis(&self) -> Result { + match &self.datum { + fcore::row::Datum::TimestampNtz(t) => Ok(t.get_millisecond()), + fcore::row::Datum::TimestampLtz(t) => Ok(t.get_epoch_millisecond()), + _ => Err(self.type_err("TIMESTAMP")), + } + } + fn v_get_ts_nanos(&self) -> Result { + match &self.datum { + fcore::row::Datum::TimestampNtz(t) => Ok(t.get_nano_of_millisecond()), + fcore::row::Datum::TimestampLtz(t) => Ok(t.get_nano_of_millisecond()), + _ => Err(self.type_err("TIMESTAMP")), + } + } + fn v_get_decimal_str(&self) -> Result { + match &self.datum { + fcore::row::Datum::Decimal(d) => Ok(d.to_big_decimal().to_string()), + _ => Err(self.type_err("DECIMAL")), + } } - fn av_get_decimal_str(&self, element: usize) -> Result { - array_reader::get_decimal_str(&self.array, &self.element_type, element) + fn v_size(&self) -> Result { + match &self.datum { + fcore::row::Datum::Array(a) => Ok(a.size()), + fcore::row::Datum::Map(m) => Ok(m.size()), + _ => Err(self.type_err("ARRAY/MAP")), + } + } + fn v_field_count(&self) -> Result { + match &self.datum { + fcore::row::Datum::Row(r) => Ok(r.values.len()), + _ => Err(self.type_err("ROW")), + } + } + fn v_at(&self, i: usize) -> Result, String> { + match &self.datum { + fcore::row::Datum::Array(a) => { + let elem_type = match &self.data_type { + fcore::metadata::DataType::Array(at) => at.get_element_type().clone(), + _ => return Err("internal: ARRAY datum without ARRAY type".to_string()), + }; + let datum = array_reader::get_datum(a, &elem_type, i)?; + Ok(Box::new(ValueInner { + datum, + data_type: elem_type, + })) + } + _ => Err(self.type_err("ARRAY")), + } + } + fn v_key_at(&self, i: usize) -> Result, String> { + match &self.datum { + fcore::row::Datum::Map(m) => { + let kt = m.key_type().clone(); + let datum = array_reader::get_datum(m.key_array(), &kt, i)?; + Ok(Box::new(ValueInner { + datum, + data_type: kt, + })) + } + _ => Err(self.type_err("MAP")), + } + } + fn v_value_at(&self, i: usize) -> Result, String> { + match &self.datum { + fcore::row::Datum::Map(m) => { + let vt = m.value_type().clone(); + let datum = array_reader::get_datum(m.value_array(), &vt, i)?; + Ok(Box::new(ValueInner { + datum, + data_type: vt, + })) + } + _ => Err(self.type_err("MAP")), + } + } + fn v_field(&self, i: usize) -> Result, String> { + match &self.datum { + fcore::row::Datum::Row(r) => { + let ft = match &self.data_type { + fcore::metadata::DataType::Row(rt) => rt + .fields() + .get(i) + .ok_or_else(|| format!("field index {i} out of range"))? + .data_type() + .clone(), + _ => return Err("internal: ROW datum without ROW type".to_string()), + }; + let datum = r + .values + .get(i) + .ok_or_else(|| format!("field index {i} out of range"))? + .clone(); + Ok(Box::new(ValueInner { + datum, + data_type: ft, + })) + } + _ => Err(self.type_err("ROW")), + } } - fn av_get_nested(&self, element: usize) -> Result, String> { - let (arr, elem) = array_reader::get_nested_array(&self.array, &self.element_type, element)?; - Ok(Box::new(ArrayViewInner { - array: arr, - element_type: elem, - })) + fn v_field_by_name(&self, name: &str) -> Result, String> { + match &self.data_type { + fcore::metadata::DataType::Row(rt) => { + let idx = rt + .fields() + .iter() + .position(|f| f.name() == name) + .ok_or_else(|| format!("no field named '{name}' in {}", self.data_type))?; + self.v_field(idx) + } + _ => Err(self.type_err("ROW")), + } } } @@ -3751,6 +3239,279 @@ pub struct ArrayWriterInner { num_elements: usize, } +// ============================================================================ +// MapWriterInner — opaque map builder (mirrors ArrayWriterInner) +// ============================================================================ + +pub struct MapWriterInner { + writer: Option, + completed: Option, + key_type: fcore::metadata::DataType, + value_type: fcore::metadata::DataType, + pending_key: Option>, + pending_value: Option>, +} + +fn ts_ntz_from(millis: i64, nanos: i32) -> fcore::row::TimestampNtz { + fcore::row::TimestampNtz::from_millis_nanos(millis, nanos) + .unwrap_or_else(|_| fcore::row::TimestampNtz::new(millis)) +} + +fn ts_ltz_from(millis: i64, nanos: i32) -> fcore::row::TimestampLtz { + fcore::row::TimestampLtz::from_millis_nanos(millis, nanos) + .unwrap_or_else(|_| fcore::row::TimestampLtz::new(millis)) +} + +/// Parse a decimal string into a `Datum::Decimal` matching `dt`'s precision and +/// scale. Used by the map key/value decimal setters, which carry no schema. +fn parse_decimal_datum( + dt: &fcore::metadata::DataType, + val: &str, +) -> Result, String> { + match dt { + fcore::metadata::DataType::Decimal(d) => { + let bd = bigdecimal::BigDecimal::from_str(val).map_err(|e| e.to_string())?; + let dec = fcore::row::Decimal::from_big_decimal(bd, d.precision(), d.scale()) + .map_err(|e| e.to_string())?; + Ok(fcore::row::Datum::Decimal(dec)) + } + other => Err(format!( + "decimal setter used on non-DECIMAL map type {other}" + )), + } +} + +fn timestamp_datum( + dt: &fcore::metadata::DataType, + millis: i64, + nanos: i32, +) -> Result, String> { + match dt { + fcore::metadata::DataType::Timestamp(_) => { + Ok(fcore::row::Datum::TimestampNtz(ts_ntz_from(millis, nanos))) + } + fcore::metadata::DataType::TimestampLTz(_) => { + Ok(fcore::row::Datum::TimestampLtz(ts_ltz_from(millis, nanos))) + } + other => Err(format!( + "timestamp setter used on non-TIMESTAMP map type {other}" + )), + } +} + +#[allow(clippy::too_many_arguments)] +fn new_map_writer( + capacity: usize, + key_leaf_type_id: i32, + key_precision: u32, + key_scale: u32, + value_leaf_type_id: i32, + value_precision: u32, + value_scale: u32, + value_array_nesting: u32, +) -> Result, String> { + let key_type = types::element_type_from_ffi(key_leaf_type_id, key_precision, key_scale, 0) + .map_err(|e| e.to_string())?; + let value_type = types::element_type_from_ffi( + value_leaf_type_id, + value_precision, + value_scale, + value_array_nesting, + ) + .map_err(|e| e.to_string())?; + let writer = fcore::row::binary_map::FlussMapWriter::new(capacity, &key_type, &value_type); + Ok(Box::new(MapWriterInner { + writer: Some(writer), + completed: None, + key_type, + value_type, + pending_key: None, + pending_value: None, + })) +} + +fn new_map_writer_arrow( + capacity: usize, + kv_schema_ptr: usize, +) -> Result, String> { + let mut types = + unsafe { types::arrow_ffi_to_data_types(kv_schema_ptr) }.map_err(|e| e.to_string())?; + if types.len() != 2 { + return Err(format!( + "map writer Arrow schema must have exactly 2 fields (key, value), got {}", + types.len() + )); + } + let value_type = types.swap_remove(1); + let key_type = types.swap_remove(0); + let writer = fcore::row::binary_map::FlussMapWriter::new(capacity, &key_type, &value_type); + Ok(Box::new(MapWriterInner { + writer: Some(writer), + completed: None, + key_type, + value_type, + pending_key: None, + pending_value: None, + })) +} + +impl MapWriterInner { + fn writer_mut(&mut self) -> Result<&mut fcore::row::binary_map::FlussMapWriter, String> { + self.writer + .as_mut() + .ok_or_else(|| "MapWriter is already finalized".to_string()) + } + + fn mw_key_bool(&mut self, val: bool) -> Result<(), String> { + self.pending_key = Some(fcore::row::Datum::Bool(val)); + Ok(()) + } + fn mw_key_i32(&mut self, val: i32) -> Result<(), String> { + self.pending_key = Some(fcore::row::Datum::Int32(val)); + Ok(()) + } + fn mw_key_i64(&mut self, val: i64) -> Result<(), String> { + self.pending_key = Some(fcore::row::Datum::Int64(val)); + Ok(()) + } + fn mw_key_f32(&mut self, val: f32) -> Result<(), String> { + self.pending_key = Some(fcore::row::Datum::Float32(val.into())); + Ok(()) + } + fn mw_key_f64(&mut self, val: f64) -> Result<(), String> { + self.pending_key = Some(fcore::row::Datum::Float64(val.into())); + Ok(()) + } + fn mw_key_str(&mut self, val: &str) -> Result<(), String> { + self.pending_key = Some(fcore::row::Datum::String(std::borrow::Cow::Owned( + val.to_string(), + ))); + Ok(()) + } + fn mw_key_bytes(&mut self, val: &[u8]) -> Result<(), String> { + self.pending_key = Some(fcore::row::Datum::Blob(std::borrow::Cow::Owned( + val.to_vec(), + ))); + Ok(()) + } + fn mw_key_date(&mut self, days: i32) -> Result<(), String> { + self.pending_key = Some(fcore::row::Datum::Date(fcore::row::Date::new(days))); + Ok(()) + } + fn mw_key_time(&mut self, millis: i32) -> Result<(), String> { + self.pending_key = Some(fcore::row::Datum::Time(fcore::row::Time::new(millis))); + Ok(()) + } + fn mw_key_timestamp(&mut self, millis: i64, nanos: i32) -> Result<(), String> { + self.pending_key = Some(timestamp_datum(&self.key_type, millis, nanos)?); + Ok(()) + } + fn mw_key_decimal_str(&mut self, val: &str) -> Result<(), String> { + self.pending_key = Some(parse_decimal_datum(&self.key_type, val)?); + Ok(()) + } + + fn mw_value_null(&mut self) -> Result<(), String> { + self.pending_value = Some(fcore::row::Datum::Null); + Ok(()) + } + fn mw_value_bool(&mut self, val: bool) -> Result<(), String> { + self.pending_value = Some(fcore::row::Datum::Bool(val)); + Ok(()) + } + fn mw_value_i32(&mut self, val: i32) -> Result<(), String> { + self.pending_value = Some(fcore::row::Datum::Int32(val)); + Ok(()) + } + fn mw_value_i64(&mut self, val: i64) -> Result<(), String> { + self.pending_value = Some(fcore::row::Datum::Int64(val)); + Ok(()) + } + fn mw_value_f32(&mut self, val: f32) -> Result<(), String> { + self.pending_value = Some(fcore::row::Datum::Float32(val.into())); + Ok(()) + } + fn mw_value_f64(&mut self, val: f64) -> Result<(), String> { + self.pending_value = Some(fcore::row::Datum::Float64(val.into())); + Ok(()) + } + fn mw_value_str(&mut self, val: &str) -> Result<(), String> { + self.pending_value = Some(fcore::row::Datum::String(std::borrow::Cow::Owned( + val.to_string(), + ))); + Ok(()) + } + fn mw_value_bytes(&mut self, val: &[u8]) -> Result<(), String> { + self.pending_value = Some(fcore::row::Datum::Blob(std::borrow::Cow::Owned( + val.to_vec(), + ))); + Ok(()) + } + fn mw_value_date(&mut self, days: i32) -> Result<(), String> { + self.pending_value = Some(fcore::row::Datum::Date(fcore::row::Date::new(days))); + Ok(()) + } + fn mw_value_time(&mut self, millis: i32) -> Result<(), String> { + self.pending_value = Some(fcore::row::Datum::Time(fcore::row::Time::new(millis))); + Ok(()) + } + fn mw_value_timestamp(&mut self, millis: i64, nanos: i32) -> Result<(), String> { + self.pending_value = Some(timestamp_datum(&self.value_type, millis, nanos)?); + Ok(()) + } + fn mw_value_decimal_str(&mut self, val: &str) -> Result<(), String> { + self.pending_value = Some(parse_decimal_datum(&self.value_type, val)?); + Ok(()) + } + fn mw_value_row(&mut self, row: &mut GenericRowInner) -> Result<(), String> { + let nested = std::mem::replace(&mut row.row, fcore::row::GenericRow::new(0)); + self.pending_value = Some(fcore::row::Datum::Row(Box::new(nested))); + Ok(()) + } + fn mw_value_map(&mut self, map: &mut MapWriterInner) -> Result<(), String> { + map.complete_if_needed()?; + let completed = map + .completed + .take() + .ok_or_else(|| "MapWriter invariant violation: nested map missing".to_string())?; + self.pending_value = Some(fcore::row::Datum::Map(completed)); + Ok(()) + } + fn mw_value_array(&mut self, array: &mut ArrayWriterInner) -> Result<(), String> { + array.complete_if_needed()?; + let completed = array + .completed + .take() + .ok_or_else(|| "ArrayWriter invariant violation: nested array missing".to_string())?; + self.pending_value = Some(fcore::row::Datum::Array(completed)); + Ok(()) + } + + /// Commit the pending key/value as one map entry. Keys cannot be null; + /// an unset value defaults to null. + fn mw_commit(&mut self) -> Result<(), String> { + let key = self + .pending_key + .take() + .ok_or_else(|| "MapWriter: entry key not set before commit".to_string())?; + let value = self.pending_value.take().unwrap_or(fcore::row::Datum::Null); + self.writer_mut()? + .write_entry(key, value) + .map_err(|e| e.to_string()) + } + + fn complete_if_needed(&mut self) -> Result<(), String> { + if self.completed.is_none() { + let w = self + .writer + .take() + .ok_or_else(|| "MapWriter has already been finalized".to_string())?; + self.completed = Some(w.complete().map_err(|e| e.to_string())?); + } + Ok(()) + } +} + fn new_array_writer( size: usize, element_leaf_type_id: i32, @@ -3770,6 +3531,26 @@ fn new_array_writer( })) } +fn new_array_writer_arrow( + size: usize, + element_schema_ptr: usize, +) -> Result, String> { + let mut types = + unsafe { types::arrow_ffi_to_data_types(element_schema_ptr) }.map_err(|e| e.to_string())?; + let element_type = if types.is_empty() { + return Err("array element Arrow schema must have one field".to_string()); + } else { + types.swap_remove(0) + }; + let writer = fcore::row::binary_array::FlussArrayWriter::new(size, &element_type); + Ok(Box::new(ArrayWriterInner { + writer: Some(writer), + completed: None, + element_type, + num_elements: size, + })) +} + impl ArrayWriterInner { fn writer_mut(&mut self) -> Result<&mut fcore::row::binary_array::FlussArrayWriter, String> { self.writer @@ -4027,6 +3808,36 @@ impl ArrayWriterInner { self.writer_mut()?.write_array(idx, arr); Ok(()) } + + fn aw_set_row(&mut self, idx: usize, row: &mut GenericRowInner) -> Result<(), String> { + self.ensure_writable(idx)?; + if !matches!(self.element_type, fcore::metadata::DataType::Row(_)) { + return Err(format!( + "ArrayWriter type mismatch: expected ROW element, got {}", + self.element_type + )); + } + self.writer_mut()? + .write_row(idx, &row.row) + .map_err(|e| e.to_string()) + } + + fn aw_set_map(&mut self, idx: usize, map: &mut MapWriterInner) -> Result<(), String> { + self.ensure_writable(idx)?; + if !matches!(self.element_type, fcore::metadata::DataType::Map(_)) { + return Err(format!( + "ArrayWriter type mismatch: expected MAP element, got {}", + self.element_type + )); + } + map.complete_if_needed()?; + let completed = map.completed.as_ref().ok_or_else(|| { + "ArrayWriter invariant violation: nested completed map missing after finalize" + .to_string() + })?; + self.writer_mut()?.write_map(idx, completed); + Ok(()) + } } /// Structural type equivalence that ignores nullability flags but preserves @@ -4035,7 +3846,6 @@ impl ArrayWriterInner { /// because the Rust-side element type is always reconstructed as nullable /// (encoding doesn't depend on it). fn structurally_compatible(a: &fcore::metadata::DataType, b: &fcore::metadata::DataType) -> bool { - use fcore::metadata::DataType; match (a, b) { (DataType::Boolean(_), DataType::Boolean(_)) | (DataType::TinyInt(_), DataType::TinyInt(_)) diff --git a/bindings/cpp/src/table.cpp b/bindings/cpp/src/table.cpp index 9b5d087e..abde4a46 100644 --- a/bindings/cpp/src/table.cpp +++ b/bindings/cpp/src/table.cpp @@ -26,10 +26,12 @@ #include "fluss.hpp" #include "lib.rs.h" #include "rust/cxx.h" +#include "type_lowering.hpp" // todo: bindings/cpp/BUILD.bazel still doesn't declare Arrow include/link dependencies. // In environments where Bazel does not already have Arrow available, this will fail at compile/link // time. #include +#include namespace fluss { @@ -86,11 +88,38 @@ int Date::Day() const { if (!inner_) throw std::logic_error(name ": not available (moved-from or null)"); \ } while (0) +namespace { + +ffi::ArrayWriterInner* make_array_writer_arrow(size_t size, + std::shared_ptr element, + bool nullable) { + auto schema = arrow::schema({arrow::field("element", std::move(element), nullable)}); + return ffi::new_array_writer_arrow(size, detail::export_arrow_schema(*schema)).into_raw(); +} + +ffi::MapWriterInner* make_map_writer_arrow(size_t capacity, std::shared_ptr key, + std::shared_ptr value, + bool value_nullable) { + auto schema = arrow::schema({ + arrow::field("key", std::move(key), /*nullable=*/false), + arrow::field("value", std::move(value), value_nullable), + }); + return ffi::new_map_writer_arrow(capacity, detail::export_arrow_schema(*schema)).into_raw(); +} + +} // namespace + // ============================================================================ // ArrayWriter — builder for array values backed by Rust ArrayWriterInner // ============================================================================ ArrayWriter::ArrayWriter(size_t size, DataType element_type) : element_type_(std::move(element_type)) { + if (detail::is_compound(element_type_)) { + inner_ = make_array_writer_arrow(size, detail::to_arrow_type(element_type_), + element_type_.nullable()); + return; + } + auto flat = utils::flatten_array_type(element_type_); int32_t leaf_type_id = flat.nesting > 0 ? flat.leaf_type : static_cast(element_type_.id()); uint32_t leaf_precision = static_cast(flat.nesting > 0 ? flat.leaf_precision @@ -102,6 +131,11 @@ ArrayWriter::ArrayWriter(size_t size, DataType element_type) : element_type_(std inner_ = box.into_raw(); } +ArrayWriter::ArrayWriter(size_t size, std::shared_ptr element_type) + : element_type_(DataType::Int()) { // placeholder; Rust holds the real element type + inner_ = make_array_writer_arrow(size, std::move(element_type), /*nullable=*/true); +} + ArrayWriter::~ArrayWriter() noexcept { Destroy(); } void ArrayWriter::Destroy() noexcept { @@ -190,121 +224,252 @@ void ArrayWriter::SetArray(size_t idx, ArrayWriter&& nested) { nested.Destroy(); } +void ArrayWriter::SetRow(size_t idx, GenericRow&& row) { + CHECK_AW("ArrayWriter"); + if (!row.inner_) { + throw std::logic_error("ArrayWriter::SetRow: nested row not available"); + } + inner_->aw_set_row(idx, *row.inner_); +} + +void ArrayWriter::SetMap(size_t idx, MapWriter&& map) { + CHECK_AW("ArrayWriter"); + if (!map.inner_) { + throw std::logic_error("ArrayWriter::SetMap: nested map not available"); + } + inner_->aw_set_map(idx, *map.inner_); + map.Destroy(); +} + // ============================================================================ -// ArrayView — read-only recursive view into an array column value +// MapWriter — builder for map values backed by Rust MapWriterInner // ============================================================================ -ArrayView::~ArrayView() noexcept { Destroy(); } +MapWriter::MapWriter(size_t capacity, DataType key_type, DataType value_type) + : key_type_(std::move(key_type)), value_type_(std::move(value_type)) { + if (detail::is_compound(key_type_) || detail::is_compound(value_type_)) { + inner_ = make_map_writer_arrow(capacity, detail::to_arrow_type(key_type_), + detail::to_arrow_type(value_type_), value_type_.nullable()); + return; + } + + // Keys are scalar; values may be a (possibly nested) ARRAY, so flatten them. + auto value_flat = utils::flatten_array_type(value_type_); + int32_t value_leaf = value_flat.nesting > 0 ? value_flat.leaf_type + : static_cast(value_type_.id()); + uint32_t value_precision = static_cast( + value_flat.nesting > 0 ? value_flat.leaf_precision : value_type_.precision()); + uint32_t value_scale = static_cast( + value_flat.nesting > 0 ? value_flat.leaf_scale : value_type_.scale()); + uint32_t value_nesting = static_cast(value_flat.nesting); + + auto box = ffi::new_map_writer(capacity, + static_cast(key_type_.id()), + static_cast(key_type_.precision()), + static_cast(key_type_.scale()), + value_leaf, value_precision, value_scale, value_nesting); + inner_ = box.into_raw(); +} + +MapWriter::MapWriter(size_t capacity, std::shared_ptr key_type, + std::shared_ptr value_type) + : key_type_(DataType::Int()), value_type_(DataType::Int()) { + // Placeholders above; Rust holds the real key/value types. + inner_ = make_map_writer_arrow(capacity, std::move(key_type), std::move(value_type), + /*nullable=*/true); +} + +MapWriter::~MapWriter() noexcept { Destroy(); } -void ArrayView::Destroy() noexcept { +void MapWriter::Destroy() noexcept { if (inner_) { - rust::Box::from_raw(inner_); + rust::Box::from_raw(inner_); inner_ = nullptr; } } -ArrayView::ArrayView(ArrayView&& other) noexcept : inner_(other.inner_) { other.inner_ = nullptr; } +MapWriter::MapWriter(MapWriter&& other) noexcept + : inner_(other.inner_), + key_type_(std::move(other.key_type_)), + value_type_(std::move(other.value_type_)) { + other.inner_ = nullptr; +} -ArrayView& ArrayView::operator=(ArrayView&& other) noexcept { +MapWriter& MapWriter::operator=(MapWriter&& other) noexcept { if (this != &other) { Destroy(); inner_ = other.inner_; + key_type_ = std::move(other.key_type_); + value_type_ = std::move(other.value_type_); other.inner_ = nullptr; } return *this; } -// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) -#define CHECK_AV() \ - do { \ - if (!inner_) throw std::logic_error("ArrayView: not available (moved-from)"); \ - } while (0) +bool MapWriter::Available() const { return inner_ != nullptr; } -size_t ArrayView::Size() const noexcept { - assert(inner_ && "ArrayView::Size called on moved-from instance"); - return inner_->av_size(); +void MapWriter::SetKeyBool(bool k) { CHECK_AW("MapWriter"); inner_->mw_key_bool(k); } +void MapWriter::SetKeyInt32(int32_t k) { CHECK_AW("MapWriter"); inner_->mw_key_i32(k); } +void MapWriter::SetKeyInt64(int64_t k) { CHECK_AW("MapWriter"); inner_->mw_key_i64(k); } +void MapWriter::SetKeyFloat32(float k) { CHECK_AW("MapWriter"); inner_->mw_key_f32(k); } +void MapWriter::SetKeyFloat64(double k) { CHECK_AW("MapWriter"); inner_->mw_key_f64(k); } +void MapWriter::SetKeyString(const std::string& k) { CHECK_AW("MapWriter"); inner_->mw_key_str(k); } +void MapWriter::SetKeyBytes(const std::vector& k) { + CHECK_AW("MapWriter"); + inner_->mw_key_bytes(rust::Slice(k.data(), k.size())); } - -TypeId ArrayView::ElementType() const noexcept { - assert(inner_ && "ArrayView::ElementType called on moved-from instance"); - return static_cast(inner_->av_element_type_id()); +void MapWriter::SetKeyDate(fluss::Date k) { CHECK_AW("MapWriter"); inner_->mw_key_date(k.days_since_epoch); } +void MapWriter::SetKeyTime(fluss::Time k) { + CHECK_AW("MapWriter"); + inner_->mw_key_time(k.millis_since_midnight); } - -bool ArrayView::IsNull(size_t element) const { - CHECK_AV(); - return inner_->av_is_null(element); +void MapWriter::SetKeyTimestamp(fluss::Timestamp k) { + CHECK_AW("MapWriter"); + inner_->mw_key_timestamp(k.epoch_millis, k.nano_of_millisecond); } +void MapWriter::SetKeyDecimal(const std::string& k) { CHECK_AW("MapWriter"); inner_->mw_key_decimal_str(k); } -bool ArrayView::GetBool(size_t element) const { - CHECK_AV(); - return inner_->av_get_bool(element); +void MapWriter::SetValueNull() { CHECK_AW("MapWriter"); inner_->mw_value_null(); } +void MapWriter::SetValueBool(bool v) { CHECK_AW("MapWriter"); inner_->mw_value_bool(v); } +void MapWriter::SetValueInt32(int32_t v) { CHECK_AW("MapWriter"); inner_->mw_value_i32(v); } +void MapWriter::SetValueInt64(int64_t v) { CHECK_AW("MapWriter"); inner_->mw_value_i64(v); } +void MapWriter::SetValueFloat32(float v) { CHECK_AW("MapWriter"); inner_->mw_value_f32(v); } +void MapWriter::SetValueFloat64(double v) { CHECK_AW("MapWriter"); inner_->mw_value_f64(v); } +void MapWriter::SetValueString(const std::string& v) { CHECK_AW("MapWriter"); inner_->mw_value_str(v); } +void MapWriter::SetValueBytes(const std::vector& v) { + CHECK_AW("MapWriter"); + inner_->mw_value_bytes(rust::Slice(v.data(), v.size())); } - -int32_t ArrayView::GetInt32(size_t element) const { - CHECK_AV(); - return inner_->av_get_i32(element); +void MapWriter::SetValueDate(fluss::Date v) { CHECK_AW("MapWriter"); inner_->mw_value_date(v.days_since_epoch); } +void MapWriter::SetValueTime(fluss::Time v) { + CHECK_AW("MapWriter"); + inner_->mw_value_time(v.millis_since_midnight); } - -int64_t ArrayView::GetInt64(size_t element) const { - CHECK_AV(); - return inner_->av_get_i64(element); +void MapWriter::SetValueTimestamp(fluss::Timestamp v) { + CHECK_AW("MapWriter"); + inner_->mw_value_timestamp(v.epoch_millis, v.nano_of_millisecond); } - -float ArrayView::GetFloat32(size_t element) const { - CHECK_AV(); - return inner_->av_get_f32(element); +void MapWriter::SetValueDecimal(const std::string& v) { + CHECK_AW("MapWriter"); + inner_->mw_value_decimal_str(v); +} +void MapWriter::SetValueRow(GenericRow&& v) { + CHECK_AW("MapWriter"); + if (!v.inner_) { + throw std::logic_error("MapWriter::SetValueRow: nested row not available"); + } + inner_->mw_value_row(*v.inner_); +} +void MapWriter::SetValueMap(MapWriter&& v) { + CHECK_AW("MapWriter"); + if (!v.inner_) { + throw std::logic_error("MapWriter::SetValueMap: nested map not available"); + } + inner_->mw_value_map(*v.inner_); + v.Destroy(); } +void MapWriter::SetValueArray(ArrayWriter&& v) { + CHECK_AW("MapWriter"); + if (!v.inner_) { + throw std::logic_error("MapWriter::SetValueArray: nested array not available"); + } + inner_->mw_value_array(*v.inner_); + v.Destroy(); +} + +void MapWriter::Commit() { CHECK_AW("MapWriter"); inner_->mw_commit(); } + +// ============================================================================ +// Value — one recursive handle for complex (ARRAY/MAP/ROW) reads +// ============================================================================ -double ArrayView::GetFloat64(size_t element) const { - CHECK_AV(); - return inner_->av_get_f64(element); +Value::~Value() noexcept { Destroy(); } + +void Value::Destroy() noexcept { + if (inner_) { + rust::Box::from_raw(inner_); + inner_ = nullptr; + } } -std::string ArrayView::GetString(size_t element) const { - CHECK_AV(); - return std::string(inner_->av_get_str(element)); +Value::Value(Value&& other) noexcept : inner_(other.inner_) { other.inner_ = nullptr; } + +Value& Value::operator=(Value&& other) noexcept { + if (this != &other) { + Destroy(); + inner_ = other.inner_; + other.inner_ = nullptr; + } + return *this; } -std::vector ArrayView::GetBytes(size_t element) const { - CHECK_AV(); - auto rv = inner_->av_get_bytes(element); +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define CHECK_VALUE() \ + do { \ + if (!inner_) throw std::logic_error("Value: not available (moved-from)"); \ + } while (0) + +TypeId Value::Type() const noexcept { + assert(inner_ && "Value::Type called on moved-from instance"); + return static_cast(inner_->v_type()); +} +bool Value::IsNull() const noexcept { + assert(inner_ && "Value::IsNull called on moved-from instance"); + return inner_->v_is_null(); +} +bool Value::GetBool() const { CHECK_VALUE(); return inner_->v_get_bool(); } +int32_t Value::GetInt32() const { CHECK_VALUE(); return inner_->v_get_i32(); } +int64_t Value::GetInt64() const { CHECK_VALUE(); return inner_->v_get_i64(); } +float Value::GetFloat32() const { CHECK_VALUE(); return inner_->v_get_f32(); } +double Value::GetFloat64() const { CHECK_VALUE(); return inner_->v_get_f64(); } +std::string Value::GetString() const { CHECK_VALUE(); return std::string(inner_->v_get_str()); } +std::vector Value::GetBytes() const { + CHECK_VALUE(); + auto rv = inner_->v_get_bytes(); return {rv.data(), rv.data() + rv.size()}; } - -fluss::Date ArrayView::GetDate(size_t element) const { - CHECK_AV(); - return fluss::Date{inner_->av_get_date_days(element)}; +fluss::Date Value::GetDate() const { CHECK_VALUE(); return fluss::Date{inner_->v_get_date_days()}; } +fluss::Time Value::GetTime() const { + CHECK_VALUE(); + return fluss::Time{inner_->v_get_time_millis()}; } - -fluss::Time ArrayView::GetTime(size_t element) const { - CHECK_AV(); - return fluss::Time{inner_->av_get_time_millis(element)}; +fluss::Timestamp Value::GetTimestamp() const { + CHECK_VALUE(); + return fluss::Timestamp{inner_->v_get_ts_millis(), inner_->v_get_ts_nanos()}; } - -fluss::Timestamp ArrayView::GetTimestampNtz(size_t element) const { - CHECK_AV(); - return fluss::Timestamp{inner_->av_get_ts_millis(element), - inner_->av_get_ts_nanos(element)}; +std::string Value::GetDecimalString() const { + CHECK_VALUE(); + return std::string(inner_->v_get_decimal_str()); } - -fluss::Timestamp ArrayView::GetTimestampLtz(size_t element) const { - CHECK_AV(); - return fluss::Timestamp{inner_->av_get_ts_millis(element), - inner_->av_get_ts_nanos(element)}; +size_t Value::Size() const { CHECK_VALUE(); return inner_->v_size(); } +size_t Value::FieldCount() const { CHECK_VALUE(); return inner_->v_field_count(); } +Value Value::At(size_t i) const { + CHECK_VALUE(); + auto box = inner_->v_at(i); + return Value(box.into_raw()); } - -std::string ArrayView::GetDecimalString(size_t element) const { - CHECK_AV(); - return std::string(inner_->av_get_decimal_str(element)); +Value Value::KeyAt(size_t i) const { + CHECK_VALUE(); + auto box = inner_->v_key_at(i); + return Value(box.into_raw()); } - -ArrayView ArrayView::GetArray(size_t element) const { - CHECK_AV(); - auto box = inner_->av_get_nested(element); - return ArrayView(box.into_raw()); +Value Value::ValueAt(size_t i) const { + CHECK_VALUE(); + auto box = inner_->v_value_at(i); + return Value(box.into_raw()); +} +Value Value::Field(size_t i) const { + CHECK_VALUE(); + auto box = inner_->v_field(i); + return Value(box.into_raw()); +} +Value Value::Field(const std::string& name) const { + CHECK_VALUE(); + auto box = inner_->v_field_by_name(name); + return Value(box.into_raw()); } -#undef CHECK_AV +#undef CHECK_VALUE // ============================================================================ // GenericRow — write-only row backed by opaque Rust GenericRowInner @@ -421,6 +586,25 @@ void GenericRow::SetArray(size_t idx, ArrayWriter&& writer) { writer.Destroy(); } +void GenericRow::SetMap(size_t idx, MapWriter&& writer) { + CHECK_INNER("GenericRow"); + if (!writer.inner_) { + throw std::logic_error("GenericRow::SetMap: MapWriter not available"); + } + inner_->gr_set_map(idx, *writer.inner_); + writer.Destroy(); +} + +void GenericRow::SetRow(size_t idx, GenericRow&& nested) { + CHECK_INNER("GenericRow"); + if (!nested.inner_) { + throw std::logic_error("GenericRow::SetRow: nested row not available"); + } + // gr_set_row moves the nested row's contents out; the emptied nested + // handle is freed by `nested`'s own destructor at end of scope. + inner_->gr_set_row(idx, *nested.inner_); +} + // ============================================================================ // ScanData — destructor must live in .cpp where rust::Box is visible // ============================================================================ @@ -514,84 +698,13 @@ std::string RowView::GetDecimalString(size_t idx) const { return std::string(data_->raw->sv_get_decimal_str(bucket_idx_, rec_idx_, idx)); } -size_t RowView::GetArraySize(size_t idx) const { - CHECK_DATA("RowView"); - return data_->raw->sv_get_array_size(bucket_idx_, rec_idx_, idx); -} - -TypeId RowView::GetArrayElementType(size_t idx) const { - CHECK_DATA("RowView"); - return static_cast(data_->raw->sv_get_array_element_type(idx)); -} - -bool RowView::IsArrayElementNull(size_t idx, size_t element) const { +Value RowView::GetValue(size_t idx) const { CHECK_DATA("RowView"); - return data_->raw->sv_get_array_is_null(bucket_idx_, rec_idx_, idx, element); + auto box = data_->raw->sv_get_value(bucket_idx_, rec_idx_, idx); + return Value(box.into_raw()); } -bool RowView::GetArrayBool(size_t idx, size_t element) const { - CHECK_DATA("RowView"); - return data_->raw->sv_get_array_bool(bucket_idx_, rec_idx_, idx, element); -} - -int32_t RowView::GetArrayInt32(size_t idx, size_t element) const { - CHECK_DATA("RowView"); - return data_->raw->sv_get_array_i32(bucket_idx_, rec_idx_, idx, element); -} - -int64_t RowView::GetArrayInt64(size_t idx, size_t element) const { - CHECK_DATA("RowView"); - return data_->raw->sv_get_array_i64(bucket_idx_, rec_idx_, idx, element); -} - -float RowView::GetArrayFloat32(size_t idx, size_t element) const { - CHECK_DATA("RowView"); - return data_->raw->sv_get_array_f32(bucket_idx_, rec_idx_, idx, element); -} - -double RowView::GetArrayFloat64(size_t idx, size_t element) const { - CHECK_DATA("RowView"); - return data_->raw->sv_get_array_f64(bucket_idx_, rec_idx_, idx, element); -} - -std::string RowView::GetArrayString(size_t idx, size_t element) const { - CHECK_DATA("RowView"); - return std::string(data_->raw->sv_get_array_str(bucket_idx_, rec_idx_, idx, element)); -} - -std::vector RowView::GetArrayBytes(size_t idx, size_t element) const { - CHECK_DATA("RowView"); - auto rv = data_->raw->sv_get_array_bytes(bucket_idx_, rec_idx_, idx, element); - return {rv.data(), rv.data() + rv.size()}; -} - -fluss::Date RowView::GetArrayDate(size_t idx, size_t element) const { - CHECK_DATA("RowView"); - return fluss::Date{data_->raw->sv_get_array_date_days(bucket_idx_, rec_idx_, idx, element)}; -} - -fluss::Time RowView::GetArrayTime(size_t idx, size_t element) const { - CHECK_DATA("RowView"); - return fluss::Time{data_->raw->sv_get_array_time_millis(bucket_idx_, rec_idx_, idx, element)}; -} - -fluss::Timestamp RowView::GetArrayTimestamp(size_t idx, size_t element) const { - CHECK_DATA("RowView"); - auto millis = data_->raw->sv_get_array_ts_millis(bucket_idx_, rec_idx_, idx, element); - auto nanos = data_->raw->sv_get_array_ts_nanos(bucket_idx_, rec_idx_, idx, element); - return fluss::Timestamp{millis, nanos}; -} - -std::string RowView::GetArrayDecimalString(size_t idx, size_t element) const { - CHECK_DATA("RowView"); - return std::string(data_->raw->sv_get_array_decimal_str(bucket_idx_, rec_idx_, idx, element)); -} - -ArrayView RowView::GetArrayView(size_t idx) const { - CHECK_DATA("RowView"); - auto box = data_->raw->sv_get_array_view(bucket_idx_, rec_idx_, idx); - return ArrayView(box.into_raw()); -} +Value RowView::GetValue(const std::string& name) const { return GetValue(Resolve(name)); } // ============================================================================ // PrefixLookupResult / PrefixRowView — read path for prefix lookups @@ -659,70 +772,12 @@ std::string PrefixRowView::GetDecimalString(size_t idx) const { return std::string(data_->raw->plv_get_decimal_str(rec_idx_, idx)); } -size_t PrefixRowView::GetArraySize(size_t idx) const { +Value PrefixRowView::GetValue(size_t idx) const { CHECK_DATA("PrefixRowView"); - return data_->raw->plv_get_array_size(rec_idx_, idx); -} -TypeId PrefixRowView::GetArrayElementType(size_t idx) const { - CHECK_DATA("PrefixRowView"); - return static_cast(data_->raw->plv_get_array_element_type(rec_idx_, idx)); -} -bool PrefixRowView::IsArrayElementNull(size_t idx, size_t element) const { - CHECK_DATA("PrefixRowView"); - return data_->raw->plv_get_array_is_null(rec_idx_, idx, element); -} -bool PrefixRowView::GetArrayBool(size_t idx, size_t element) const { - CHECK_DATA("PrefixRowView"); - return data_->raw->plv_get_array_bool(rec_idx_, idx, element); -} -int32_t PrefixRowView::GetArrayInt32(size_t idx, size_t element) const { - CHECK_DATA("PrefixRowView"); - return data_->raw->plv_get_array_i32(rec_idx_, idx, element); -} -int64_t PrefixRowView::GetArrayInt64(size_t idx, size_t element) const { - CHECK_DATA("PrefixRowView"); - return data_->raw->plv_get_array_i64(rec_idx_, idx, element); -} -float PrefixRowView::GetArrayFloat32(size_t idx, size_t element) const { - CHECK_DATA("PrefixRowView"); - return data_->raw->plv_get_array_f32(rec_idx_, idx, element); -} -double PrefixRowView::GetArrayFloat64(size_t idx, size_t element) const { - CHECK_DATA("PrefixRowView"); - return data_->raw->plv_get_array_f64(rec_idx_, idx, element); -} -std::string PrefixRowView::GetArrayString(size_t idx, size_t element) const { - CHECK_DATA("PrefixRowView"); - return std::string(data_->raw->plv_get_array_str(rec_idx_, idx, element)); -} -std::vector PrefixRowView::GetArrayBytes(size_t idx, size_t element) const { - CHECK_DATA("PrefixRowView"); - auto rv = data_->raw->plv_get_array_bytes(rec_idx_, idx, element); - return {rv.data(), rv.data() + rv.size()}; -} -fluss::Date PrefixRowView::GetArrayDate(size_t idx, size_t element) const { - CHECK_DATA("PrefixRowView"); - return fluss::Date{data_->raw->plv_get_array_date_days(rec_idx_, idx, element)}; -} -fluss::Time PrefixRowView::GetArrayTime(size_t idx, size_t element) const { - CHECK_DATA("PrefixRowView"); - return fluss::Time{data_->raw->plv_get_array_time_millis(rec_idx_, idx, element)}; -} -fluss::Timestamp PrefixRowView::GetArrayTimestamp(size_t idx, size_t element) const { - CHECK_DATA("PrefixRowView"); - auto millis = data_->raw->plv_get_array_ts_millis(rec_idx_, idx, element); - auto nanos = data_->raw->plv_get_array_ts_nanos(rec_idx_, idx, element); - return fluss::Timestamp{millis, nanos}; -} -std::string PrefixRowView::GetArrayDecimalString(size_t idx, size_t element) const { - CHECK_DATA("PrefixRowView"); - return std::string(data_->raw->plv_get_array_decimal_str(rec_idx_, idx, element)); -} -ArrayView PrefixRowView::GetArrayView(size_t idx) const { - CHECK_DATA("PrefixRowView"); - auto box = data_->raw->plv_get_array_view(rec_idx_, idx); - return ArrayView(box.into_raw()); + auto box = data_->raw->plv_get_value(rec_idx_, idx); + return Value(box.into_raw()); } +Value PrefixRowView::GetValue(const std::string& name) const { return GetValue(Resolve(name)); } // ============================================================================ // ScanRecords — backed by opaque Rust ScanResultInner @@ -930,83 +985,14 @@ std::string LookupResult::GetDecimalString(size_t idx) const { return std::string(inner_->lv_get_decimal_str(idx)); } -size_t LookupResult::GetArraySize(size_t idx) const { - CHECK_INNER("LookupResult"); - return inner_->lv_get_array_size(idx); -} - -TypeId LookupResult::GetArrayElementType(size_t idx) const { - CHECK_INNER("LookupResult"); - return static_cast(inner_->lv_get_array_element_type(idx)); -} - -bool LookupResult::IsArrayElementNull(size_t idx, size_t element) const { - CHECK_INNER("LookupResult"); - return inner_->lv_get_array_is_null(idx, element); -} - -bool LookupResult::GetArrayBool(size_t idx, size_t element) const { - CHECK_INNER("LookupResult"); - return inner_->lv_get_array_bool(idx, element); -} - -int32_t LookupResult::GetArrayInt32(size_t idx, size_t element) const { - CHECK_INNER("LookupResult"); - return inner_->lv_get_array_i32(idx, element); -} - -int64_t LookupResult::GetArrayInt64(size_t idx, size_t element) const { - CHECK_INNER("LookupResult"); - return inner_->lv_get_array_i64(idx, element); -} - -float LookupResult::GetArrayFloat32(size_t idx, size_t element) const { +Value LookupResult::GetValue(size_t idx) const { CHECK_INNER("LookupResult"); - return inner_->lv_get_array_f32(idx, element); + auto box = inner_->lv_get_value(idx); + return Value(box.into_raw()); } -double LookupResult::GetArrayFloat64(size_t idx, size_t element) const { - CHECK_INNER("LookupResult"); - return inner_->lv_get_array_f64(idx, element); -} - -std::string LookupResult::GetArrayString(size_t idx, size_t element) const { - CHECK_INNER("LookupResult"); - return std::string(inner_->lv_get_array_str(idx, element)); -} - -std::vector LookupResult::GetArrayBytes(size_t idx, size_t element) const { - CHECK_INNER("LookupResult"); - auto rv = inner_->lv_get_array_bytes(idx, element); - return {rv.data(), rv.data() + rv.size()}; -} - -fluss::Date LookupResult::GetArrayDate(size_t idx, size_t element) const { - CHECK_INNER("LookupResult"); - return fluss::Date{inner_->lv_get_array_date_days(idx, element)}; -} - -fluss::Time LookupResult::GetArrayTime(size_t idx, size_t element) const { - CHECK_INNER("LookupResult"); - return fluss::Time{inner_->lv_get_array_time_millis(idx, element)}; -} - -fluss::Timestamp LookupResult::GetArrayTimestamp(size_t idx, size_t element) const { - CHECK_INNER("LookupResult"); - auto millis = inner_->lv_get_array_ts_millis(idx, element); - auto nanos = inner_->lv_get_array_ts_nanos(idx, element); - return fluss::Timestamp{millis, nanos}; -} - -std::string LookupResult::GetArrayDecimalString(size_t idx, size_t element) const { - CHECK_INNER("LookupResult"); - return std::string(inner_->lv_get_array_decimal_str(idx, element)); -} - -ArrayView LookupResult::GetArrayView(size_t idx) const { - CHECK_INNER("LookupResult"); - auto box = inner_->lv_get_array_view(idx); - return ArrayView(box.into_raw()); +Value LookupResult::GetValue(const std::string& name) const { + return GetValue(Resolve(name)); } // ============================================================================ diff --git a/bindings/cpp/src/type_lowering.hpp b/bindings/cpp/src/type_lowering.hpp new file mode 100644 index 00000000..98e67688 --- /dev/null +++ b/bindings/cpp/src/type_lowering.hpp @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include +#include +#include +#include + +#include "fluss.hpp" + +namespace fluss { +namespace detail { + +inline arrow::TimeUnit::type arrow_time_unit(int32_t precision) { + if (precision <= 0) return arrow::TimeUnit::SECOND; + if (precision <= 3) return arrow::TimeUnit::MILLI; + if (precision <= 6) return arrow::TimeUnit::MICRO; + return arrow::TimeUnit::NANO; +} + +/// Mirrors core's `to_arrow_type` so a lowered type round-trips through `from_arrow_field`. +inline std::shared_ptr to_arrow_type(const DataType& dt) { + switch (dt.id()) { + case TypeId::Boolean: + return arrow::boolean(); + case TypeId::TinyInt: + return arrow::int8(); + case TypeId::SmallInt: + return arrow::int16(); + case TypeId::Int: + return arrow::int32(); + case TypeId::BigInt: + return arrow::int64(); + case TypeId::Float: + return arrow::float32(); + case TypeId::Double: + return arrow::float64(); + case TypeId::String: + case TypeId::Char: + return arrow::utf8(); + case TypeId::Bytes: + return arrow::binary(); + case TypeId::Binary: + return arrow::fixed_size_binary(dt.precision()); + case TypeId::Decimal: + return arrow::decimal128(dt.precision(), dt.scale()); + case TypeId::Date: + return arrow::date32(); + case TypeId::Time: { + auto unit = arrow_time_unit(dt.precision()); + return (unit == arrow::TimeUnit::SECOND || unit == arrow::TimeUnit::MILLI) + ? arrow::time32(unit) + : arrow::time64(unit); + } + case TypeId::Timestamp: + return arrow::timestamp(arrow_time_unit(dt.precision())); + case TypeId::TimestampLtz: + return arrow::timestamp(arrow_time_unit(dt.precision()), "UTC"); + case TypeId::Array: { + const DataType* elem = dt.element_type(); + if (!elem) throw std::runtime_error("ARRAY DataType is missing its element type"); + return arrow::list(arrow::field("element", to_arrow_type(*elem), elem->nullable())); + } + case TypeId::Map: { + const DataType* k = dt.key_type(); + const DataType* v = dt.value_type(); + if (!k || !v) throw std::runtime_error("MAP DataType is missing its key/value type"); + return arrow::map(to_arrow_type(*k), to_arrow_type(*v)); + } + case TypeId::Row: { + std::vector> fields; + fields.reserve(dt.field_count()); + for (size_t i = 0; i < dt.field_count(); i++) { + const DataType* ft = dt.field_type(i); + fields.push_back( + arrow::field(dt.field_name(i), to_arrow_type(*ft), ft->nullable())); + } + return arrow::struct_(std::move(fields)); + } + default: + throw std::runtime_error("Unsupported DataType for Arrow lowering"); + } +} + +/// True if the type is (or nests) a MAP/ROW the flat FFI column can't express, +/// so it must route through Arrow. Array-of-scalar stays on the flat path. +inline bool is_compound(const DataType& dt) { + switch (dt.id()) { + case TypeId::Map: + case TypeId::Row: + return true; + case TypeId::Array: { + const DataType* elem = dt.element_type(); + return elem != nullptr && is_compound(*elem); + } + default: + return false; + } +} + +inline std::shared_ptr columns_to_arrow_schema(const std::vector& columns) { + std::vector> fields; + fields.reserve(columns.size()); + for (const auto& col : columns) { + fields.push_back( + arrow::field(col.name, to_arrow_type(col.data_type), col.data_type.nullable())); + } + return arrow::schema(std::move(fields)); +} + +/// Exports an Arrow schema to a heap FFI_ArrowSchema; the Rust bridge takes +/// ownership of the returned pointer. Throws on failure. +inline size_t export_arrow_schema(const arrow::Schema& schema) { + ArrowSchema c_schema; + auto status = arrow::ExportSchema(schema, &c_schema); + if (!status.ok()) { + throw std::runtime_error("Failed to export Arrow schema: " + status.ToString()); + } + return reinterpret_cast(new ArrowSchema(std::move(c_schema))); +} + +} // namespace detail +} // namespace fluss diff --git a/bindings/cpp/src/types.rs b/bindings/cpp/src/types.rs index 8836e25e..22a4adef 100644 --- a/bindings/cpp/src/types.rs +++ b/bindings/cpp/src/types.rs @@ -19,6 +19,7 @@ use crate::ffi; use anyhow::{Result, anyhow}; use arrow::array::Array; use arrow::ffi::{FFI_ArrowArray, FFI_ArrowSchema}; +use fcore::row::Datum; use fluss as fcore; use std::borrow::Cow; use std::str::FromStr; @@ -40,6 +41,8 @@ pub const DATA_TYPE_DECIMAL: i32 = 14; pub const DATA_TYPE_CHAR: i32 = 15; pub const DATA_TYPE_BINARY: i32 = 16; pub const DATA_TYPE_ARRAY: i32 = 17; +pub const DATA_TYPE_MAP: i32 = 18; +pub const DATA_TYPE_ROW: i32 = 19; /// Separates scalar and array type specs so each variant only carries /// the fields it actually needs — no zeroed-out placeholders. @@ -237,7 +240,8 @@ pub fn core_data_type_to_ffi(dt: &fcore::metadata::DataType) -> i32 { fcore::metadata::DataType::Char(_) => DATA_TYPE_CHAR, fcore::metadata::DataType::Binary(_) => DATA_TYPE_BINARY, fcore::metadata::DataType::Array(_) => DATA_TYPE_ARRAY, - _ => 0, + fcore::metadata::DataType::Map(_) => DATA_TYPE_MAP, + fcore::metadata::DataType::Row(_) => DATA_TYPE_ROW, } } @@ -289,8 +293,16 @@ pub fn ffi_descriptor_to_core( schema_builder = schema_builder.primary_key(descriptor.schema.primary_keys.clone()); } - let schema = schema_builder.build()?; + build_descriptor(schema_builder.build()?, descriptor) +} +/// Assemble a core `TableDescriptor` from a pre-built `Schema` plus the +/// descriptor's table-level metadata (partition/bucket keys, properties, +/// comment). Shared by the flat-column and Arrow-schema create-table paths. +fn build_descriptor( + schema: fcore::metadata::Schema, + descriptor: &ffi::FfiTableDescriptor, +) -> Result { let mut builder = fcore::metadata::TableDescriptor::builder() .schema(schema) .partitioned_by(descriptor.partition_keys.clone()); @@ -324,6 +336,52 @@ pub fn ffi_descriptor_to_core( Ok(builder.build()?) } +/// Build a core `TableDescriptor` whose columns come from an Arrow schema +/// imported over the C Data Interface. Lets C++ define nested MAP/ROW columns +/// the flat `FfiColumn` encoding can't express, reusing core's canonical +/// `from_arrow_field` converter rather than a second conversion copy. +/// +/// # Safety +/// `schema_ptr` must be a valid `FFI_ArrowSchema` heap pointer exported by C++ +/// (e.g. via `arrow::ExportSchema`); ownership is taken and released here. +pub unsafe fn arrow_ffi_to_core_descriptor( + schema_ptr: usize, + descriptor: &ffi::FfiTableDescriptor, +) -> Result { + let ffi_schema = unsafe { Box::from_raw(schema_ptr as *mut arrow::ffi::FFI_ArrowSchema) }; + let arrow_schema = arrow::datatypes::Schema::try_from(ffi_schema.as_ref()) + .map_err(|e| anyhow!("Failed to import Arrow schema: {e}"))?; + + let mut schema_builder = fcore::metadata::Schema::builder(); + for field in arrow_schema.fields() { + let dt = fcore::record::from_arrow_field(field.as_ref())?; + schema_builder = schema_builder.column(field.name(), dt); + } + if !descriptor.schema.primary_keys.is_empty() { + schema_builder = schema_builder.primary_key(descriptor.schema.primary_keys.clone()); + } + + build_descriptor(schema_builder.build()?, descriptor) +} + +/// Import a heap `FFI_ArrowSchema` (exported by C++) and return its fields as +/// Fluss DataTypes. Lets ArrayWriter/MapWriter carry ROW/MAP element and +/// key/value types that the flat FFI encoding cannot express. +/// +/// # Safety +/// `schema_ptr` must be a valid `FFI_ArrowSchema` heap pointer exported by C++ +/// (e.g. via `arrow::ExportSchema`); ownership is taken and released here. +pub unsafe fn arrow_ffi_to_data_types(schema_ptr: usize) -> Result> { + let ffi_schema = unsafe { Box::from_raw(schema_ptr as *mut arrow::ffi::FFI_ArrowSchema) }; + let arrow_schema = arrow::datatypes::Schema::try_from(ffi_schema.as_ref()) + .map_err(|e| anyhow!("Failed to import Arrow schema: {e}"))?; + let mut out = Vec::with_capacity(arrow_schema.fields().len()); + for field in arrow_schema.fields() { + out.push(fcore::record::from_arrow_field(field.as_ref())?); + } + Ok(out) +} + pub fn core_table_info_to_ffi(info: &fcore::metadata::TableInfo) -> ffi::FfiTableInfo { let schema = info.get_schema(); let columns: Vec = schema.columns().iter().map(core_column_to_ffi).collect(); @@ -478,126 +536,160 @@ pub fn resolve_row_types( row: &fcore::row::GenericRow<'_>, schema: Option<&fcore::metadata::Schema>, ) -> Result> { - use fcore::row::Datum; - let mut out = fcore::row::GenericRow::new(row.values.len()); for (idx, datum) in row.values.iter().enumerate() { - let resolved = match datum { - Datum::Null => Datum::Null, - Datum::Bool(v) => Datum::Bool(*v), - Datum::Int32(v) => match schema - .and_then(|s| s.columns().get(idx)) - .map(|c| c.data_type()) - { - Some(fcore::metadata::DataType::TinyInt(_)) => Datum::Int8( - i8::try_from(*v).map_err(|_| anyhow!("Column {idx}: {v} overflows TinyInt"))?, - ), - Some(fcore::metadata::DataType::SmallInt(_)) => Datum::Int16( - i16::try_from(*v) - .map_err(|_| anyhow!("Column {idx}: {v} overflows SmallInt"))?, - ), - _ => Datum::Int32(*v), - }, - Datum::Int64(v) => Datum::Int64(*v), - Datum::Float32(v) => Datum::Float32(*v), - Datum::Float64(v) => Datum::Float64(*v), - Datum::Int8(v) => Datum::Int8(*v), - Datum::Int16(v) => Datum::Int16(*v), - Datum::String(cow) => { - // Check if the schema column is Decimal — if so, parse the string as decimal - match schema - .and_then(|s| s.columns().get(idx)) - .map(|c| c.data_type()) - { - Some(fcore::metadata::DataType::Decimal(dt)) => { - let (precision, scale) = (dt.precision(), dt.scale()); - let bd = bigdecimal::BigDecimal::from_str(cow.as_ref()).map_err(|e| { - anyhow!("Column {idx}: invalid decimal string '{cow}': {e}") - })?; - let decimal = fcore::row::Decimal::from_big_decimal(bd, precision, scale) - .map_err(|e| anyhow!("Column {idx}: {e}"))?; - Datum::Decimal(decimal) - } - _ => Datum::String(Cow::Owned(cow.to_string())), - } - } - Datum::Blob(cow) => Datum::Blob(Cow::Owned(cow.to_vec())), - Datum::Decimal(d) => Datum::Decimal(d.clone()), - Datum::Date(d) => Datum::Date(*d), - Datum::Time(t) => Datum::Time(*t), - Datum::TimestampNtz(ts) => Datum::TimestampNtz(*ts), - Datum::TimestampLtz(ts) => Datum::TimestampLtz(*ts), - Datum::Array(a) => Datum::Array(a.clone()), - Datum::Map(m) => Datum::Map(m.clone()), - Datum::Row(_) => return Err(anyhow!("Row datum is not yet supported in C++ bindings")), - }; - out.set_field(idx, resolved); + let target = schema + .and_then(|s| s.columns().get(idx)) + .map(|c| c.data_type()); + out.set_field(idx, resolve_datum(datum, target, idx)?); } Ok(out) } +/// Resolve a single datum against its (optional) target column type, recursing +/// into nested ROW values. Narrows Int32 → Int8/Int16, parses decimal strings, +/// and leaves already-typed ARRAY/MAP binaries (built by the writers) untouched. +fn resolve_datum( + datum: &fcore::row::Datum<'_>, + target: Option<&fcore::metadata::DataType>, + idx: usize, +) -> Result> { + Ok(match datum { + Datum::Null => Datum::Null, + Datum::Bool(v) => Datum::Bool(*v), + Datum::Int32(v) => match target { + Some(fcore::metadata::DataType::TinyInt(_)) => Datum::Int8( + i8::try_from(*v).map_err(|_| anyhow!("Column {idx}: {v} overflows TinyInt"))?, + ), + Some(fcore::metadata::DataType::SmallInt(_)) => Datum::Int16( + i16::try_from(*v).map_err(|_| anyhow!("Column {idx}: {v} overflows SmallInt"))?, + ), + _ => Datum::Int32(*v), + }, + Datum::Int64(v) => Datum::Int64(*v), + Datum::Float32(v) => Datum::Float32(*v), + Datum::Float64(v) => Datum::Float64(*v), + Datum::Int8(v) => Datum::Int8(*v), + Datum::Int16(v) => Datum::Int16(*v), + Datum::String(cow) => match target { + // String standing in for a Decimal column — parse it. + Some(fcore::metadata::DataType::Decimal(dt)) => { + let (precision, scale) = (dt.precision(), dt.scale()); + let bd = bigdecimal::BigDecimal::from_str(cow.as_ref()) + .map_err(|e| anyhow!("Column {idx}: invalid decimal string '{cow}': {e}"))?; + let decimal = fcore::row::Decimal::from_big_decimal(bd, precision, scale) + .map_err(|e| anyhow!("Column {idx}: {e}"))?; + Datum::Decimal(decimal) + } + _ => Datum::String(Cow::Owned(cow.to_string())), + }, + Datum::Blob(cow) => Datum::Blob(Cow::Owned(cow.to_vec())), + Datum::Decimal(d) => Datum::Decimal(d.clone()), + Datum::Date(d) => Datum::Date(*d), + Datum::Time(t) => Datum::Time(*t), + Datum::TimestampNtz(ts) => Datum::TimestampNtz(*ts), + Datum::TimestampLtz(ts) => Datum::TimestampLtz(*ts), + Datum::Array(a) => Datum::Array(a.clone()), + Datum::Map(m) => Datum::Map(m.clone()), + Datum::Row(nested) => { + // A nested row carries untyped datums; resolve each field against + // the ROW's own field types so decimals/narrowing work recursively. + let field_types = match target { + Some(fcore::metadata::DataType::Row(rt)) => Some(rt.fields()), + _ => None, + }; + let mut out = fcore::row::GenericRow::new(nested.values.len()); + for (i, d) in nested.values.iter().enumerate() { + let ft = field_types.and_then(|f| f.get(i)).map(|f| f.data_type()); + out.set_field(i, resolve_datum(d, ft, i)?); + } + Datum::Row(Box::new(out)) + } + }) +} + /// Convert a CompactedRow (lookup result) to an owned GenericRow<'static>. /// One copy for strings/bytes (Cow::Owned), but no second copy into FfiDatum. pub fn compacted_row_to_owned( row: &dyn fcore::row::InternalRow, table_info: &fcore::metadata::TableInfo, ) -> Result> { - use fcore::row::Datum; + internal_row_to_owned_generic(row, table_info.get_schema().columns()) +} - let schema = table_info.get_schema(); - let columns = schema.columns(); - let mut out = fcore::row::GenericRow::new(columns.len()); +/// Read a single field of an InternalRow into an owned `'static` Datum, using +/// the column's declared type. This is the per-field core of +/// `internal_row_to_owned_generic`; the scan read path uses it to materialize +/// one complex cell without unpacking the whole row. +pub fn field_to_owned_datum( + row: &dyn fcore::row::InternalRow, + columns: &[fcore::metadata::Column], + field: usize, +) -> Result> { + let col = columns.get(field).ok_or_else(|| { + anyhow!( + "field index {field} out of range ({} columns)", + columns.len() + ) + })?; + if row.is_null_at(field)? { + return Ok(Datum::Null); + } - for (i, col) in columns.iter().enumerate() { - if row.is_null_at(i)? { - out.set_field(i, Datum::Null); - continue; + Ok(match col.data_type() { + fcore::metadata::DataType::Boolean(_) => Datum::Bool(row.get_boolean(field)?), + fcore::metadata::DataType::TinyInt(_) => Datum::Int8(row.get_byte(field)?), + fcore::metadata::DataType::SmallInt(_) => Datum::Int16(row.get_short(field)?), + fcore::metadata::DataType::Int(_) => Datum::Int32(row.get_int(field)?), + fcore::metadata::DataType::BigInt(_) => Datum::Int64(row.get_long(field)?), + fcore::metadata::DataType::Float(_) => Datum::Float32(row.get_float(field)?.into()), + fcore::metadata::DataType::Double(_) => Datum::Float64(row.get_double(field)?.into()), + fcore::metadata::DataType::String(_) => { + Datum::String(Cow::Owned(row.get_string(field)?.to_string())) } + fcore::metadata::DataType::Bytes(_) => { + Datum::Blob(Cow::Owned(row.get_bytes(field)?.to_vec())) + } + fcore::metadata::DataType::Date(_) => Datum::Date(row.get_date(field)?), + fcore::metadata::DataType::Time(_) => Datum::Time(row.get_time(field)?), + fcore::metadata::DataType::Timestamp(dt) => { + Datum::TimestampNtz(row.get_timestamp_ntz(field, dt.precision())?) + } + fcore::metadata::DataType::TimestampLTz(dt) => { + Datum::TimestampLtz(row.get_timestamp_ltz(field, dt.precision())?) + } + fcore::metadata::DataType::Decimal(dt) => { + Datum::Decimal(row.get_decimal(field, dt.precision() as usize, dt.scale() as usize)?) + } + fcore::metadata::DataType::Char(dt) => Datum::String(Cow::Owned( + row.get_char(field, dt.length() as usize)?.to_string(), + )), + fcore::metadata::DataType::Binary(dt) => { + Datum::Blob(Cow::Owned(row.get_binary(field, dt.length())?.to_vec())) + } + fcore::metadata::DataType::Array(_) => { + Datum::Array(row.get_array(field)?.try_into_binary()?) + } + fcore::metadata::DataType::Map(_) => Datum::Map(row.get_map(field)?.try_into_binary()?), + fcore::metadata::DataType::Row(rt) => { + Datum::Row(Box::new(row.get_row(field)?.try_into_generic(rt)?)) + } + }) +} - let datum = match col.data_type() { - fcore::metadata::DataType::Boolean(_) => Datum::Bool(row.get_boolean(i)?), - fcore::metadata::DataType::TinyInt(_) => Datum::Int8(row.get_byte(i)?), - fcore::metadata::DataType::SmallInt(_) => Datum::Int16(row.get_short(i)?), - fcore::metadata::DataType::Int(_) => Datum::Int32(row.get_int(i)?), - fcore::metadata::DataType::BigInt(_) => Datum::Int64(row.get_long(i)?), - fcore::metadata::DataType::Float(_) => Datum::Float32(row.get_float(i)?.into()), - fcore::metadata::DataType::Double(_) => Datum::Float64(row.get_double(i)?.into()), - fcore::metadata::DataType::String(_) => { - Datum::String(Cow::Owned(row.get_string(i)?.to_string())) - } - fcore::metadata::DataType::Bytes(_) => { - Datum::Blob(Cow::Owned(row.get_bytes(i)?.to_vec())) - } - fcore::metadata::DataType::Date(_) => Datum::Date(row.get_date(i)?), - fcore::metadata::DataType::Time(_) => Datum::Time(row.get_time(i)?), - fcore::metadata::DataType::Timestamp(dt) => { - Datum::TimestampNtz(row.get_timestamp_ntz(i, dt.precision())?) - } - fcore::metadata::DataType::TimestampLTz(dt) => { - Datum::TimestampLtz(row.get_timestamp_ltz(i, dt.precision())?) - } - fcore::metadata::DataType::Decimal(dt) => { - let decimal = row.get_decimal(i, dt.precision() as usize, dt.scale() as usize)?; - Datum::Decimal(decimal) - } - fcore::metadata::DataType::Char(dt) => Datum::String(Cow::Owned( - row.get_char(i, dt.length() as usize)?.to_string(), - )), - fcore::metadata::DataType::Binary(dt) => { - Datum::Blob(Cow::Owned(row.get_binary(i, dt.length())?.to_vec())) - } - fcore::metadata::DataType::Array(_) => { - Datum::Array(row.get_array(i)?.try_into_binary()?) - } - fcore::metadata::DataType::Map(_) => Datum::Map(row.get_map(i)?.try_into_binary()?), - other => return Err(anyhow!("Unsupported data type for column {i}: {other:?}")), - }; - - out.set_field(i, datum); +/// Walk an InternalRow field-by-field into an owned GenericRow<'static>, using +/// the given column types. Recurses through nested ROW/MAP/ARRAY values, so it +/// also materializes a ROW element read out of an array or map. +pub fn internal_row_to_owned_generic( + row: &dyn fcore::row::InternalRow, + columns: &[fcore::metadata::Column], +) -> Result> { + let mut out = fcore::row::GenericRow::new(columns.len()); + for i in 0..columns.len() { + out.set_field(i, field_to_owned_datum(row, columns, i)?); } - Ok(out) } diff --git a/bindings/cpp/test/test_kv_table.cpp b/bindings/cpp/test/test_kv_table.cpp index 2ef7e60f..6b4e6bb4 100644 --- a/bindings/cpp/test/test_kv_table.cpp +++ b/bindings/cpp/test/test_kv_table.cpp @@ -19,6 +19,8 @@ #include +#include + #include "test_utils.h" class KvTableTest : public ::testing::Test { @@ -155,7 +157,7 @@ TEST_F(KvTableTest, UpsertDeleteAndLookup) { ASSERT_OK(adm.DropTable(table_path, false)); } -TEST_F(KvTableTest, LookupWithNestedArrayArrayView) { +TEST_F(KvTableTest, LookupWithNestedArray) { auto& adm = admin(); auto& conn = connection(); @@ -212,24 +214,776 @@ TEST_F(KvTableTest, LookupWithNestedArrayArrayView) { fluss::LookupResult result; ASSERT_OK(lookuper.Lookup(key, result)); ASSERT_TRUE(result.Found()); - EXPECT_EQ(result.GetArraySize("matrix"), 2u); - EXPECT_EQ(result.GetArrayElementType("matrix"), fluss::TypeId::Array); - - auto outer = result.GetArrayView("matrix"); + auto outer = result.GetValue("matrix"); + EXPECT_EQ(outer.Type(), fluss::TypeId::Array); ASSERT_EQ(outer.Size(), 2u); - EXPECT_EQ(outer.ElementType(), fluss::TypeId::Array); - auto first = outer.GetArray(0); + auto first = outer.At(0); + EXPECT_EQ(first.Type(), fluss::TypeId::Array); ASSERT_EQ(first.Size(), 2u); - EXPECT_EQ(first.ElementType(), fluss::TypeId::Int); - EXPECT_EQ(first.GetInt32(0), 11); - EXPECT_EQ(first.GetInt32(1), 12); + EXPECT_EQ(first.At(0).GetInt32(), 11); + EXPECT_EQ(first.At(1).GetInt32(), 12); - auto second = outer.GetArray(1); + auto second = outer.At(1); ASSERT_EQ(second.Size(), 2u); - EXPECT_EQ(second.ElementType(), fluss::TypeId::Int); - EXPECT_EQ(second.GetInt32(0), 21); - EXPECT_EQ(second.GetInt32(1), 22); + EXPECT_EQ(second.At(0).GetInt32(), 21); + EXPECT_EQ(second.At(1).GetInt32(), 22); + + ASSERT_OK(adm.DropTable(table_path, false)); +} + +TEST_F(KvTableTest, LookupComplexTypesMatrix) { + auto& adm = admin(); + auto& conn = connection(); + + fluss::TablePath table_path("fluss", "test_lookup_complex_matrix_cpp"); + + auto row_seq_label = arrow::struct_( + {arrow::field("seq", arrow::int32()), arrow::field("label", arrow::utf8())}); + + auto arrow_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("m_str_int", arrow::map(arrow::utf8(), arrow::int32())), + arrow::field("m_str_row", arrow::map(arrow::utf8(), row_seq_label)), + arrow::field("m_str_map", + arrow::map(arrow::utf8(), arrow::map(arrow::utf8(), arrow::int32()))), + arrow::field("m_str_arr", arrow::map(arrow::utf8(), arrow::list(arrow::int32()))), + arrow::field("arr_map", arrow::list(arrow::map(arrow::utf8(), arrow::int32()))), + arrow::field("arr_row", arrow::list(row_seq_label)), + arrow::field("r_deep", arrow::struct_({arrow::field( + "inner", arrow::struct_({arrow::field("n", arrow::int32())}))})), + arrow::field("r_with_arr", + arrow::struct_({arrow::field("f_int", arrow::int32()), + arrow::field("f_arr", arrow::list(arrow::int32()))})), + // row_rich: every scalar type + an array field in one ROW. + arrow::field("r_rich", + arrow::struct_({ + arrow::field("f_bool", arrow::boolean()), + arrow::field("f_int", arrow::int32()), + arrow::field("f_long", arrow::int64()), + arrow::field("f_float", arrow::float32()), + arrow::field("f_double", arrow::float64()), + arrow::field("f_str", arrow::utf8()), + arrow::field("f_bytes", arrow::binary()), + arrow::field("f_decimal", arrow::decimal128(10, 2)), + arrow::field("f_date", arrow::date32()), + arrow::field("f_time", arrow::time32(arrow::TimeUnit::MILLI)), + arrow::field("f_ts_ntz", arrow::timestamp(arrow::TimeUnit::MICRO)), + arrow::field("f_ts_ltz", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC")), + arrow::field("f_binary", arrow::fixed_size_binary(4)), + arrow::field("f_arr", arrow::list(arrow::int32())), + })), + }); + auto schema = fluss::Schema::FromArrow(arrow_schema, {"id"}); + + auto table_descriptor = fluss::TableDescriptor::NewBuilder() + .SetSchema(schema) + .SetProperty("table.replication.factor", "1") + .Build(); + fluss_test::CreateTable(adm, table_path, table_descriptor); + + fluss::Table table; + ASSERT_OK(conn.GetTable(table_path, table)); + auto upsert = table.NewUpsert(); + fluss::UpsertWriter writer; + ASSERT_OK(upsert.CreateWriter(writer)); + + { + auto row = table.NewRow(); + row.Set("id", 1); + + // map — second entry has a NULL value. + { + fluss::MapWriter m(2, fluss::DataType::String(), fluss::DataType::Int()); + m.SetKeyString("a"); + m.SetValueInt32(1); + m.Commit(); + m.SetKeyString("b"); + m.SetValueNull(); + m.Commit(); + row.Set("m_str_int", std::move(m)); + } + // map> — value is a ROW, so the Arrow ctor. + { + fluss::MapWriter m(1, arrow::utf8(), row_seq_label); + m.SetKeyString("k"); + fluss::GenericRow v(2); + v.SetInt32(0, 7); + v.SetString(1, "seven"); + m.SetValueRow(std::move(v)); + m.Commit(); + row.Set("m_str_row", std::move(m)); + } + // map> + { + fluss::MapWriter m(1, arrow::utf8(), arrow::map(arrow::utf8(), arrow::int32())); + m.SetKeyString("k"); + fluss::MapWriter inner(1, fluss::DataType::String(), fluss::DataType::Int()); + inner.SetKeyString("x"); + inner.SetValueInt32(9); + inner.Commit(); + m.SetValueMap(std::move(inner)); + m.Commit(); + row.Set("m_str_map", std::move(m)); + } + // map> — value array fits the flat ctor. + { + fluss::MapWriter m(1, fluss::DataType::String(), + fluss::DataType::Array(fluss::DataType::Int())); + m.SetKeyString("k"); + fluss::ArrayWriter v(2, fluss::DataType::Int()); + v.SetInt32(0, 10); + v.SetInt32(1, 20); + m.SetValueArray(std::move(v)); + m.Commit(); + row.Set("m_str_arr", std::move(m)); + } + // array> — element is a MAP, so the Arrow ctor. + { + fluss::ArrayWriter a(1, arrow::map(arrow::utf8(), arrow::int32())); + fluss::MapWriter e(1, fluss::DataType::String(), fluss::DataType::Int()); + e.SetKeyString("p"); + e.SetValueInt32(5); + e.Commit(); + a.SetMap(0, std::move(e)); + row.Set("arr_map", std::move(a)); + } + // array> — element is a ROW, so the Arrow ctor. + { + fluss::ArrayWriter a(2, row_seq_label); + fluss::GenericRow e0(2); + e0.SetInt32(0, 1); + e0.SetString(1, "one"); + fluss::GenericRow e1(2); + e1.SetInt32(0, 2); + e1.SetString(1, "two"); + a.SetRow(0, std::move(e0)); + a.SetRow(1, std::move(e1)); + row.Set("arr_row", std::move(a)); + } + // row> + { + fluss::GenericRow inner(1); + inner.SetInt32(0, 42); + fluss::GenericRow outer(1); + outer.SetRow(0, std::move(inner)); + row.Set("r_deep", std::move(outer)); + } + // row> + { + fluss::GenericRow r(2); + r.SetInt32(0, 100); + fluss::ArrayWriter arr(3, fluss::DataType::Int()); + arr.SetInt32(0, 1); + arr.SetInt32(1, 2); + arr.SetInt32(2, 3); + r.SetArray(1, std::move(arr)); + row.Set("r_with_arr", std::move(r)); + } + // row_rich: exercise every Value leaf getter + an array field. + { + fluss::GenericRow rr(14); + rr.SetBool(0, true); + rr.SetInt32(1, 100000); + rr.SetInt64(2, 9876543210LL); + rr.SetFloat32(3, 3.5F); + rr.SetFloat64(4, 2.5); + rr.SetString(5, "hello world"); + rr.SetBytes(6, {'b', 'i', 'n'}); + rr.SetDecimal(7, "123.45"); + rr.SetDate(8, fluss::Date{20476}); + rr.SetTime(9, fluss::Time{36827123}); + rr.SetTimestampNtz(10, fluss::Timestamp{1769163227123LL, 456000}); + rr.SetTimestampLtz(11, fluss::Timestamp{1769163227456LL, 0}); + rr.SetBytes(12, {1, 2, 3, 4}); + fluss::ArrayWriter farr(3, fluss::DataType::Int()); + farr.SetInt32(0, 7); + farr.SetNull(1); + farr.SetInt32(2, 11); + rr.SetArray(13, std::move(farr)); + row.Set("r_rich", std::move(rr)); + } + + ASSERT_OK(writer.Upsert(row)); + ASSERT_OK(writer.Flush()); + } + + fluss::Lookuper lookuper; + ASSERT_OK(table.NewLookup().CreateLookuper(lookuper)); + auto key = table.NewRow(); + key.Set("id", 1); + fluss::LookupResult result; + ASSERT_OK(lookuper.Lookup(key, result)); + ASSERT_TRUE(result.Found()); + + // map — entry 1 has a NULL value. + { + auto m = result.GetValue("m_str_int"); + ASSERT_EQ(m.Size(), 2u); + EXPECT_EQ(m.KeyAt(0).GetString(), "a"); + EXPECT_FALSE(m.ValueAt(0).IsNull()); + EXPECT_EQ(m.ValueAt(0).GetInt32(), 1); + EXPECT_EQ(m.KeyAt(1).GetString(), "b"); + EXPECT_TRUE(m.ValueAt(1).IsNull()); + } + // map> + { + auto m = result.GetValue("m_str_row"); + ASSERT_EQ(m.Size(), 1u); + EXPECT_EQ(m.KeyAt(0).GetString(), "k"); + auto v = m.ValueAt(0); + EXPECT_EQ(v.Field(0).GetInt32(), 7); + EXPECT_EQ(v.Field(1).GetString(), "seven"); + } + // map> + { + auto inner = result.GetValue("m_str_map").ValueAt(0); + ASSERT_EQ(inner.Size(), 1u); + EXPECT_EQ(inner.KeyAt(0).GetString(), "x"); + EXPECT_EQ(inner.ValueAt(0).GetInt32(), 9); + } + // map> + { + auto av = result.GetValue("m_str_arr").ValueAt(0); + ASSERT_EQ(av.Size(), 2u); + EXPECT_EQ(av.At(0).GetInt32(), 10); + EXPECT_EQ(av.At(1).GetInt32(), 20); + } + // array> + { + auto a = result.GetValue("arr_map"); + ASSERT_EQ(a.Size(), 1u); + auto e = a.At(0); + EXPECT_EQ(e.KeyAt(0).GetString(), "p"); + EXPECT_EQ(e.ValueAt(0).GetInt32(), 5); + } + // array> + { + auto a = result.GetValue("arr_row"); + ASSERT_EQ(a.Size(), 2u); + EXPECT_EQ(a.At(0).Field(0).GetInt32(), 1); + EXPECT_EQ(a.At(0).Field(1).GetString(), "one"); + EXPECT_EQ(a.At(1).Field(0).GetInt32(), 2); + EXPECT_EQ(a.At(1).Field(1).GetString(), "two"); + } + // row> + { + auto inner = result.GetValue("r_deep").Field(0); + EXPECT_EQ(inner.Field(0).GetInt32(), 42); + } + // row + { + auto r = result.GetValue("r_with_arr"); + EXPECT_EQ(r.Field(0).GetInt32(), 100); + EXPECT_EQ(r.Field("f_int").GetInt32(), 100); // ROW field by name + auto arr = r.Field(1); + ASSERT_EQ(arr.Size(), 3u); + EXPECT_EQ(arr.At(2).GetInt32(), 3); + } + // row_rich — every leaf getter on one Value handle. + { + auto rr = result.GetValue("r_rich"); + ASSERT_EQ(rr.FieldCount(), 14u); + EXPECT_TRUE(rr.Field(0).GetBool()); + EXPECT_EQ(rr.Field(1).GetInt32(), 100000); + EXPECT_EQ(rr.Field(2).GetInt64(), 9876543210LL); + EXPECT_FLOAT_EQ(rr.Field(3).GetFloat32(), 3.5F); + EXPECT_DOUBLE_EQ(rr.Field(4).GetFloat64(), 2.5); + EXPECT_EQ(rr.Field(5).GetString(), "hello world"); + auto by = rr.Field(6).GetBytes(); + ASSERT_EQ(by.size(), 3u); + EXPECT_EQ(by[0], 'b'); + EXPECT_EQ(rr.Field(7).GetDecimalString(), "123.45"); + EXPECT_EQ(rr.Field(8).GetDate().days_since_epoch, 20476); + EXPECT_EQ(rr.Field(9).GetTime().millis_since_midnight, 36827123); + EXPECT_EQ(rr.Field(10).GetTimestamp().epoch_millis, 1769163227123LL); + EXPECT_EQ(rr.Field(11).GetTimestamp().epoch_millis, 1769163227456LL); + auto bin = rr.Field(12).GetBytes(); + ASSERT_EQ(bin.size(), 4u); + EXPECT_EQ(bin[3], 4); + auto fa = rr.Field(13); + ASSERT_EQ(fa.Size(), 3u); + EXPECT_TRUE(fa.At(1).IsNull()); + EXPECT_EQ(fa.At(2).GetInt32(), 11); + } + + // Row 2 (id=2) — every compound column NULL. + { + auto row = table.NewRow(); + row.SetInt32(0, 2); + for (size_t i = 1; i <= 9; ++i) { + row.SetNull(i); + } + ASSERT_OK(writer.Upsert(row)); + ASSERT_OK(writer.Flush()); + + auto key2 = table.NewRow(); + key2.SetInt32(0, 2); + fluss::LookupResult result2; + ASSERT_OK(lookuper.Lookup(key2, result2)); + ASSERT_TRUE(result2.Found()); + EXPECT_EQ(result2.GetInt32(0), 2); + for (size_t i = 1; i <= 9; ++i) { + EXPECT_TRUE(result2.IsNull(i)) << "column " << i << " should be null"; + } + } + + ASSERT_OK(adm.DropTable(table_path, false)); +} + +TEST_F(KvTableTest, LookupWithValueHandle) { + auto& adm = admin(); + auto& conn = connection(); + + fluss::TablePath table_path("fluss", "test_lookup_value_handle_cpp"); + + auto row_seq_label = arrow::struct_( + {arrow::field("seq", arrow::int32()), arrow::field("label", arrow::utf8())}); + auto arrow_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("m", arrow::map(arrow::utf8(), arrow::int32())), + arrow::field("mr", arrow::map(arrow::utf8(), row_seq_label)), + arrow::field("ar", arrow::list(row_seq_label)), + arrow::field("r", arrow::struct_({arrow::field("f_int", arrow::int32()), + arrow::field("f_arr", arrow::list(arrow::int32()))})), + }); + auto schema = fluss::Schema::FromArrow(arrow_schema, {"id"}); + + auto table_descriptor = fluss::TableDescriptor::NewBuilder() + .SetSchema(schema) + .SetProperty("table.replication.factor", "1") + .Build(); + fluss_test::CreateTable(adm, table_path, table_descriptor); + + fluss::Table table; + ASSERT_OK(conn.GetTable(table_path, table)); + fluss::UpsertWriter writer; + ASSERT_OK(table.NewUpsert().CreateWriter(writer)); + + { + auto row = table.NewRow(); + row.SetInt32(0, 1); + // m = {"a":1, "b":null} + fluss::MapWriter m(2, fluss::DataType::String(), fluss::DataType::Int()); + m.SetKeyString("a"); m.SetValueInt32(1); m.Commit(); + m.SetKeyString("b"); m.SetValueNull(); m.Commit(); + row.SetMap(1, std::move(m)); + // mr = {"k": {7,"seven"}} + fluss::MapWriter mr(1, arrow::utf8(), row_seq_label); + mr.SetKeyString("k"); + fluss::GenericRow rv(2); rv.SetInt32(0, 7); rv.SetString(1, "seven"); + mr.SetValueRow(std::move(rv)); + mr.Commit(); + row.SetMap(2, std::move(mr)); + // ar = [{1,"one"},{2,"two"}] + fluss::ArrayWriter ar(2, row_seq_label); + fluss::GenericRow e0(2); e0.SetInt32(0, 1); e0.SetString(1, "one"); + fluss::GenericRow e1(2); e1.SetInt32(0, 2); e1.SetString(1, "two"); + ar.SetRow(0, std::move(e0)); ar.SetRow(1, std::move(e1)); + row.SetArray(3, std::move(ar)); + // r = {100, [1,2,3]} + fluss::GenericRow r(2); r.SetInt32(0, 100); + fluss::ArrayWriter fa(3, fluss::DataType::Int()); + fa.SetInt32(0, 1); fa.SetInt32(1, 2); fa.SetInt32(2, 3); + r.SetArray(1, std::move(fa)); + row.SetRow(4, std::move(r)); + ASSERT_OK(writer.Upsert(row)); + ASSERT_OK(writer.Flush()); + } + + fluss::Lookuper lookuper; + ASSERT_OK(table.NewLookup().CreateLookuper(lookuper)); + auto key = table.NewRow(); + key.SetInt32(0, 1); + fluss::LookupResult result; + ASSERT_OK(lookuper.Lookup(key, result)); + ASSERT_TRUE(result.Found()); + + // map, with a null value — navigate + leaf-read off one handle. + auto m = result.GetValue("m"); + EXPECT_EQ(m.Type(), fluss::TypeId::Map); + ASSERT_EQ(m.Size(), 2u); + EXPECT_EQ(m.KeyAt(0).GetString(), "a"); + EXPECT_EQ(m.ValueAt(0).GetInt32(), 1); + EXPECT_EQ(m.KeyAt(1).GetString(), "b"); + EXPECT_TRUE(m.ValueAt(1).IsNull()); + + // map + auto mr = result.GetValue("mr"); + EXPECT_EQ(mr.ValueAt(0).Field(0).GetInt32(), 7); + EXPECT_EQ(mr.ValueAt(0).Field(1).GetString(), "seven"); + + // array + auto ar = result.GetValue("ar"); + EXPECT_EQ(ar.Type(), fluss::TypeId::Array); + ASSERT_EQ(ar.Size(), 2u); + EXPECT_EQ(ar.At(0).Field(0).GetInt32(), 1); + EXPECT_EQ(ar.At(1).Field(1).GetString(), "two"); + + // row + auto r = result.GetValue("r"); + EXPECT_EQ(r.Type(), fluss::TypeId::Row); + ASSERT_EQ(r.FieldCount(), 2u); + EXPECT_EQ(r.Field(0).GetInt32(), 100); + auto fa = r.Field(1); + ASSERT_EQ(fa.Size(), 3u); + EXPECT_EQ(fa.At(2).GetInt32(), 3); + + ASSERT_OK(adm.DropTable(table_path, false)); +} + +// Regression: timestamp map values built via the Arrow MapWriter ctor must pick +// NTZ vs LTZ from the declared value type (previously always wrote NTZ). +TEST_F(KvTableTest, MapWithTimestampValuesNtzAndLtz) { + auto& adm = admin(); + auto& conn = connection(); + + fluss::TablePath table_path("fluss", "test_map_timestamp_values_cpp"); + + auto arrow_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("mn", arrow::map(arrow::utf8(), arrow::timestamp(arrow::TimeUnit::MICRO))), + arrow::field("ml", + arrow::map(arrow::utf8(), arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"))), + }); + auto schema = fluss::Schema::FromArrow(arrow_schema, {"id"}); + + auto table_descriptor = fluss::TableDescriptor::NewBuilder() + .SetSchema(schema) + .SetProperty("table.replication.factor", "1") + .Build(); + fluss_test::CreateTable(adm, table_path, table_descriptor); + + fluss::Table table; + ASSERT_OK(conn.GetTable(table_path, table)); + fluss::UpsertWriter writer; + ASSERT_OK(table.NewUpsert().CreateWriter(writer)); + + const fluss::Timestamp ntz_val{1769163227123LL, 456000}; + const fluss::Timestamp ltz_val{1700000000789LL, 123000}; + { + auto row = table.NewRow(); + row.SetInt32(0, 1); + // mn: map (NTZ), via the Arrow ctor. + fluss::MapWriter mn(1, arrow::utf8(), arrow::timestamp(arrow::TimeUnit::MICRO)); + mn.SetKeyString("k"); + mn.SetValueTimestamp(ntz_val); + mn.Commit(); + row.SetMap(1, std::move(mn)); + // ml: map (LTZ), via the Arrow ctor. + fluss::MapWriter ml(1, arrow::utf8(), arrow::timestamp(arrow::TimeUnit::MICRO, "UTC")); + ml.SetKeyString("k"); + ml.SetValueTimestamp(ltz_val); + ml.Commit(); + row.SetMap(2, std::move(ml)); + ASSERT_OK(writer.Upsert(row)); + ASSERT_OK(writer.Flush()); + } + + fluss::Lookuper lookuper; + ASSERT_OK(table.NewLookup().CreateLookuper(lookuper)); + auto key = table.NewRow(); + key.SetInt32(0, 1); + fluss::LookupResult result; + ASSERT_OK(lookuper.Lookup(key, result)); + ASSERT_TRUE(result.Found()); + + auto mn = result.GetValue("mn"); + ASSERT_EQ(mn.Size(), 1u); + EXPECT_EQ(mn.ValueAt(0).GetTimestamp().epoch_millis, ntz_val.epoch_millis); + EXPECT_EQ(mn.ValueAt(0).GetTimestamp().nano_of_millisecond, ntz_val.nano_of_millisecond); + + auto ml = result.GetValue("ml"); + ASSERT_EQ(ml.Size(), 1u); + EXPECT_EQ(ml.ValueAt(0).GetTimestamp().epoch_millis, ltz_val.epoch_millis); + EXPECT_EQ(ml.ValueAt(0).GetTimestamp().nano_of_millisecond, ltz_val.nano_of_millisecond); + + ASSERT_OK(adm.DropTable(table_path, false)); +} + +// Deeply nested MAP/ROW via native DataType::Map / DataType::Row only — no Arrow. +TEST_F(KvTableTest, NativeNestedBuilderNoArrow) { + auto& adm = admin(); + auto& conn = connection(); + + fluss::TablePath table_path("fluss", "test_native_nested_builder_cpp"); + + // array>> + auto event = fluss::DataType::Row({ + {"seq", fluss::DataType::Int()}, + {"attrs", fluss::DataType::Map(fluss::DataType::String(), fluss::DataType::Int())}, + }); + auto schema = fluss::Schema::NewBuilder() + .AddColumn("id", fluss::DataType::Int()) + .AddColumn("events", fluss::DataType::Array(event)) + .AddColumn("profile", fluss::DataType::Row({ + {"name", fluss::DataType::String()}, + {"score", fluss::DataType::Double()}, + })) + .SetPrimaryKeys({"id"}) + .Build(); + + auto table_descriptor = fluss::TableDescriptor::NewBuilder() + .SetSchema(schema) + .SetProperty("table.replication.factor", "1") + .Build(); + fluss_test::CreateTable(adm, table_path, table_descriptor); + + fluss::Table table; + ASSERT_OK(conn.GetTable(table_path, table)); + fluss::UpsertWriter writer; + ASSERT_OK(table.NewUpsert().CreateWriter(writer)); + + { + auto row = table.NewRow(); + row.Set("id", 1); + + // events = [ {0, {"a":0}}, {1, {"a":10,"b":11}} ] + fluss::ArrayWriter events(2, event); + for (int i = 0; i < 2; i++) { + fluss::GenericRow ev(2); + ev.SetInt32(0, i); + fluss::MapWriter attrs(static_cast(i + 1), fluss::DataType::String(), + fluss::DataType::Int()); + attrs.SetKeyString("a"); + attrs.SetValueInt32(i * 10); + attrs.Commit(); + if (i == 1) { + attrs.SetKeyString("b"); + attrs.SetValueInt32(11); + attrs.Commit(); + } + ev.SetMap(1, std::move(attrs)); + events.SetRow(i, std::move(ev)); + } + row.Set("events", std::move(events)); + + fluss::GenericRow profile(2); + profile.SetString(0, "alice"); + profile.SetFloat64(1, 9.5); + row.Set("profile", std::move(profile)); + + ASSERT_OK(writer.Upsert(row)); + ASSERT_OK(writer.Flush()); + } + + fluss::Lookuper lookuper; + ASSERT_OK(table.NewLookup().CreateLookuper(lookuper)); + auto key = table.NewRow(); + key.SetInt32(0, 1); + fluss::LookupResult result; + ASSERT_OK(lookuper.Lookup(key, result)); + ASSERT_TRUE(result.Found()); + + auto events = result.GetValue("events"); // ARRAY>> + ASSERT_EQ(events.Type(), fluss::TypeId::Array); + ASSERT_EQ(events.Size(), 2u); + EXPECT_EQ(events.At(0).Field("seq").GetInt32(), 0); + EXPECT_EQ(events.At(0).Field("attrs").ValueAt(0).GetInt32(), 0); + auto e1_attrs = events.At(1).Field("attrs"); + ASSERT_EQ(e1_attrs.Size(), 2u); + EXPECT_EQ(e1_attrs.KeyAt(1).GetString(), "b"); + EXPECT_EQ(e1_attrs.ValueAt(1).GetInt32(), 11); + + auto profile = result.GetValue("profile"); // ROW + EXPECT_EQ(profile.Field("name").GetString(), "alice"); + EXPECT_DOUBLE_EQ(profile.Field("score").GetFloat64(), 9.5); + + ASSERT_OK(adm.DropTable(table_path, false)); +} + +TEST_F(KvTableTest, ComplexTypesPartialUpdate) { + auto& adm = admin(); + auto& conn = connection(); + + fluss::TablePath table_path("fluss", "test_complex_partial_update_cpp"); + + auto arrow_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("name", arrow::utf8()), + arrow::field("score", arrow::int64()), + arrow::field("nested", arrow::struct_({arrow::field("seq", arrow::int32()), + arrow::field("label", arrow::utf8())})), + arrow::field("attrs", arrow::map(arrow::utf8(), arrow::int32())), + arrow::field("tags", arrow::list(arrow::utf8())), + }); + auto schema = fluss::Schema::FromArrow(arrow_schema, {"id"}); + + auto table_descriptor = fluss::TableDescriptor::NewBuilder() + .SetSchema(schema) + .SetProperty("table.replication.factor", "1") + .Build(); + fluss_test::CreateTable(adm, table_path, table_descriptor); + + fluss::Table table; + ASSERT_OK(conn.GetTable(table_path, table)); + + { + auto upsert = table.NewUpsert(); + fluss::UpsertWriter w; + ASSERT_OK(upsert.CreateWriter(w)); + auto row = table.NewRow(); + row.SetInt32(0, 1); + row.SetString(1, "Verso"); + row.SetInt64(2, 100); + fluss::GenericRow nested(2); + nested.SetInt32(0, 10); + nested.SetString(1, "alpha"); + row.SetRow(3, std::move(nested)); + fluss::MapWriter attrs(2, fluss::DataType::String(), fluss::DataType::Int()); + attrs.SetKeyString("a"); + attrs.SetValueInt32(1); + attrs.Commit(); + attrs.SetKeyString("b"); + attrs.SetValueInt32(2); + attrs.Commit(); + row.SetMap(4, std::move(attrs)); + fluss::ArrayWriter tags(2, fluss::DataType::String()); + tags.SetString(0, "x"); + tags.SetString(1, "y"); + row.SetArray(5, std::move(tags)); + ASSERT_OK(w.Upsert(row)); + ASSERT_OK(w.Flush()); + } + + fluss::Lookuper lookuper; + ASSERT_OK(table.NewLookup().CreateLookuper(lookuper)); + auto key = table.NewRow(); + key.SetInt32(0, 1); + + // Partial update of a scalar column — compound columns must be preserved. + { + auto pu = table.NewUpsert(); + pu.PartialUpdateByName({"id", "score"}); + fluss::UpsertWriter pw; + ASSERT_OK(pu.CreateWriter(pw)); + auto row = table.NewRow(); + row.SetInt32(0, 1); + row.SetNull(1); + row.SetInt64(2, 420); + row.SetNull(3); + row.SetNull(4); + row.SetNull(5); + ASSERT_OK(pw.Upsert(row)); + ASSERT_OK(pw.Flush()); + + fluss::LookupResult result; + ASSERT_OK(lookuper.Lookup(key, result)); + ASSERT_TRUE(result.Found()); + EXPECT_EQ(result.GetString(1), "Verso"); + EXPECT_EQ(result.GetInt64(2), 420); + auto n = result.GetValue(3); + EXPECT_EQ(n.Field(0).GetInt32(), 10); + EXPECT_EQ(n.Field(1).GetString(), "alpha"); + EXPECT_EQ(result.GetValue(4).Size(), 2u); + EXPECT_EQ(result.GetValue(5).Size(), 2u); + } + + // Partial update of the ROW column — other compound columns preserved. + { + auto pu = table.NewUpsert(); + pu.PartialUpdateByName({"id", "nested"}); + fluss::UpsertWriter pw; + ASSERT_OK(pu.CreateWriter(pw)); + auto row = table.NewRow(); + row.SetInt32(0, 1); + row.SetNull(1); + row.SetNull(2); + fluss::GenericRow nn(2); + nn.SetInt32(0, 99); + nn.SetString(1, "omega"); + row.SetRow(3, std::move(nn)); + row.SetNull(4); + row.SetNull(5); + ASSERT_OK(pw.Upsert(row)); + ASSERT_OK(pw.Flush()); + + fluss::LookupResult result; + ASSERT_OK(lookuper.Lookup(key, result)); + ASSERT_TRUE(result.Found()); + EXPECT_EQ(result.GetInt64(2), 420); + auto n = result.GetValue(3); + EXPECT_EQ(n.Field(0).GetInt32(), 99); + EXPECT_EQ(n.Field(1).GetString(), "omega"); + EXPECT_EQ(result.GetValue(4).Size(), 2u); + EXPECT_EQ(result.GetValue(5).Size(), 2u); + } + + ASSERT_OK(adm.DropTable(table_path, false)); +} + +TEST_F(KvTableTest, PartitionedComplexTypes) { + auto& adm = admin(); + auto& conn = connection(); + + fluss::TablePath table_path("fluss", "test_partitioned_complex_cpp"); + + auto arrow_schema = arrow::schema({ + arrow::field("region", arrow::utf8()), + arrow::field("user_id", arrow::int32()), + arrow::field("nested", arrow::struct_({arrow::field("seq", arrow::int32()), + arrow::field("label", arrow::utf8())})), + arrow::field("attrs", arrow::map(arrow::utf8(), arrow::int32())), + }); + auto schema = fluss::Schema::FromArrow(arrow_schema, {"region", "user_id"}); + + auto table_descriptor = fluss::TableDescriptor::NewBuilder() + .SetSchema(schema) + .SetPartitionKeys({"region"}) + .SetProperty("table.replication.factor", "1") + .Build(); + fluss_test::CreateTable(adm, table_path, table_descriptor); + fluss_test::CreatePartitions(adm, table_path, "region", {"US", "EU"}); + + fluss::Table table; + ASSERT_OK(conn.GetTable(table_path, table)); + + fluss::UpsertWriter writer; + ASSERT_OK(table.NewUpsert().CreateWriter(writer)); + + struct Rec { + const char* region; + int32_t user_id; + int32_t seq; + const char* label; + }; + const Rec data[] = {{"US", 1, 11, "alpha"}, {"EU", 2, 22, "beta"}}; + + for (const auto& d : data) { + auto row = table.NewRow(); + row.SetString(0, d.region); + row.SetInt32(1, d.user_id); + fluss::GenericRow nested(2); + nested.SetInt32(0, d.seq); + nested.SetString(1, d.label); + row.SetRow(2, std::move(nested)); + fluss::MapWriter attrs(1, fluss::DataType::String(), fluss::DataType::Int()); + attrs.SetKeyString(d.label); + attrs.SetValueInt32(d.seq); + attrs.Commit(); + row.SetMap(3, std::move(attrs)); + ASSERT_OK(writer.Upsert(row)); + } + ASSERT_OK(writer.Flush()); + + fluss::Lookuper lookuper; + ASSERT_OK(table.NewLookup().CreateLookuper(lookuper)); + + for (const auto& d : data) { + auto key = table.NewRow(); + key.SetString(0, d.region); + key.SetInt32(1, d.user_id); + fluss::LookupResult result; + ASSERT_OK(lookuper.Lookup(key, result)); + ASSERT_TRUE(result.Found()); + auto nested = result.GetValue(2); + EXPECT_EQ(nested.Field(0).GetInt32(), d.seq); + EXPECT_EQ(nested.Field(1).GetString(), d.label); + auto attrs = result.GetValue(3); + ASSERT_EQ(attrs.Size(), 1u); + EXPECT_EQ(attrs.KeyAt(0).GetString(), d.label); + EXPECT_EQ(attrs.ValueAt(0).GetInt32(), d.seq); + } ASSERT_OK(adm.DropTable(table_path, false)); } @@ -275,42 +1029,30 @@ TEST_F(KvTableTest, LookupArrayValidationErrors) { ASSERT_OK(lookuper.Lookup(key, result)); ASSERT_TRUE(result.Found()); + auto view = result.GetValue("vals"); + EXPECT_EQ(view.Type(), fluss::TypeId::Array); + ASSERT_EQ(view.Size(), 2u); + EXPECT_EQ(view.At(0).GetInt32(), 99); + EXPECT_TRUE(view.At(1).IsNull()); + + // A wrong-type leaf read throws. bool wrong_type_threw = false; try { - (void)result.GetArrayInt64("vals", 0); + (void)view.At(0).GetInt64(); } catch (const std::exception&) { wrong_type_threw = true; } EXPECT_TRUE(wrong_type_threw); + // A typed read of a null element throws. bool null_typed_getter_threw = false; try { - (void)result.GetArrayInt32("vals", 1); + (void)view.At(1).GetInt32(); } catch (const std::exception&) { null_typed_getter_threw = true; } EXPECT_TRUE(null_typed_getter_threw); - auto view = result.GetArrayView("vals"); - EXPECT_EQ(view.Size(), 2u); - EXPECT_TRUE(view.IsNull(1)); - - bool view_wrong_type_threw = false; - try { - (void)view.GetInt64(0); - } catch (const std::exception&) { - view_wrong_type_threw = true; - } - EXPECT_TRUE(view_wrong_type_threw); - - bool view_null_typed_getter_threw = false; - try { - (void)view.GetInt32(1); - } catch (const std::exception&) { - view_null_typed_getter_threw = true; - } - EXPECT_TRUE(view_null_typed_getter_threw); - ASSERT_OK(adm.DropTable(table_path, false)); } diff --git a/bindings/cpp/test/test_log_table.cpp b/bindings/cpp/test/test_log_table.cpp index 5678e4bb..aa43c6d6 100644 --- a/bindings/cpp/test/test_log_table.cpp +++ b/bindings/cpp/test/test_log_table.cpp @@ -926,18 +926,17 @@ TEST_F(LogTableTest, AppendAndScanWithArray) { Record rec; rec.id = rv.GetInt32(0); - rec.tag_count = rv.GetArraySize(1); + auto tags = rv.GetValue(1); + rec.tag_count = tags.Size(); for (size_t i = 0; i < rec.tag_count; ++i) { - if (rv.IsArrayElementNull(1, i)) { - rec.tags.push_back(""); - } else { - rec.tags.push_back(rv.GetArrayString(1, i)); - } + auto el = tags.At(i); + rec.tags.push_back(el.IsNull() ? "" : el.GetString()); } - rec.score_count = rv.GetArraySize(2); + auto scores = rv.GetValue(2); + rec.score_count = scores.Size(); for (size_t i = 0; i < rec.score_count; ++i) { - rec.scores.push_back(rv.GetArrayInt32(2, i)); + rec.scores.push_back(scores.At(i).GetInt32()); } return rec; @@ -967,6 +966,191 @@ TEST_F(LogTableTest, AppendAndScanWithArray) { ASSERT_OK(adm.DropTable(table_path, false)); } +TEST_F(LogTableTest, AppendAndScanWithMapAndRow) { + auto& adm = admin(); + auto& conn = connection(); + + fluss::TablePath table_path("fluss", "test_append_scan_map_row_cpp"); + + // MAP / ROW columns can't be built with the flat schema builder. + auto arrow_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("attrs", arrow::map(arrow::utf8(), arrow::int32())), + arrow::field("nested", arrow::struct_({arrow::field("seq", arrow::int32()), + arrow::field("label", arrow::utf8())})), + }); + auto schema = fluss::Schema::FromArrow(arrow_schema); + + auto table_descriptor = fluss::TableDescriptor::NewBuilder() + .SetSchema(schema) + .SetBucketCount(1) + .SetProperty("table.replication.factor", "1") + .Build(); + fluss_test::CreateTable(adm, table_path, table_descriptor); + + fluss::Table table; + ASSERT_OK(conn.GetTable(table_path, table)); + + fluss::AppendWriter append_writer; + ASSERT_OK(table.NewAppend().CreateWriter(append_writer)); + { + auto row = table.NewRow(); + row.Set("id", 1); + + fluss::MapWriter attrs(2, fluss::DataType::String(), fluss::DataType::Int()); + attrs.SetKeyString("a"); + attrs.SetValueInt32(1); + attrs.Commit(); + attrs.SetKeyString("b"); + attrs.SetValueInt32(2); + attrs.Commit(); + row.Set("attrs", std::move(attrs)); + + fluss::GenericRow nested(2); + nested.SetInt32(0, 7); + nested.SetString(1, "seven"); + row.Set("nested", std::move(nested)); + + ASSERT_OK(append_writer.Append(row)); + } + ASSERT_OK(append_writer.Flush()); + + auto scan = table.NewScan(); + fluss::LogScanner scanner; + ASSERT_OK(scan.CreateLogScanner(scanner)); + ASSERT_OK(scanner.Subscribe(0, 0)); + + struct Record { + int32_t id; + size_t attr_count; + std::string k0; + int32_t v0; + int32_t seq; + std::string label; + }; + std::vector collected; + auto extract = [](const fluss::ScanRecord& scan_rec) { + const auto& rv = scan_rec.row; + Record rec; + rec.id = rv.GetInt32(0); + auto attrs = rv.GetValue(1); + rec.attr_count = attrs.Size(); + rec.k0 = attrs.KeyAt(0).GetString(); + rec.v0 = attrs.ValueAt(0).GetInt32(); + auto nested = rv.GetValue("nested"); + rec.seq = nested.Field(0).GetInt32(); + rec.label = nested.Field(1).GetString(); + return rec; + }; + fluss_test::PollRecords(scanner, 1, extract, collected); + + ASSERT_EQ(collected.size(), 1u); + EXPECT_EQ(collected[0].id, 1); + EXPECT_EQ(collected[0].attr_count, 2u); + EXPECT_EQ(collected[0].k0, "a"); + EXPECT_EQ(collected[0].v0, 1); + EXPECT_EQ(collected[0].seq, 7); + EXPECT_EQ(collected[0].label, "seven"); + + ASSERT_OK(adm.DropTable(table_path, false)); +} + +TEST_F(LogTableTest, ProjectionWithCompoundTypes) { + auto& adm = admin(); + auto& conn = connection(); + + fluss::TablePath table_path("fluss", "test_log_projection_compound_cpp"); + + auto arrow_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("nested", arrow::struct_({arrow::field("seq", arrow::int32()), + arrow::field("label", arrow::utf8())})), + arrow::field("attrs", arrow::map(arrow::utf8(), arrow::int32())), + arrow::field("tags", arrow::list(arrow::utf8())), + arrow::field("extra", arrow::utf8()), + }); + auto schema = fluss::Schema::FromArrow(arrow_schema); + + auto table_descriptor = fluss::TableDescriptor::NewBuilder() + .SetSchema(schema) + .SetBucketCount(1) + .SetProperty("table.replication.factor", "1") + .Build(); + fluss_test::CreateTable(adm, table_path, table_descriptor); + + fluss::Table table; + ASSERT_OK(conn.GetTable(table_path, table)); + + fluss::AppendWriter append_writer; + ASSERT_OK(table.NewAppend().CreateWriter(append_writer)); + { + auto row = table.NewRow(); + row.SetInt32(0, 7); + fluss::GenericRow nested(2); + nested.SetInt32(0, 42); + nested.SetString(1, "hello"); + row.SetRow(1, std::move(nested)); + fluss::MapWriter attrs(2, fluss::DataType::String(), fluss::DataType::Int()); + attrs.SetKeyString("x"); + attrs.SetValueInt32(1); + attrs.Commit(); + attrs.SetKeyString("y"); + attrs.SetValueInt32(2); + attrs.Commit(); + row.SetMap(2, std::move(attrs)); + fluss::ArrayWriter tags(2, fluss::DataType::String()); + tags.SetString(0, "alpha"); + tags.SetString(1, "beta"); + row.SetArray(3, std::move(tags)); + row.SetString(4, "ignore-me"); + ASSERT_OK(append_writer.Append(row)); + } + ASSERT_OK(append_writer.Flush()); + + // Project columns reordered, dropping `extra`: new layout is + // [nested=0, attrs=1, tags=2, id=3]. + auto scan = table.NewScan(); + scan.ProjectByName({"nested", "attrs", "tags", "id"}); + fluss::LogScanner scanner; + ASSERT_OK(scan.CreateLogScanner(scanner)); + ASSERT_OK(scanner.Subscribe(0, 0)); + + struct Rec { + int32_t id; + int32_t seq; + std::string label; + size_t attr_count; + size_t tag_count; + std::string tag0; + }; + std::vector collected; + auto extract = [](const fluss::ScanRecord& sr) { + const auto& rv = sr.row; + Rec rec; + auto nested = rv.GetValue(0); + rec.seq = nested.Field(0).GetInt32(); + rec.label = nested.Field(1).GetString(); + auto m = rv.GetValue(1); + rec.attr_count = m.Size(); + auto a = rv.GetValue(2); + rec.tag_count = a.Size(); + rec.tag0 = a.At(0).GetString(); + rec.id = rv.GetInt32(3); + return rec; + }; + fluss_test::PollRecords(scanner, 1, extract, collected); + + ASSERT_EQ(collected.size(), 1u); + EXPECT_EQ(collected[0].id, 7); + EXPECT_EQ(collected[0].seq, 42); + EXPECT_EQ(collected[0].label, "hello"); + EXPECT_EQ(collected[0].attr_count, 2u); + EXPECT_EQ(collected[0].tag_count, 2u); + EXPECT_EQ(collected[0].tag0, "alpha"); + + ASSERT_OK(adm.DropTable(table_path, false)); +} + TEST_F(LogTableTest, AppendAndScanWithNestedArray) { auto& adm = admin(); auto& conn = connection(); @@ -1034,16 +1218,16 @@ TEST_F(LogTableTest, AppendAndScanWithNestedArray) { const auto& rv = scan_rec.row; Record rec; rec.id = rv.GetInt32(0); - rec.outer_count = rv.GetArraySize(1); - rec.element_type = rv.GetArrayElementType(1); - auto outer = rv.GetArrayView(1); + auto outer = rv.GetValue(1); + rec.outer_count = outer.Size(); + rec.element_type = outer.Size() > 0 ? outer.At(0).Type() : fluss::TypeId::Array; rec.values.reserve(outer.Size()); for (size_t i = 0; i < outer.Size(); ++i) { - auto inner = outer.GetArray(i); + auto inner = outer.At(i); std::vector row; row.reserve(inner.Size()); for (size_t j = 0; j < inner.Size(); ++j) { - row.push_back(inner.GetInt32(j)); + row.push_back(inner.At(j).GetInt32()); } rec.values.push_back(std::move(row)); } @@ -1140,29 +1324,35 @@ TEST_F(LogTableTest, AppendAndScanWithArrayRichTypes) { auto rec = *it; const auto& rv = rec.row; - EXPECT_EQ(rv.GetArraySize(1), 2u); - auto bytes0 = rv.GetArrayBytes(1, 0); + auto arr_bytes = rv.GetValue(1); + EXPECT_EQ(arr_bytes.Size(), 2u); + auto bytes0 = arr_bytes.At(0).GetBytes(); ASSERT_EQ(bytes0.size(), 3u); EXPECT_EQ(bytes0[0], 0x10); EXPECT_EQ(bytes0[1], 0x20); EXPECT_EQ(bytes0[2], 0x30); - EXPECT_TRUE(rv.IsArrayElementNull(1, 1)); + EXPECT_TRUE(arr_bytes.At(1).IsNull()); - EXPECT_EQ(rv.GetArraySize(2), 2u); - EXPECT_EQ(rv.GetArrayDate(2, 0).days_since_epoch, fluss::Date::FromDays(20000).days_since_epoch); - EXPECT_TRUE(rv.IsArrayElementNull(2, 1)); + auto arr_date = rv.GetValue(2); + EXPECT_EQ(arr_date.Size(), 2u); + EXPECT_EQ(arr_date.At(0).GetDate().days_since_epoch, fluss::Date::FromDays(20000).days_since_epoch); + EXPECT_TRUE(arr_date.At(1).IsNull()); - EXPECT_EQ(rv.GetArraySize(3), 1u); - EXPECT_EQ(rv.GetArrayTime(3, 0).millis_since_midnight, fluss::Time::FromMillis(3600000).millis_since_midnight); + auto arr_time = rv.GetValue(3); + EXPECT_EQ(arr_time.Size(), 1u); + EXPECT_EQ(arr_time.At(0).GetTime().millis_since_midnight, + fluss::Time::FromMillis(3600000).millis_since_midnight); - EXPECT_EQ(rv.GetArraySize(4), 1u); - auto ts = rv.GetArrayTimestamp(4, 0); + auto arr_ts = rv.GetValue(4); + EXPECT_EQ(arr_ts.Size(), 1u); + auto ts = arr_ts.At(0).GetTimestamp(); EXPECT_EQ(ts.epoch_millis, 1769163227123); EXPECT_EQ(ts.nano_of_millisecond, 456000); - EXPECT_EQ(rv.GetArraySize(5), 2u); - EXPECT_EQ(rv.GetArrayDecimalString(5, 0), "123.45"); - EXPECT_TRUE(rv.IsArrayElementNull(5, 1)); + auto arr_dec = rv.GetValue(5); + EXPECT_EQ(arr_dec.Size(), 2u); + EXPECT_EQ(arr_dec.At(0).GetDecimalString(), "123.45"); + EXPECT_TRUE(arr_dec.At(1).IsNull()); ASSERT_OK(adm.DropTable(table_path, false)); } @@ -1220,50 +1410,38 @@ TEST_F(LogTableTest, ArrayApiValidationErrors) { ASSERT_TRUE(it != records.end()); auto rec = *it; + auto view = rec.row.GetValue(1); + EXPECT_EQ(view.Type(), fluss::TypeId::Array); + EXPECT_EQ(view.Size(), 2u); + EXPECT_TRUE(view.At(1).IsNull()); + + // Out-of-bounds navigation throws. bool oob_threw = false; try { - (void)rec.row.GetArrayInt32(1, 5); + (void)view.At(5).GetInt32(); } catch (const std::exception&) { oob_threw = true; } EXPECT_TRUE(oob_threw); + // Wrong-type leaf read throws. bool wrong_type_threw = false; try { - (void)rec.row.GetArrayInt64(1, 0); + (void)view.At(0).GetInt64(); } catch (const std::exception&) { wrong_type_threw = true; } EXPECT_TRUE(wrong_type_threw); + // Typed read of a null element throws. bool null_typed_getter_threw = false; try { - (void)rec.row.GetArrayInt32(1, 1); + (void)view.At(1).GetInt32(); } catch (const std::exception&) { null_typed_getter_threw = true; } EXPECT_TRUE(null_typed_getter_threw); - auto view = rec.row.GetArrayView(1); - EXPECT_EQ(view.Size(), 2u); - EXPECT_TRUE(view.IsNull(1)); - - bool view_wrong_type_threw = false; - try { - (void)view.GetInt64(0); - } catch (const std::exception&) { - view_wrong_type_threw = true; - } - EXPECT_TRUE(view_wrong_type_threw); - - bool view_null_typed_getter_threw = false; - try { - (void)view.GetInt32(1); - } catch (const std::exception&) { - view_null_typed_getter_threw = true; - } - EXPECT_TRUE(view_null_typed_getter_threw); - ASSERT_OK(adm.DropTable(table_path, false)); } @@ -1358,46 +1536,52 @@ TEST_F(LogTableTest, AppendAndScanWithArrayEncodingEdgeCases) { const auto& rv = rec.row; // Long strings: heap-encoded variable-length round-trip - EXPECT_EQ(rv.GetArraySize(1), 2u); - EXPECT_EQ(rv.GetArrayString(1, 0), "abcdefgh"); - EXPECT_EQ(rv.GetArrayString(1, 1), "this is a much longer string that definitely exceeds inline"); + auto strs = rv.GetValue(1); + EXPECT_EQ(strs.Size(), 2u); + EXPECT_EQ(strs.At(0).GetString(), "abcdefgh"); + EXPECT_EQ(strs.At(1).GetString(), "this is a much longer string that definitely exceeds inline"); // Non-compact decimal (precision 22 > MAX_COMPACT_PRECISION 18) - EXPECT_EQ(rv.GetArraySize(2), 2u); - EXPECT_EQ(rv.GetArrayDecimalString(2, 0), "12345678901234567.12345"); - EXPECT_EQ(rv.GetArrayDecimalString(2, 1), "-99999999999999999.99999"); + auto decs = rv.GetValue(2); + EXPECT_EQ(decs.Size(), 2u); + EXPECT_EQ(decs.At(0).GetDecimalString(), "12345678901234567.12345"); + EXPECT_EQ(decs.At(1).GetDecimalString(), "-99999999999999999.99999"); // Non-compact timestamp (precision 9 > MAX_COMPACT_TIMESTAMP_PRECISION 3) - EXPECT_EQ(rv.GetArraySize(3), 1u); - auto ts = rv.GetArrayTimestamp(3, 0); + auto tss = rv.GetValue(3); + EXPECT_EQ(tss.Size(), 1u); + auto ts = tss.At(0).GetTimestamp(); EXPECT_EQ(ts.epoch_millis, 1769163227123); EXPECT_EQ(ts.nano_of_millisecond, 456789); // Float NaN / Infinity round-trip - EXPECT_EQ(rv.GetArraySize(4), 3u); - EXPECT_TRUE(std::isnan(rv.GetArrayFloat32(4, 0))); - EXPECT_TRUE(std::isinf(rv.GetArrayFloat32(4, 1))); - EXPECT_GT(rv.GetArrayFloat32(4, 1), 0.0f); - EXPECT_TRUE(std::isinf(rv.GetArrayFloat32(4, 2))); - EXPECT_LT(rv.GetArrayFloat32(4, 2), 0.0f); + auto floats = rv.GetValue(4); + EXPECT_EQ(floats.Size(), 3u); + EXPECT_TRUE(std::isnan(floats.At(0).GetFloat32())); + EXPECT_TRUE(std::isinf(floats.At(1).GetFloat32())); + EXPECT_GT(floats.At(1).GetFloat32(), 0.0f); + EXPECT_TRUE(std::isinf(floats.At(2).GetFloat32())); + EXPECT_LT(floats.At(2).GetFloat32(), 0.0f); // Double NaN / Infinity round-trip - EXPECT_EQ(rv.GetArraySize(5), 3u); - EXPECT_TRUE(std::isnan(rv.GetArrayFloat64(5, 0))); - EXPECT_TRUE(std::isinf(rv.GetArrayFloat64(5, 1))); - EXPECT_GT(rv.GetArrayFloat64(5, 1), 0.0); - EXPECT_TRUE(std::isinf(rv.GetArrayFloat64(5, 2))); - EXPECT_LT(rv.GetArrayFloat64(5, 2), 0.0); + auto doubles = rv.GetValue(5); + EXPECT_EQ(doubles.Size(), 3u); + EXPECT_TRUE(std::isnan(doubles.At(0).GetFloat64())); + EXPECT_TRUE(std::isinf(doubles.At(1).GetFloat64())); + EXPECT_GT(doubles.At(1).GetFloat64(), 0.0); + EXPECT_TRUE(std::isinf(doubles.At(2).GetFloat64())); + EXPECT_LT(doubles.At(2).GetFloat64(), 0.0); // Fixed-length binary round-trip - EXPECT_EQ(rv.GetArraySize(6), 2u); - auto bin = rv.GetArrayBytes(6, 0); + auto bins = rv.GetValue(6); + EXPECT_EQ(bins.Size(), 2u); + auto bin = bins.At(0).GetBytes(); ASSERT_EQ(bin.size(), 4u); EXPECT_EQ(bin[0], 0xDE); EXPECT_EQ(bin[1], 0xAD); EXPECT_EQ(bin[2], 0xBE); EXPECT_EQ(bin[3], 0xEF); - EXPECT_TRUE(rv.IsArrayElementNull(6, 1)); + EXPECT_TRUE(bins.At(1).IsNull()); ASSERT_OK(adm.DropTable(table_path, false)); } diff --git a/website/docs/user-guide/cpp/api-reference.md b/website/docs/user-guide/cpp/api-reference.md index 60a43eb4..46027e6e 100644 --- a/website/docs/user-guide/cpp/api-reference.md +++ b/website/docs/user-guide/cpp/api-reference.md @@ -224,6 +224,8 @@ Performs prefix (bucket-key) lookups, returning all rows whose primary key start | `SetTimestampLtz(size_t idx, const Timestamp& value)` | Set timestamp with timezone | | `SetDecimal(size_t idx, const std::string& value)` | Set decimal from string | | `SetArray(size_t idx, ArrayWriter&& writer)` | Set array value (consumes the writer) | +| `SetMap(size_t idx, MapWriter&& writer)` | Set map value (consumes the writer) | +| `SetRow(size_t idx, GenericRow&& nested)` | Set ROW value from a nested row (consumes it) | ### Name-Based Setters @@ -241,6 +243,9 @@ When using `table.NewRow()`, the `Set()` method auto-routes to the correct type | `Set(const std::string& name, const Date& value)` | Set date by column name | | `Set(const std::string& name, const Time& value)` | Set time by column name | | `Set(const std::string& name, const Timestamp& value)` | Set timestamp by column name | +| `Set(const std::string& name, ArrayWriter&& writer)` | Set array value by column name | +| `Set(const std::string& name, MapWriter&& writer)` | Set map value by column name | +| `Set(const std::string& name, GenericRow&& nested)` | Set ROW value by column name | ## `RowView` @@ -270,27 +275,14 @@ Read-only row view for scan results. Provides zero-copy access to string and byt | `IsDecimal(size_t idx) -> bool` | Check if field is a decimal type| | `GetDecimalString(size_t idx) -> std::string` | Get decimal as string at index | -### Array Getters (Index-Based) - -| Method | Description | -|--------------------------------------------------------------------|-------------------------------------------| -| `GetArraySize(size_t idx) -> size_t` | Get element count of array at index | -| `GetArrayElementType(size_t idx) -> TypeId` | Get element type of array at index | -| `IsArrayElementNull(size_t idx, size_t element) -> bool` | Check if array element is null | -| `GetArrayBool(size_t idx, size_t element) -> bool` | Get boolean array element | -| `GetArrayInt32(size_t idx, size_t element) -> int32_t` | Get 32-bit integer array element | -| `GetArrayInt64(size_t idx, size_t element) -> int64_t` | Get 64-bit integer array element | -| `GetArrayFloat32(size_t idx, size_t element) -> float` | Get 32-bit float array element | -| `GetArrayFloat64(size_t idx, size_t element) -> double` | Get 64-bit float array element | -| `GetArrayString(size_t idx, size_t element) -> std::string` | Get string array element | -| `GetArrayBytes(size_t idx, size_t element) -> std::vector`| Get binary array element | -| `GetArrayDate(size_t idx, size_t element) -> Date` | Get date array element | -| `GetArrayTime(size_t idx, size_t element) -> Time` | Get time array element | -| `GetArrayTimestamp(size_t idx, size_t element) -> Timestamp` | Get timestamp array element | -| `GetArrayDecimalString(size_t idx, size_t element) -> std::string` | Get decimal array element as string | -| `GetArrayView(size_t idx) -> ArrayView` | Get owning ArrayView for nested access | - -All array getters are also available by column name (e.g., `GetArraySize("col")`, `GetArrayView("col")`). +### Complex Getters (ARRAY / MAP / ROW) + +| Method | Description | +|----------------------------------------------|--------------------------------------------------------------| +| `GetValue(size_t idx) -> Value` | Get a complex column as a recursive [`Value`](#value) handle | +| `GetValue(const std::string& name) -> Value` | Same, by column name | + +Navigate the returned [`Value`](#value) with `At` / `KeyAt` / `ValueAt` / `Field` and read leaves with its `Get*()` methods. ### Name-Based Getters @@ -398,9 +390,9 @@ Read-only result for lookup operations. Provides zero-copy access to field value | `IsDecimal(size_t idx) -> bool` | Check if field is a decimal type| | `GetDecimalString(size_t idx) -> std::string` | Get decimal as string at index | -### Array Getters (Index-Based) +### Complex Getters (ARRAY / MAP / ROW) -Same array getters as [`RowView`](#array-getters-index-based) — `GetArraySize`, `GetArrayInt32`, `GetArrayView`, etc. Also available by column name. +`GetValue(size_t idx) -> Value` and `GetValue(const std::string& name) -> Value`, same as on [`RowView`](#rowview). Navigate the returned [`Value`](#value). ### Name-Based Getters @@ -421,7 +413,7 @@ Same array getters as [`RowView`](#array-getters-index-based) — `GetArraySize` ## `PrefixLookupResult` -Read-only result of a prefix lookup — zero or more matched rows. Each row is a `PrefixRowView` exposing the same getters as [`RowView`](#rowview) (index- and name-based, including arrays). +Read-only result of a prefix lookup — zero or more matched rows. Each row is a `PrefixRowView` exposing the same scalar getters as [`RowView`](#rowview) (index- and name-based) plus `GetValue(...)` for complex (ARRAY / MAP / ROW) columns. | Method | Description | |-----------------------------------------|---------------------------------------------------| @@ -456,6 +448,7 @@ Read-only result of a prefix lookup — zero or more matched rows. Each row is a | Method | Description | |-----------------------------------|-----------------------------| | `NewBuilder() -> Schema::Builder` | Create a new schema builder | +| `FromArrow(std::shared_ptr schema, std::vector primary_keys = {}) -> Schema` | Build a schema from an existing Arrow schema (escape hatch; prefer `DataType::Map` / `DataType::Row`) | ## `Schema::Builder` @@ -505,6 +498,10 @@ Read-only result of a prefix lookup — zero or more matched rows. Each row is a | `DataType::TimestampLtz(int precision)` | Timestamp with timezone | | `DataType::Decimal(int precision, int scale)` | Decimal with precision and scale | | `DataType::Array(DataType element)` | Array of the given element type | +| `DataType::Map(DataType key, DataType value)` | Map of key/value types (either may be complex) | +| `DataType::Row(std::vector> fields)` | Row of named fields (types may be complex) | + +`MAP` / `ROW` (and arrays nesting them) compose to any depth; the binding lowers them to Arrow internally when the table is created or a writer is built. ### Accessors @@ -515,15 +512,20 @@ Read-only result of a prefix lookup — zero or more matched rows. Each row is a | `scale() -> int` | Get scale (for Decimal type) | | `nullable() -> bool` | Returns `true` if this type is nullable (default), `false` if `NOT NULL` | | `element_type() -> const DataType*` | Get element type (for Array type, nullptr otherwise) | +| `key_type() / value_type() -> const DataType*` | MAP key / value types (nullptr otherwise) | +| `field_count() -> size_t` | ROW field count (0 otherwise) | +| `field_name(size_t i) -> const std::string&` | ROW field name at `i` | +| `field_type(size_t i) -> const DataType*` | ROW field type at `i` | | `NotNull() -> DataType` | Returns a copy of this type with nullable set to `false` | ## `ArrayWriter` -Write-only builder for array column values. Constructed with a fixed size and element type, then populated element-by-element. Move-only — consumed by `GenericRow::SetArray()` or `ArrayWriter::SetArray()` for nested arrays. +Write-only builder for array column values. Constructed with a fixed size and element type, then populated element-by-element. Move-only — consumed by `GenericRow::SetArray()` or `ArrayWriter::SetArray()` for nested arrays. For arrays whose element is a `ROW` / `MAP`, construct from an Arrow element type (`arrow::struct_(...)` / `arrow::map(...)`). | Method | Description | |-----------------------------------------------------------|-------------------------------------------| -| `ArrayWriter(size_t size, DataType element_type)` | Create an array writer | +| `ArrayWriter(size_t size, DataType element_type)` | Create an array writer; element may be complex (`DataType::Map`/`Row`) | +| `ArrayWriter(size_t size, std::shared_ptr element_type)` | Arrow escape hatch for the element type | | `SetNull(size_t idx)` | Set element to null | | `SetBool(size_t idx, bool value)` | Set boolean element | | `SetInt32(size_t idx, int32_t value)` | Set 32-bit integer element | @@ -538,29 +540,56 @@ Write-only builder for array column values. Constructed with a fixed size and el | `SetTimestampLtz(size_t idx, const Timestamp& value)` | Set timestamp with timezone element | | `SetDecimal(size_t idx, const std::string& value)` | Set decimal element from string | | `SetArray(size_t idx, ArrayWriter&& nested)` | Set nested array element (consumes nested)| - -## `ArrayView` - -Read-only view over an array column value. Obtained from `RowView::GetArrayView()` or `LookupResult::GetArrayView()`, and recursively from `ArrayView::GetArray()` for nested `ARRAY>` columns. Move-only. - -| Method | Description | -|---------------------------------------------------------|-------------------------------------------| -| `Size() -> size_t` | Get element count | -| `ElementType() -> TypeId` | Get element type | -| `IsNull(size_t element) -> bool` | Check if element is null | -| `GetBool(size_t element) -> bool` | Get boolean element | -| `GetInt32(size_t element) -> int32_t` | Get 32-bit integer element | -| `GetInt64(size_t element) -> int64_t` | Get 64-bit integer element | -| `GetFloat32(size_t element) -> float` | Get 32-bit float element | -| `GetFloat64(size_t element) -> double` | Get 64-bit float element | -| `GetString(size_t element) -> std::string` | Get string element | -| `GetBytes(size_t element) -> std::vector` | Get binary element | -| `GetDate(size_t element) -> Date` | Get date element | -| `GetTime(size_t element) -> Time` | Get time element | -| `GetTimestamp(size_t element) -> Timestamp` | Get timestamp element | -| `GetTimestampLtz(size_t element) -> Timestamp` | Get timestamp with timezone element | -| `GetDecimalString(size_t element) -> std::string` | Get decimal element as string | -| `GetArray(size_t element) -> ArrayView` | Get nested array as child ArrayView | +| `SetRow(size_t idx, GenericRow&& row)` | Set ROW element (consumes the row) | +| `SetMap(size_t idx, MapWriter&& map)` | Set MAP element (consumes the map) | + +## `MapWriter` + +Write-only builder for map column values. Construct with the key/value types, then for each entry set the key and value and call `Commit()`. Keys cannot be null; an unset value defaults to null. Move-only — consumed by `GenericRow::SetMap()`, `ArrayWriter::SetMap()`, and `MapWriter::SetValueMap()`. For a key/value that is itself a `ROW` / `MAP` / `ARRAY`, construct from Arrow types and use the `SetValue{Row,Map,Array}` setters. + +| Method | Description | +|---------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------| +| `MapWriter(size_t capacity, DataType key_type, DataType value_type)` | Create a map writer; key/value may be complex (`DataType::Map`/`Row`/`Array`) | +| `MapWriter(size_t capacity, std::shared_ptr key_type, std::shared_ptr value_type)`| Arrow escape hatch for key/value types | +| `SetKey{Bool,Int32,Int64,Float32,Float64,String,Bytes,Date,Time,Timestamp,Decimal}(...)` | Stage the entry key (NTZ vs LTZ chosen from the declared key type)| +| `SetValueNull()` | Stage a null entry value | +| `SetValue{Bool,Int32,Int64,Float32,Float64,String,Bytes,Date,Time,Timestamp,Decimal}(...)` | Stage the entry value (NTZ vs LTZ from the declared value type) | +| `SetValueRow(GenericRow&&)` / `SetValueMap(MapWriter&&)` / `SetValueArray(ArrayWriter&&)` | Stage a compound entry value (consumes the writer) | +| `Commit()` | Append the staged key/value as one map entry | + +## `Value` + +Read-only handle for a complex (`ARRAY` / `MAP` / `ROW`) column value, obtained from `RowView::GetValue()`, `LookupResult::GetValue()`, or `PrefixRowView::GetValue()`. **Navigate** to children with `At` / `KeyAt` / `ValueAt` / `Field` (each returns a child `Value`); **read** a leaf with the `Get*()` methods. A typed read on a null or wrong-typed value throws. Move-only; owns a Rust handle released on destruction. + +### Leaf reads + +| Method | Description | +|-------------------------------------|-------------------------------| +| `Type() -> TypeId` | Type of this value | +| `IsNull() -> bool` | Whether this value is null | +| `GetBool() -> bool` | Read a boolean leaf | +| `GetInt32() -> int32_t` | Read a 32-bit integer leaf | +| `GetInt64() -> int64_t` | Read a 64-bit integer leaf | +| `GetFloat32() -> float` | Read a 32-bit float leaf | +| `GetFloat64() -> double` | Read a 64-bit float leaf | +| `GetString() -> std::string` | Read a string leaf | +| `GetBytes() -> std::vector`| Read a binary leaf | +| `GetDate() -> Date` | Read a date leaf | +| `GetTime() -> Time` | Read a time leaf | +| `GetTimestamp() -> Timestamp` | Read a timestamp leaf | +| `GetDecimalString() -> std::string` | Read a decimal leaf as string | + +### Navigation + +| Method | Description | +|----------------------------------------------|------------------------------------| +| `Size() -> size_t` | Element count (ARRAY) / entries (MAP) | +| `FieldCount() -> size_t` | Field count (ROW) | +| `At(size_t i) -> Value` | ARRAY element `i` | +| `KeyAt(size_t i) -> Value` | MAP key at entry `i` | +| `ValueAt(size_t i) -> Value` | MAP value at entry `i` | +| `Field(size_t i) -> Value` | ROW field `i` | +| `Field(const std::string& name) -> Value` | ROW field by name | ## `TablePath` @@ -730,6 +759,8 @@ inline const char* ChangeTypeShortString(ChangeType ct) { | `TimestampLtz` | Timestamp with timezone | | `Decimal` | Decimal | | `Array` | Array of elements | +| `Map` | Map of key/value pairs | +| `Row` | Row (struct) of fields | ### `ChangeType` diff --git a/website/docs/user-guide/cpp/data-types.md b/website/docs/user-guide/cpp/data-types.md index cce40cef..9b8ab0c6 100644 --- a/website/docs/user-guide/cpp/data-types.md +++ b/website/docs/user-guide/cpp/data-types.md @@ -22,6 +22,11 @@ sidebar_position: 3 | `DataType::TimestampLtz()` | Timestamp with timezone (default precision 6, microseconds) | | `DataType::Decimal(p, s)` | Decimal with precision and scale | | `DataType::Array(element)` | Array of the given element type (supports nesting) | +| `DataType::Map(key, value)`| Map of key/value types (either may itself be complex) | +| `DataType::Row({{name, type}, ...})` | Row (struct) of named fields (types may be complex) | + +`MAP` / `ROW` columns (and arrays nesting them) compose to any depth — see +[Declaring MAP and ROW columns](#declaring-map-and-row-columns). ## Nullability @@ -57,6 +62,36 @@ auto info = table.GetTableInfo(); bool is_nullable = info.schema.columns[0].data_type.nullable(); ``` +## Declaring MAP and ROW columns + +`MAP` and `ROW<...>` columns are declared with the `DataType::Map` and +`DataType::Row` factories, composed like any other type — to any depth: + +```cpp +auto schema = fluss::Schema::NewBuilder() + .AddColumn("id", fluss::DataType::Int()) + .AddColumn("attrs", fluss::DataType::Map(fluss::DataType::String(), + fluss::DataType::Int())) + .AddColumn("profile", fluss::DataType::Row({ + {"seq", fluss::DataType::Int()}, + {"label", fluss::DataType::String()}, + })) + // arbitrarily nested, e.g. array>>: + .AddColumn("events", fluss::DataType::Array(fluss::DataType::Row({ + {"seq", fluss::DataType::Int()}, + {"attrs", fluss::DataType::Map(fluss::DataType::String(), fluss::DataType::Int())}, + }))) + .SetPrimaryKeys({"id"}) + .Build(); +``` + +:::note Arrow escape hatch +If you already have an `arrow::Schema`, pass it directly with +`fluss::Schema::FromArrow(arrow_schema, /*primary_keys=*/{"id"})`. It's +equivalent — the native factories above lower to the same Arrow types +internally, without pulling Arrow into your code. +::: + ## GenericRow Setters `SetInt32` is used for `TinyInt`, `SmallInt`, and `Int` columns. For `TinyInt` and `SmallInt`, the value is validated at write time — an error is returned if it overflows the column's range (e.g., \[-128, 127\] for `TinyInt`, \[-32768, 32767\] for `SmallInt`). @@ -97,6 +132,43 @@ outer.SetArray(0, std::move(inner)); row.SetArray(9, std::move(outer)); ``` +### Map Columns + +Map values are built with `MapWriter`: stage each entry's key and value, then +`Commit()`. Keys cannot be null; an unset value defaults to null. + +```cpp +fluss::MapWriter mw(2, fluss::DataType::String(), fluss::DataType::Int()); +mw.SetKeyString("a"); mw.SetValueInt32(1); mw.Commit(); +mw.SetKeyString("b"); mw.SetValueNull(); mw.Commit(); +row.SetMap(10, std::move(mw)); +``` + +When the key or value is itself a `ROW` / `MAP` / `ARRAY`, pass the compound type +natively and stage values with the compound setters (`SetValueRow` / +`SetValueMap` / `SetValueArray`): + +```cpp +fluss::MapWriter mr(1, fluss::DataType::String(), + fluss::DataType::Row({{"seq", fluss::DataType::Int()}})); +mr.SetKeyString("k"); +fluss::GenericRow v(1); v.SetInt32(0, 7); +mr.SetValueRow(std::move(v)); +mr.Commit(); +row.SetMap(11, std::move(mr)); +``` + +### Row Columns + +A `ROW` value is a nested, schema-less `GenericRow`, attached via `SetRow`: + +```cpp +fluss::GenericRow nested(2); +nested.SetInt32(0, 7); +nested.SetString(1, "seven"); +row.SetRow(12, std::move(nested)); +``` + ## Name-Based Setters When using `table.NewRow()`, you can set fields by column name. The setter automatically routes to the correct type based on the schema: @@ -168,37 +240,45 @@ if (result.Found()) { } ``` -### Reading Array Columns +### Reading Complex Columns (ARRAY / MAP / ROW) -Array columns can be read element-by-element using index-based getters, or via an `ArrayView` for recursive access: +Complex columns are read through a single recursive `Value` handle returned by +`GetValue(idx)` / `GetValue(name)`. **Navigate** the structure with +`At` / `KeyAt` / `ValueAt` / `Field` (each returns a child `Value`); **read** a +leaf with the `Get*()` methods (the handle already points at one value, so no +index): ```cpp -// Element-by-element access (flat arrays) -size_t len = rec.row.GetArraySize(8); -for (size_t i = 0; i < len; i++) { - if (!rec.row.IsArrayElementNull(8, i)) { - int32_t val = rec.row.GetArrayInt32(8, i); +// ARRAY +auto arr = rec.row.GetValue("ids"); +for (size_t i = 0; i < arr.Size(); i++) { + if (!arr.At(i).IsNull()) { + int32_t v = arr.At(i).GetInt32(); } } -// ArrayView for nested arrays or when you need a standalone handle -fluss::ArrayView av = rec.row.GetArrayView(8); -for (size_t i = 0; i < av.Size(); i++) { - if (!av.IsNull(i)) { - int32_t val = av.GetInt32(i); +// MAP — entries addressed by position +auto m = rec.row.GetValue("attrs"); +for (size_t i = 0; i < m.Size(); i++) { + auto key = m.KeyAt(i).GetString(); + if (!m.ValueAt(i).IsNull()) { + int32_t v = m.ValueAt(i).GetInt32(); } } -// Nested arrays: ArrayView::GetArray() returns a child ArrayView -fluss::ArrayView outer = rec.row.GetArrayView(9); -for (size_t i = 0; i < outer.Size(); i++) { - fluss::ArrayView inner = outer.GetArray(i); - for (size_t j = 0; j < inner.Size(); j++) { - int32_t val = inner.GetInt32(j); - } -} +// ROW — fields by index or name +auto r = rec.row.GetValue("profile"); +int32_t seq = r.Field(0).GetInt32(); +auto label = r.Field("label").GetString(); + +// Nesting is just chained navigation, with the same Get* leaf reads: +auto rows = rec.row.GetValue("arr_of_rows"); // ARRAY> +int32_t first_seq = rows.At(0).Field("seq").GetInt32(); ``` +`GetValue` works the same on `RowView` (scan), `LookupResult`, and +`PrefixRowView`. A typed `Get*()` on a null or wrong-typed value throws. + ## TypeId Enum `TinyInt` and `SmallInt` values are widened to `int32_t` on read. @@ -219,7 +299,9 @@ for (size_t i = 0; i < outer.Size(); i++) { | `Timestamp` | `Timestamp` | `GetTimestamp(idx)` | | `TimestampLtz` | `Timestamp` | `GetTimestamp(idx)` | | `Decimal` | `std::string` | `GetDecimalString(idx)` | -| `Array` | `ArrayView` | `GetArrayView(idx)` | +| `Array` | `Value` | `GetValue(idx)` | +| `Map` | `Value` | `GetValue(idx)` | +| `Row` | `Value` | `GetValue(idx)` | ## Type Checking diff --git a/website/docs/user-guide/cpp/example/prefix-lookup.md b/website/docs/user-guide/cpp/example/prefix-lookup.md index e5d52015..93f7a052 100644 --- a/website/docs/user-guide/cpp/example/prefix-lookup.md +++ b/website/docs/user-guide/cpp/example/prefix-lookup.md @@ -63,6 +63,8 @@ for (size_t i = 0; i < result.Size(); ++i) { Unlike primary-key lookup (which returns a single row via `LookupResult::Found()`), prefix lookup returns zero or more rows via `Size()` / `GetRow(i)`, in primary-key order. +Each `GetRow(i)` is a `PrefixRowView` with the same getters as a scan `RowView`: scalars by index or name, and complex (`ARRAY` / `MAP` / `ROW`) columns via `GetValue(...)` — see [Reading Complex Columns](../data-types.md#reading-complex-columns-array--map--row). + ## Partitioned Table On a partitioned table, the partition columns are stripped from the primary key before the bucket-prefix rule is evaluated. The lookup key must still carry the partition values so the client can route the request to the right partition — so the columns passed to `NewPrefixLookup()` are `partition_keys ++ bucket_key`.