From 58d145c8892c1d902b901d9cac8931b8c569db3f Mon Sep 17 00:00:00 2001 From: Rusty Conover Date: Wed, 2 Sep 2026 14:04:59 -0400 Subject: [PATCH 1/7] feat: append array views to arrays --- src/nanoarrow/common/array.c | 168 +++++++++++++++++++++ src/nanoarrow/common/array_test.cc | 229 +++++++++++++++++++++++++++++ src/nanoarrow/nanoarrow.h | 12 ++ 3 files changed, 409 insertions(+) diff --git a/src/nanoarrow/common/array.c b/src/nanoarrow/common/array.c index 68c42d0d2..7f867ec1c 100644 --- a/src/nanoarrow/common/array.c +++ b/src/nanoarrow/common/array.c @@ -77,6 +77,174 @@ int ArrowArrayIsInternal(struct ArrowArray* array) { return array->release == &ArrowArrayReleaseInternal; } +static ArrowErrorCode ArrowArrayAppendArrayViewElement(struct ArrowArray* dst, + const struct ArrowArrayView* src, + int64_t i, + struct ArrowError* error) { + if (ArrowArrayViewIsNull(src, i)) { + return ArrowArrayAppendNull(dst, 1); + } + + switch (src->storage_type) { + case NANOARROW_TYPE_NA: + return ArrowArrayAppendNull(dst, 1); + case NANOARROW_TYPE_BOOL: + case NANOARROW_TYPE_INT8: + case NANOARROW_TYPE_INT16: + case NANOARROW_TYPE_INT32: + case NANOARROW_TYPE_INT64: + case NANOARROW_TYPE_DATE32: + case NANOARROW_TYPE_DATE64: + case NANOARROW_TYPE_TIMESTAMP: + case NANOARROW_TYPE_TIME32: + case NANOARROW_TYPE_TIME64: + case NANOARROW_TYPE_DURATION: + return ArrowArrayAppendInt(dst, ArrowArrayViewGetIntUnsafe(src, i)); + case NANOARROW_TYPE_UINT8: + case NANOARROW_TYPE_UINT16: + case NANOARROW_TYPE_UINT32: + case NANOARROW_TYPE_UINT64: + return ArrowArrayAppendUInt(dst, ArrowArrayViewGetUIntUnsafe(src, i)); + case NANOARROW_TYPE_HALF_FLOAT: + case NANOARROW_TYPE_FLOAT: + case NANOARROW_TYPE_DOUBLE: + return ArrowArrayAppendDouble(dst, ArrowArrayViewGetDoubleUnsafe(src, i)); + case NANOARROW_TYPE_STRING: + case NANOARROW_TYPE_BINARY: + case NANOARROW_TYPE_FIXED_SIZE_BINARY: + case NANOARROW_TYPE_LARGE_STRING: + case NANOARROW_TYPE_LARGE_BINARY: + case NANOARROW_TYPE_BINARY_VIEW: + case NANOARROW_TYPE_STRING_VIEW: + return ArrowArrayAppendBytes(dst, ArrowArrayViewGetBytesUnsafe(src, i)); + case NANOARROW_TYPE_INTERVAL_MONTHS: + case NANOARROW_TYPE_INTERVAL_DAY_TIME: + case NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO: { + struct ArrowInterval interval; + ArrowIntervalInit(&interval, src->storage_type); + ArrowArrayViewGetIntervalUnsafe(src, i, &interval); + return ArrowArrayAppendInterval(dst, &interval); + } + case NANOARROW_TYPE_DECIMAL32: + case NANOARROW_TYPE_DECIMAL64: + case NANOARROW_TYPE_DECIMAL128: + case NANOARROW_TYPE_DECIMAL256: { + int32_t bitwidth = 32; + if (src->storage_type == NANOARROW_TYPE_DECIMAL64) { + bitwidth = 64; + } else if (src->storage_type == NANOARROW_TYPE_DECIMAL128) { + bitwidth = 128; + } else if (src->storage_type == NANOARROW_TYPE_DECIMAL256) { + bitwidth = 256; + } + + struct ArrowDecimal decimal; + ArrowDecimalInit(&decimal, bitwidth, /*precision=*/0, /*scale=*/0); + ArrowArrayViewGetDecimalUnsafe(src, i, &decimal); + return ArrowArrayAppendDecimal(dst, &decimal); + } + case NANOARROW_TYPE_STRUCT: + for (int64_t child_i = 0; child_i < src->n_children; child_i++) { + NANOARROW_RETURN_NOT_OK(ArrowArrayAppendArrayViewElement( + dst->children[child_i], src->children[child_i], src->offset + i, error)); + } + return ArrowArrayFinishElement(dst); + case NANOARROW_TYPE_LIST: + case NANOARROW_TYPE_LARGE_LIST: + case NANOARROW_TYPE_MAP: + case NANOARROW_TYPE_LIST_VIEW: + case NANOARROW_TYPE_LARGE_LIST_VIEW: { + int64_t logical_i = src->offset + i; + int64_t child_offset = ArrowArrayViewListChildOffset(src, logical_i); + int64_t child_length; + if (src->storage_type == NANOARROW_TYPE_LIST_VIEW) { + child_length = src->buffer_views[2].data.as_int32[logical_i]; + } else if (src->storage_type == NANOARROW_TYPE_LARGE_LIST_VIEW) { + child_length = src->buffer_views[2].data.as_int64[logical_i]; + } else { + child_length = ArrowArrayViewListChildOffset(src, logical_i + 1) - child_offset; + } + + for (int64_t child_i = 0; child_i < child_length; child_i++) { + NANOARROW_RETURN_NOT_OK(ArrowArrayAppendArrayViewElement( + dst->children[0], src->children[0], child_offset + child_i, error)); + } + return ArrowArrayFinishElement(dst); + } + case NANOARROW_TYPE_FIXED_SIZE_LIST: { + int64_t child_offset = (src->offset + i) * src->layout.child_size_elements; + for (int64_t child_i = 0; child_i < src->layout.child_size_elements; child_i++) { + NANOARROW_RETURN_NOT_OK(ArrowArrayAppendArrayViewElement( + dst->children[0], src->children[0], child_offset + child_i, error)); + } + return ArrowArrayFinishElement(dst); + } + case NANOARROW_TYPE_DENSE_UNION: + case NANOARROW_TYPE_SPARSE_UNION: { + int8_t type_id = ArrowArrayViewUnionTypeId(src, i); + int8_t child_index = ArrowArrayViewUnionChildIndex(src, i); + int64_t child_offset = ArrowArrayViewUnionChildOffset(src, i); + NANOARROW_RETURN_NOT_OK(ArrowArrayAppendArrayViewElement( + dst->children[child_index], src->children[child_index], child_offset, error)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppend(ArrowArrayBuffer(dst, 0), &type_id, sizeof(type_id))); + if (src->storage_type == NANOARROW_TYPE_DENSE_UNION) { + _NANOARROW_CHECK_RANGE(dst->children[child_index]->length - 1, 0, INT32_MAX); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32( + ArrowArrayBuffer(dst, 1), (int32_t)dst->children[child_index]->length - 1)); + } else { + for (int64_t child_i = 0; child_i < dst->n_children; child_i++) { + if (child_i != child_index && dst->children[child_i]->length == dst->length) { + NANOARROW_RETURN_NOT_OK(ArrowArrayAppendEmpty(dst->children[child_i], 1)); + } + if (dst->children[child_i]->length != dst->length + 1) { + ArrowErrorSet(error, + "Expected sparse union child length of %" PRId64 + " but found %" PRId64, + dst->length + 1, dst->children[child_i]->length); + return EINVAL; + } + } + } + dst->length++; + return NANOARROW_OK; + } + case NANOARROW_TYPE_RUN_END_ENCODED: + case NANOARROW_TYPE_UNINITIALIZED: + default: + ArrowErrorSet(error, "Appending array views is not supported for %s", + ArrowTypeString(src->storage_type)); + return ENOTSUP; + } +} + +ArrowErrorCode ArrowArrayAppendArrayView(struct ArrowArray* dst, + const struct ArrowArrayView* src, + struct ArrowError* error) { + if (src->storage_type == NANOARROW_TYPE_RUN_END_ENCODED) { + if (src->offset != 0) { + ArrowErrorSet(error, "Can't append a sliced run-end encoded array view"); + return ENOTSUP; + } + + int64_t run_end_offset = dst->length; + for (int64_t i = 0; i < src->children[0]->length; i++) { + NANOARROW_RETURN_NOT_OK(ArrowArrayAppendInt( + dst->children[0], + run_end_offset + ArrowArrayViewGetIntUnsafe(src->children[0], i))); + NANOARROW_RETURN_NOT_OK( + ArrowArrayAppendArrayViewElement(dst->children[1], src->children[1], i, error)); + } + dst->length += src->length; + return NANOARROW_OK; + } + + for (int64_t i = 0; i < src->length; i++) { + NANOARROW_RETURN_NOT_OK(ArrowArrayAppendArrayViewElement(dst, src, i, error)); + } + return NANOARROW_OK; +} + static ArrowErrorCode ArrowArraySetStorageType(struct ArrowArray* array, enum ArrowType storage_type) { switch (storage_type) { diff --git a/src/nanoarrow/common/array_test.cc b/src/nanoarrow/common/array_test.cc index bc4686c2a..2757e3241 100644 --- a/src/nanoarrow/common/array_test.cc +++ b/src/nanoarrow/common/array_test.cc @@ -5087,3 +5087,232 @@ TEST(ArrayMoveSharedTest, ArrayWithNullBuffers) { ArrowArrayRelease(&shared); } + +static ArrowErrorCode AppendArrayViewForTest(const struct ArrowArrayView* src, + struct ArrowArray* dst, + struct ArrowError* error) { + NANOARROW_RETURN_NOT_OK(ArrowArrayInitFromArrayView(dst, src, error)); + NANOARROW_RETURN_NOT_OK(ArrowArrayStartAppending(dst)); + NANOARROW_RETURN_NOT_OK(ArrowArrayAppendArrayView(dst, src, error)); + return ArrowArrayFinishBuildingDefault(dst, error); +} + +static void ExpectPrimitiveArrayViewAppendIdentical(struct ArrowArray* src, + enum ArrowType type) { + struct ArrowError error; + struct ArrowArrayView src_view; + ArrowArrayViewInitFromType(&src_view, type); + ASSERT_EQ(ArrowArrayViewSetArray(&src_view, src, &error), NANOARROW_OK) + << error.message; + + struct ArrowArray dst; + ASSERT_EQ(AppendArrayViewForTest(&src_view, &dst, &error), NANOARROW_OK) + << error.message; + struct ArrowArrayView dst_view; + ArrowArrayViewInitFromType(&dst_view, type); + ASSERT_EQ(ArrowArrayViewSetArray(&dst_view, &dst, &error), NANOARROW_OK) + << error.message; + + int identical = 0; + ASSERT_EQ(ArrowArrayViewCompare(&src_view, &dst_view, NANOARROW_COMPARE_IDENTICAL, + &identical, &error), + NANOARROW_OK); + EXPECT_EQ(identical, 1) << error.message; + + ArrowArrayViewReset(&dst_view); + ArrowArrayRelease(&dst); + ArrowArrayViewReset(&src_view); +} + +TEST(ArrayTest, ArrayAppendArrayViewPrimitiveTypes) { + struct ArrowArray array; + + ASSERT_EQ(ArrowArrayInitFromType(&array, NANOARROW_TYPE_INT64), NANOARROW_OK); + ASSERT_EQ(ArrowArrayStartAppending(&array), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendInt(&array, -42), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendNull(&array, 1), NANOARROW_OK); + ASSERT_EQ(ArrowArrayFinishBuildingDefault(&array, nullptr), NANOARROW_OK); + ExpectPrimitiveArrayViewAppendIdentical(&array, NANOARROW_TYPE_INT64); + ArrowArrayRelease(&array); + + ASSERT_EQ(ArrowArrayInitFromType(&array, NANOARROW_TYPE_UINT64), NANOARROW_OK); + ASSERT_EQ(ArrowArrayStartAppending(&array), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendUInt(&array, UINT64_MAX), NANOARROW_OK); + ASSERT_EQ(ArrowArrayFinishBuildingDefault(&array, nullptr), NANOARROW_OK); + ExpectPrimitiveArrayViewAppendIdentical(&array, NANOARROW_TYPE_UINT64); + ArrowArrayRelease(&array); + + ASSERT_EQ(ArrowArrayInitFromType(&array, NANOARROW_TYPE_DOUBLE), NANOARROW_OK); + ASSERT_EQ(ArrowArrayStartAppending(&array), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendDouble(&array, 1.25), NANOARROW_OK); + ASSERT_EQ(ArrowArrayFinishBuildingDefault(&array, nullptr), NANOARROW_OK); + ExpectPrimitiveArrayViewAppendIdentical(&array, NANOARROW_TYPE_DOUBLE); + ArrowArrayRelease(&array); + + ASSERT_EQ(ArrowArrayInitFromType(&array, NANOARROW_TYPE_STRING), NANOARROW_OK); + ASSERT_EQ(ArrowArrayStartAppending(&array), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendString(&array, "some value"_asv), NANOARROW_OK); + ASSERT_EQ(ArrowArrayFinishBuildingDefault(&array, nullptr), NANOARROW_OK); + ExpectPrimitiveArrayViewAppendIdentical(&array, NANOARROW_TYPE_STRING); + ArrowArrayRelease(&array); + + ASSERT_EQ(ArrowArrayInitFromType(&array, NANOARROW_TYPE_INTERVAL_MONTHS), NANOARROW_OK); + ASSERT_EQ(ArrowArrayStartAppending(&array), NANOARROW_OK); + struct ArrowInterval interval; + ArrowIntervalInit(&interval, NANOARROW_TYPE_INTERVAL_MONTHS); + interval.months = 42; + ASSERT_EQ(ArrowArrayAppendInterval(&array, &interval), NANOARROW_OK); + ASSERT_EQ(ArrowArrayFinishBuildingDefault(&array, nullptr), NANOARROW_OK); + ExpectPrimitiveArrayViewAppendIdentical(&array, NANOARROW_TYPE_INTERVAL_MONTHS); + ArrowArrayRelease(&array); + + ASSERT_EQ(ArrowArrayInitFromType(&array, NANOARROW_TYPE_DECIMAL128), NANOARROW_OK); + ASSERT_EQ(ArrowArrayStartAppending(&array), NANOARROW_OK); + struct ArrowDecimal decimal; + ArrowDecimalInit(&decimal, 128, 10, 2); + ArrowDecimalSetInt(&decimal, 1234); + ASSERT_EQ(ArrowArrayAppendDecimal(&array, &decimal), NANOARROW_OK); + ASSERT_EQ(ArrowArrayFinishBuildingDefault(&array, nullptr), NANOARROW_OK); + ExpectPrimitiveArrayViewAppendIdentical(&array, NANOARROW_TYPE_DECIMAL128); + ArrowArrayRelease(&array); +} + +TEST(ArrayTest, ArrayAppendArrayViewNestedAndSliced) { + struct ArrowError error; + struct ArrowSchema schema; + ArrowSchemaInit(&schema); + ASSERT_EQ(ArrowSchemaSetTypeStruct(&schema, 2), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetType(schema.children[0], NANOARROW_TYPE_LIST), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetType(schema.children[0]->children[0], NANOARROW_TYPE_INT32), + NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetType(schema.children[1], NANOARROW_TYPE_STRING), NANOARROW_OK); + + struct ArrowArray src; + ASSERT_EQ(ArrowArrayInitFromSchema(&src, &schema, &error), NANOARROW_OK); + ASSERT_EQ(ArrowArrayStartAppending(&src), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendInt(src.children[0]->children[0], 1), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendInt(src.children[0]->children[0], 2), NANOARROW_OK); + ASSERT_EQ(ArrowArrayFinishElement(src.children[0]), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendString(src.children[1], "first"_asv), NANOARROW_OK); + ASSERT_EQ(ArrowArrayFinishElement(&src), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendNull(&src, 1), NANOARROW_OK); + ASSERT_EQ(ArrowArrayFinishElement(src.children[0]), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendString(src.children[1], "third"_asv), NANOARROW_OK); + ASSERT_EQ(ArrowArrayFinishElement(&src), NANOARROW_OK); + ASSERT_EQ(ArrowArrayFinishBuildingDefault(&src, &error), NANOARROW_OK) << error.message; + + struct ArrowArrayView src_view; + ASSERT_EQ(ArrowArrayViewInitFromSchema(&src_view, &schema, &error), NANOARROW_OK); + ASSERT_EQ(ArrowArrayViewSetArray(&src_view, &src, &error), NANOARROW_OK); + src_view.offset = 1; + src_view.length = 2; + src_view.null_count = 1; + + struct ArrowArray dst; + ASSERT_EQ(AppendArrayViewForTest(&src_view, &dst, &error), NANOARROW_OK) + << error.message; + EXPECT_EQ(dst.length, 2); + EXPECT_EQ(dst.children[0]->length, 2); + EXPECT_EQ(dst.children[0]->children[0]->length, 0); + EXPECT_EQ(dst.children[1]->length, 2); + + struct ArrowArrayView dst_view; + ASSERT_EQ(ArrowArrayViewInitFromSchema(&dst_view, &schema, &error), NANOARROW_OK); + ASSERT_EQ(ArrowArrayViewSetArray(&dst_view, &dst, &error), NANOARROW_OK); + EXPECT_TRUE(ArrowArrayViewIsNull(&dst_view, 0)); + EXPECT_EQ(ArrowArrayViewGetStringUnsafe(dst_view.children[1], 1), "third"_asv); + + ArrowArrayViewReset(&dst_view); + ArrowArrayRelease(&dst); + ArrowArrayViewReset(&src_view); + ArrowArrayRelease(&src); + ArrowSchemaRelease(&schema); +} + +TEST(ArrayTest, ArrayAppendArrayViewUnions) { + for (enum ArrowType union_type : + {NANOARROW_TYPE_DENSE_UNION, NANOARROW_TYPE_SPARSE_UNION}) { + struct ArrowError error; + struct ArrowSchema schema; + ArrowSchemaInit(&schema); + ASSERT_EQ(ArrowSchemaSetTypeUnion(&schema, union_type, 2), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetType(schema.children[0], NANOARROW_TYPE_INT32), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetType(schema.children[1], NANOARROW_TYPE_STRING), + NANOARROW_OK); + + struct ArrowArray src; + ASSERT_EQ(ArrowArrayInitFromSchema(&src, &schema, &error), NANOARROW_OK); + ASSERT_EQ(ArrowArrayStartAppending(&src), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendInt(src.children[0], 42), NANOARROW_OK); + ASSERT_EQ(ArrowArrayFinishUnionElement(&src, 0), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendString(src.children[1], "value"_asv), NANOARROW_OK); + ASSERT_EQ(ArrowArrayFinishUnionElement(&src, 1), NANOARROW_OK); + ASSERT_EQ(ArrowArrayFinishBuildingDefault(&src, &error), NANOARROW_OK); + + struct ArrowArrayView src_view; + ASSERT_EQ(ArrowArrayViewInitFromSchema(&src_view, &schema, &error), NANOARROW_OK); + ASSERT_EQ(ArrowArrayViewSetArray(&src_view, &src, &error), NANOARROW_OK); + struct ArrowArray dst; + ASSERT_EQ(AppendArrayViewForTest(&src_view, &dst, &error), NANOARROW_OK) + << error.message; + EXPECT_EQ(dst.length, 2); + + struct ArrowArrayView dst_view; + ASSERT_EQ(ArrowArrayViewInitFromSchema(&dst_view, &schema, &error), NANOARROW_OK); + ASSERT_EQ(ArrowArrayViewSetArray(&dst_view, &dst, &error), NANOARROW_OK); + EXPECT_EQ(ArrowArrayViewUnionTypeId(&dst_view, 0), 0); + EXPECT_EQ(ArrowArrayViewUnionTypeId(&dst_view, 1), 1); + + ArrowArrayViewReset(&dst_view); + ArrowArrayRelease(&dst); + ArrowArrayViewReset(&src_view); + ArrowArrayRelease(&src); + ArrowSchemaRelease(&schema); + } +} + +TEST(ArrayTest, ArrayAppendArrayViewRunEndEncoded) { + struct ArrowError error; + struct ArrowSchema schema; + ArrowSchemaInit(&schema); + ASSERT_EQ(ArrowSchemaSetTypeRunEndEncoded(&schema, NANOARROW_TYPE_INT32), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetType(schema.children[1], NANOARROW_TYPE_STRING), NANOARROW_OK); + + struct ArrowArray src; + ASSERT_EQ(ArrowArrayInitFromSchema(&src, &schema, &error), NANOARROW_OK); + ASSERT_EQ(ArrowArrayStartAppending(&src), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendInt(src.children[0], 2), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendInt(src.children[0], 3), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendString(src.children[1], "a"_asv), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendString(src.children[1], "b"_asv), NANOARROW_OK); + src.length = 3; + ASSERT_EQ(ArrowArrayFinishBuildingDefault(&src, &error), NANOARROW_OK); + + struct ArrowArrayView src_view; + ASSERT_EQ(ArrowArrayViewInitFromSchema(&src_view, &schema, &error), NANOARROW_OK); + ASSERT_EQ(ArrowArrayViewSetArray(&src_view, &src, &error), NANOARROW_OK); + struct ArrowArray dst; + ASSERT_EQ(ArrowArrayInitFromArrayView(&dst, &src_view, &error), NANOARROW_OK); + ASSERT_EQ(ArrowArrayStartAppending(&dst), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendArrayView(&dst, &src_view, &error), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendArrayView(&dst, &src_view, &error), NANOARROW_OK); + ASSERT_EQ(ArrowArrayFinishBuildingDefault(&dst, &error), NANOARROW_OK) << error.message; + EXPECT_EQ(dst.length, 6); + + struct ArrowArrayView dst_view; + ASSERT_EQ(ArrowArrayViewInitFromSchema(&dst_view, &schema, &error), NANOARROW_OK); + ASSERT_EQ(ArrowArrayViewSetArray(&dst_view, &dst, &error), NANOARROW_OK); + EXPECT_EQ(ArrowArrayViewGetIntUnsafe(dst_view.children[0], 2), 5); + EXPECT_EQ(ArrowArrayViewGetIntUnsafe(dst_view.children[0], 3), 6); + + src_view.offset = 1; + src_view.length = 1; + EXPECT_EQ(ArrowArrayAppendArrayView(&dst, &src_view, &error), ENOTSUP); + EXPECT_STREQ(error.message, "Can't append a sliced run-end encoded array view"); + + ArrowArrayViewReset(&dst_view); + ArrowArrayRelease(&dst); + ArrowArrayViewReset(&src_view); + ArrowArrayRelease(&src); + ArrowSchemaRelease(&schema); +} diff --git a/src/nanoarrow/nanoarrow.h b/src/nanoarrow/nanoarrow.h index 564db40a4..4e3c7e63d 100644 --- a/src/nanoarrow/nanoarrow.h +++ b/src/nanoarrow/nanoarrow.h @@ -122,6 +122,8 @@ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArraySetValidityBitmap) #define ArrowArraySetBuffer NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArraySetBuffer) #define ArrowArrayReserve NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayReserve) +#define ArrowArrayAppendArrayView \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayAppendArrayView) #define ArrowArrayFinishBuilding \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayFinishBuilding) #define ArrowArrayFinishBuildingDefault \ @@ -1032,6 +1034,16 @@ static inline ArrowErrorCode ArrowArrayStartAppending(struct ArrowArray* array); NANOARROW_DLL ArrowErrorCode ArrowArrayReserve(struct ArrowArray* array, int64_t additional_size_elements); +/// \brief Append the contents of an ArrowArrayView to an ArrowArray +/// +/// Appends each logical element of array_view to array. array must have been +/// initialized with the same storage type and layout as array_view and prepared +/// using ArrowArrayStartAppending(). Dictionary values referenced by array_view +/// are not copied. Returns ENOTSUP for unsupported storage types. +NANOARROW_DLL ArrowErrorCode ArrowArrayAppendArrayView( + struct ArrowArray* array, const struct ArrowArrayView* array_view, + struct ArrowError* error); + /// \brief Append a null value to an array static inline ArrowErrorCode ArrowArrayAppendNull(struct ArrowArray* array, int64_t n); From bab29ade58c0906df62e4866bbb87001df51b7ac Mon Sep 17 00:00:00 2001 From: Rusty Conover Date: Thu, 3 Sep 2026 09:00:49 -0400 Subject: [PATCH 2/7] refactor: validate array view storage appends --- src/nanoarrow/common/array.c | 318 +++++++++++++++++++++++++---- src/nanoarrow/common/array_test.cc | 13 +- src/nanoarrow/nanoarrow.h | 17 +- 3 files changed, 293 insertions(+), 55 deletions(-) diff --git a/src/nanoarrow/common/array.c b/src/nanoarrow/common/array.c index 7f867ec1c..385eea048 100644 --- a/src/nanoarrow/common/array.c +++ b/src/nanoarrow/common/array.c @@ -77,17 +77,172 @@ int ArrowArrayIsInternal(struct ArrowArray* array) { return array->release == &ArrowArrayReleaseInternal; } -static ArrowErrorCode ArrowArrayAppendArrayViewElement(struct ArrowArray* dst, - const struct ArrowArrayView* src, - int64_t i, - struct ArrowError* error) { +static int ArrowTypeIsSignedInteger(enum ArrowType type) { + switch (type) { + case NANOARROW_TYPE_BOOL: + case NANOARROW_TYPE_INT8: + case NANOARROW_TYPE_INT16: + case NANOARROW_TYPE_INT32: + case NANOARROW_TYPE_INT64: + case NANOARROW_TYPE_DATE32: + case NANOARROW_TYPE_DATE64: + case NANOARROW_TYPE_TIMESTAMP: + case NANOARROW_TYPE_TIME32: + case NANOARROW_TYPE_TIME64: + case NANOARROW_TYPE_DURATION: + return 1; + default: + return 0; + } +} + +static int ArrowTypeIsUnsignedInteger(enum ArrowType type) { + switch (type) { + case NANOARROW_TYPE_UINT8: + case NANOARROW_TYPE_UINT16: + case NANOARROW_TYPE_UINT32: + case NANOARROW_TYPE_UINT64: + return 1; + default: + return 0; + } +} + +static int ArrowTypeIsFloatingPoint(enum ArrowType type) { + return type == NANOARROW_TYPE_HALF_FLOAT || type == NANOARROW_TYPE_FLOAT || + type == NANOARROW_TYPE_DOUBLE; +} + +static int ArrowTypeIsBinaryLike(enum ArrowType type) { + switch (type) { + case NANOARROW_TYPE_STRING: + case NANOARROW_TYPE_BINARY: + case NANOARROW_TYPE_FIXED_SIZE_BINARY: + case NANOARROW_TYPE_LARGE_STRING: + case NANOARROW_TYPE_LARGE_BINARY: + case NANOARROW_TYPE_BINARY_VIEW: + case NANOARROW_TYPE_STRING_VIEW: + return 1; + default: + return 0; + } +} + +static int ArrowTypeIsListLike(enum ArrowType type) { + switch (type) { + case NANOARROW_TYPE_LIST: + case NANOARROW_TYPE_LARGE_LIST: + case NANOARROW_TYPE_MAP: + case NANOARROW_TYPE_LIST_VIEW: + case NANOARROW_TYPE_LARGE_LIST_VIEW: + return 1; + default: + return 0; + } +} + +static int ArrowArrayCanAppendStorageType(enum ArrowType dst_type, + enum ArrowType src_type) { + if (src_type == NANOARROW_TYPE_NA) { + return 1; + } + + if (ArrowTypeIsSignedInteger(src_type) || ArrowTypeIsUnsignedInteger(src_type)) { + return ArrowTypeIsSignedInteger(dst_type) || ArrowTypeIsUnsignedInteger(dst_type) || + ArrowTypeIsFloatingPoint(dst_type); + } + + if (ArrowTypeIsFloatingPoint(src_type)) { + return ArrowTypeIsFloatingPoint(dst_type); + } + + if (ArrowTypeIsBinaryLike(src_type)) { + return ArrowTypeIsBinaryLike(dst_type); + } + + if (ArrowTypeIsListLike(src_type)) { + return ArrowTypeIsListLike(dst_type); + } + + return dst_type == src_type; +} + +static ArrowErrorCode ArrowArrayCheckCanAppendStorageFromArrayView( + struct ArrowArray* dst, const struct ArrowArrayView* src, struct ArrowError* error) { + if (!ArrowArrayIsInternal(dst)) { + ArrowErrorSet(error, "Expected destination to be an internal ArrowArray"); + return EINVAL; + } + + struct ArrowArrayPrivateData* private_data = + (struct ArrowArrayPrivateData*)dst->private_data; + enum ArrowType dst_type = private_data->storage_type; + if (!ArrowArrayCanAppendStorageType(dst_type, src->storage_type)) { + ArrowErrorSet(error, "Can't append %s storage to an array with %s storage", + ArrowTypeString(src->storage_type), ArrowTypeString(dst_type)); + return EINVAL; + } + + if ((src->dictionary == NULL) != (dst->dictionary == NULL)) { + ArrowErrorSet(error, + "Can't append storage when exactly one of source and destination " + "is dictionary-encoded"); + return EINVAL; + } + + if (src->storage_type == NANOARROW_TYPE_NA) { + return NANOARROW_OK; + } + + if (src->n_children != dst->n_children) { + ArrowErrorSet(error, + "Expected source and destination to have the same number of " + "children but found %" PRId64 " and %" PRId64, + src->n_children, dst->n_children); + return EINVAL; + } + + if (src->storage_type == NANOARROW_TYPE_FIXED_SIZE_LIST && + src->layout.child_size_elements != private_data->layout.child_size_elements) { + ArrowErrorSet(error, + "Expected source and destination fixed-size list child sizes to " + "match but found %" PRId64 " and %" PRId64, + src->layout.child_size_elements, + private_data->layout.child_size_elements); + return EINVAL; + } + + if (src->storage_type == NANOARROW_TYPE_FIXED_SIZE_BINARY && + src->layout.element_size_bits[1] != private_data->layout.element_size_bits[1]) { + ArrowErrorSet(error, + "Expected source and destination fixed-size binary widths to " + "match but found %" PRId64 " and %" PRId64 " bits", + src->layout.element_size_bits[1], + private_data->layout.element_size_bits[1]); + return EINVAL; + } + + for (int64_t i = 0; i < src->n_children; i++) { + NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowArrayCheckCanAppendStorageFromArrayView( + dst->children[i], src->children[i], error), + error); + } + + return NANOARROW_OK; +} + +static ArrowErrorCode ArrowArrayAppendStorageFromArrayViewElement( + struct ArrowArray* dst, const struct ArrowArrayView* src, int64_t i, + struct ArrowError* error) { if (ArrowArrayViewIsNull(src, i)) { - return ArrowArrayAppendNull(dst, 1); + NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowArrayAppendNull(dst, 1), error); + return NANOARROW_OK; } switch (src->storage_type) { case NANOARROW_TYPE_NA: - return ArrowArrayAppendNull(dst, 1); + NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowArrayAppendNull(dst, 1), error); + return NANOARROW_OK; case NANOARROW_TYPE_BOOL: case NANOARROW_TYPE_INT8: case NANOARROW_TYPE_INT16: @@ -99,16 +254,22 @@ static ArrowErrorCode ArrowArrayAppendArrayViewElement(struct ArrowArray* dst, case NANOARROW_TYPE_TIME32: case NANOARROW_TYPE_TIME64: case NANOARROW_TYPE_DURATION: - return ArrowArrayAppendInt(dst, ArrowArrayViewGetIntUnsafe(src, i)); + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + ArrowArrayAppendInt(dst, ArrowArrayViewGetIntUnsafe(src, i)), error); + return NANOARROW_OK; case NANOARROW_TYPE_UINT8: case NANOARROW_TYPE_UINT16: case NANOARROW_TYPE_UINT32: case NANOARROW_TYPE_UINT64: - return ArrowArrayAppendUInt(dst, ArrowArrayViewGetUIntUnsafe(src, i)); + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + ArrowArrayAppendUInt(dst, ArrowArrayViewGetUIntUnsafe(src, i)), error); + return NANOARROW_OK; case NANOARROW_TYPE_HALF_FLOAT: case NANOARROW_TYPE_FLOAT: case NANOARROW_TYPE_DOUBLE: - return ArrowArrayAppendDouble(dst, ArrowArrayViewGetDoubleUnsafe(src, i)); + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + ArrowArrayAppendDouble(dst, ArrowArrayViewGetDoubleUnsafe(src, i)), error); + return NANOARROW_OK; case NANOARROW_TYPE_STRING: case NANOARROW_TYPE_BINARY: case NANOARROW_TYPE_FIXED_SIZE_BINARY: @@ -116,14 +277,17 @@ static ArrowErrorCode ArrowArrayAppendArrayViewElement(struct ArrowArray* dst, case NANOARROW_TYPE_LARGE_BINARY: case NANOARROW_TYPE_BINARY_VIEW: case NANOARROW_TYPE_STRING_VIEW: - return ArrowArrayAppendBytes(dst, ArrowArrayViewGetBytesUnsafe(src, i)); + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + ArrowArrayAppendBytes(dst, ArrowArrayViewGetBytesUnsafe(src, i)), error); + return NANOARROW_OK; case NANOARROW_TYPE_INTERVAL_MONTHS: case NANOARROW_TYPE_INTERVAL_DAY_TIME: case NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO: { struct ArrowInterval interval; ArrowIntervalInit(&interval, src->storage_type); ArrowArrayViewGetIntervalUnsafe(src, i, &interval); - return ArrowArrayAppendInterval(dst, &interval); + NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowArrayAppendInterval(dst, &interval), error); + return NANOARROW_OK; } case NANOARROW_TYPE_DECIMAL32: case NANOARROW_TYPE_DECIMAL64: @@ -141,14 +305,18 @@ static ArrowErrorCode ArrowArrayAppendArrayViewElement(struct ArrowArray* dst, struct ArrowDecimal decimal; ArrowDecimalInit(&decimal, bitwidth, /*precision=*/0, /*scale=*/0); ArrowArrayViewGetDecimalUnsafe(src, i, &decimal); - return ArrowArrayAppendDecimal(dst, &decimal); + NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowArrayAppendDecimal(dst, &decimal), error); + return NANOARROW_OK; } case NANOARROW_TYPE_STRUCT: for (int64_t child_i = 0; child_i < src->n_children; child_i++) { - NANOARROW_RETURN_NOT_OK(ArrowArrayAppendArrayViewElement( - dst->children[child_i], src->children[child_i], src->offset + i, error)); + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + ArrowArrayAppendStorageFromArrayViewElement( + dst->children[child_i], src->children[child_i], src->offset + i, error), + error); } - return ArrowArrayFinishElement(dst); + NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowArrayFinishElement(dst), error); + return NANOARROW_OK; case NANOARROW_TYPE_LIST: case NANOARROW_TYPE_LARGE_LIST: case NANOARROW_TYPE_MAP: @@ -166,36 +334,54 @@ static ArrowErrorCode ArrowArrayAppendArrayViewElement(struct ArrowArray* dst, } for (int64_t child_i = 0; child_i < child_length; child_i++) { - NANOARROW_RETURN_NOT_OK(ArrowArrayAppendArrayViewElement( - dst->children[0], src->children[0], child_offset + child_i, error)); + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + ArrowArrayAppendStorageFromArrayViewElement( + dst->children[0], src->children[0], child_offset + child_i, error), + error); } - return ArrowArrayFinishElement(dst); + NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowArrayFinishElement(dst), error); + return NANOARROW_OK; } case NANOARROW_TYPE_FIXED_SIZE_LIST: { int64_t child_offset = (src->offset + i) * src->layout.child_size_elements; for (int64_t child_i = 0; child_i < src->layout.child_size_elements; child_i++) { - NANOARROW_RETURN_NOT_OK(ArrowArrayAppendArrayViewElement( - dst->children[0], src->children[0], child_offset + child_i, error)); + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + ArrowArrayAppendStorageFromArrayViewElement( + dst->children[0], src->children[0], child_offset + child_i, error), + error); } - return ArrowArrayFinishElement(dst); + NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowArrayFinishElement(dst), error); + return NANOARROW_OK; } case NANOARROW_TYPE_DENSE_UNION: case NANOARROW_TYPE_SPARSE_UNION: { int8_t type_id = ArrowArrayViewUnionTypeId(src, i); int8_t child_index = ArrowArrayViewUnionChildIndex(src, i); int64_t child_offset = ArrowArrayViewUnionChildOffset(src, i); - NANOARROW_RETURN_NOT_OK(ArrowArrayAppendArrayViewElement( - dst->children[child_index], src->children[child_index], child_offset, error)); - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppend(ArrowArrayBuffer(dst, 0), &type_id, sizeof(type_id))); + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + ArrowArrayAppendStorageFromArrayViewElement(dst->children[child_index], + src->children[child_index], + child_offset, error), + error); + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + ArrowBufferAppend(ArrowArrayBuffer(dst, 0), &type_id, sizeof(type_id)), error); if (src->storage_type == NANOARROW_TYPE_DENSE_UNION) { - _NANOARROW_CHECK_RANGE(dst->children[child_index]->length - 1, 0, INT32_MAX); - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32( - ArrowArrayBuffer(dst, 1), (int32_t)dst->children[child_index]->length - 1)); + int64_t dst_child_offset = dst->children[child_index]->length - 1; + if (dst_child_offset < 0 || dst_child_offset > INT32_MAX) { + ArrowErrorSet(error, + "Expected dense union child offset to fit in int32 but " + "found %" PRId64, + dst_child_offset); + return EOVERFLOW; + } + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + ArrowBufferAppendInt32(ArrowArrayBuffer(dst, 1), (int32_t)dst_child_offset), + error); } else { for (int64_t child_i = 0; child_i < dst->n_children; child_i++) { if (child_i != child_index && dst->children[child_i]->length == dst->length) { - NANOARROW_RETURN_NOT_OK(ArrowArrayAppendEmpty(dst->children[child_i], 1)); + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + ArrowArrayAppendEmpty(dst->children[child_i], 1), error); } if (dst->children[child_i]->length != dst->length + 1) { ArrowErrorSet(error, @@ -218,29 +404,77 @@ static ArrowErrorCode ArrowArrayAppendArrayViewElement(struct ArrowArray* dst, } } -ArrowErrorCode ArrowArrayAppendArrayView(struct ArrowArray* dst, - const struct ArrowArrayView* src, - struct ArrowError* error) { +static int64_t ArrowArrayViewResolveRun(const struct ArrowArrayView* run_ends, + int64_t logical_offset) { + if (run_ends->length <= 1) { + return 0; + } + + switch (run_ends->storage_type) { + case NANOARROW_TYPE_INT32: + return ArrowResolveChunk32( + (int32_t)(logical_offset + 1), + run_ends->buffer_views[1].data.as_int32 + run_ends->offset, 0, + (int32_t)run_ends->length); + case NANOARROW_TYPE_INT64: + return ArrowResolveChunk64( + logical_offset + 1, run_ends->buffer_views[1].data.as_int64 + run_ends->offset, + 0, run_ends->length); + case NANOARROW_TYPE_INT16: + default: { + int64_t run = 0; + while (run + 1 < run_ends->length && + ArrowArrayViewGetIntUnsafe(run_ends, run) <= logical_offset) { + run++; + } + return run; + } + } +} + +ArrowErrorCode ArrowArrayAppendStorageFromArrayView(struct ArrowArray* dst, + const struct ArrowArrayView* src, + struct ArrowError* error) { + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + ArrowArrayCheckCanAppendStorageFromArrayView(dst, src, error), error); + if (src->storage_type == NANOARROW_TYPE_RUN_END_ENCODED) { - if (src->offset != 0) { - ArrowErrorSet(error, "Can't append a sliced run-end encoded array view"); - return ENOTSUP; + if (src->length == 0) { + return NANOARROW_OK; + } + + const struct ArrowArrayView* run_ends = src->children[0]; + if (run_ends->length == 0) { + ArrowErrorSet(error, + "Expected a non-empty run ends array for a non-empty " + "run-end encoded array"); + return EINVAL; } + int64_t first_run = ArrowArrayViewResolveRun(run_ends, src->offset); + int64_t slice_end = src->offset + src->length; int64_t run_end_offset = dst->length; - for (int64_t i = 0; i < src->children[0]->length; i++) { - NANOARROW_RETURN_NOT_OK(ArrowArrayAppendInt( - dst->children[0], - run_end_offset + ArrowArrayViewGetIntUnsafe(src->children[0], i))); - NANOARROW_RETURN_NOT_OK( - ArrowArrayAppendArrayViewElement(dst->children[1], src->children[1], i, error)); + for (int64_t i = first_run; i < run_ends->length; i++) { + int64_t src_run_end = ArrowArrayViewGetIntUnsafe(run_ends, i); + int64_t clipped_run_end = src_run_end < slice_end ? src_run_end : slice_end; + int64_t dst_run_end = run_end_offset + clipped_run_end - src->offset; + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + ArrowArrayAppendInt(dst->children[0], dst_run_end), error); + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + ArrowArrayAppendStorageFromArrayViewElement(dst->children[1], src->children[1], + i, error), + error); + if (src_run_end >= slice_end) { + break; + } } dst->length += src->length; return NANOARROW_OK; } for (int64_t i = 0; i < src->length; i++) { - NANOARROW_RETURN_NOT_OK(ArrowArrayAppendArrayViewElement(dst, src, i, error)); + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + ArrowArrayAppendStorageFromArrayViewElement(dst, src, i, error), error); } return NANOARROW_OK; } diff --git a/src/nanoarrow/common/array_test.cc b/src/nanoarrow/common/array_test.cc index 2757e3241..b8337e1a9 100644 --- a/src/nanoarrow/common/array_test.cc +++ b/src/nanoarrow/common/array_test.cc @@ -5093,7 +5093,7 @@ static ArrowErrorCode AppendArrayViewForTest(const struct ArrowArrayView* src, struct ArrowError* error) { NANOARROW_RETURN_NOT_OK(ArrowArrayInitFromArrayView(dst, src, error)); NANOARROW_RETURN_NOT_OK(ArrowArrayStartAppending(dst)); - NANOARROW_RETURN_NOT_OK(ArrowArrayAppendArrayView(dst, src, error)); + NANOARROW_RETURN_NOT_OK(ArrowArrayAppendStorageFromArrayView(dst, src, error)); return ArrowArrayFinishBuildingDefault(dst, error); } @@ -5294,8 +5294,8 @@ TEST(ArrayTest, ArrayAppendArrayViewRunEndEncoded) { struct ArrowArray dst; ASSERT_EQ(ArrowArrayInitFromArrayView(&dst, &src_view, &error), NANOARROW_OK); ASSERT_EQ(ArrowArrayStartAppending(&dst), NANOARROW_OK); - ASSERT_EQ(ArrowArrayAppendArrayView(&dst, &src_view, &error), NANOARROW_OK); - ASSERT_EQ(ArrowArrayAppendArrayView(&dst, &src_view, &error), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendStorageFromArrayView(&dst, &src_view, &error), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendStorageFromArrayView(&dst, &src_view, &error), NANOARROW_OK); ASSERT_EQ(ArrowArrayFinishBuildingDefault(&dst, &error), NANOARROW_OK) << error.message; EXPECT_EQ(dst.length, 6); @@ -5307,8 +5307,11 @@ TEST(ArrayTest, ArrayAppendArrayViewRunEndEncoded) { src_view.offset = 1; src_view.length = 1; - EXPECT_EQ(ArrowArrayAppendArrayView(&dst, &src_view, &error), ENOTSUP); - EXPECT_STREQ(error.message, "Can't append a sliced run-end encoded array view"); + ASSERT_EQ(ArrowArrayAppendStorageFromArrayView(&dst, &src_view, &error), NANOARROW_OK) + << error.message; + EXPECT_EQ(dst.length, 7); + EXPECT_EQ(dst.children[0]->length, 5); + EXPECT_EQ(dst.children[1]->length, 5); ArrowArrayViewReset(&dst_view); ArrowArrayRelease(&dst); diff --git a/src/nanoarrow/nanoarrow.h b/src/nanoarrow/nanoarrow.h index 4e3c7e63d..c165317b3 100644 --- a/src/nanoarrow/nanoarrow.h +++ b/src/nanoarrow/nanoarrow.h @@ -122,8 +122,8 @@ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArraySetValidityBitmap) #define ArrowArraySetBuffer NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArraySetBuffer) #define ArrowArrayReserve NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayReserve) -#define ArrowArrayAppendArrayView \ - NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayAppendArrayView) +#define ArrowArrayAppendStorageFromArrayView \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayAppendStorageFromArrayView) #define ArrowArrayFinishBuilding \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayFinishBuilding) #define ArrowArrayFinishBuildingDefault \ @@ -1034,13 +1034,14 @@ static inline ArrowErrorCode ArrowArrayStartAppending(struct ArrowArray* array); NANOARROW_DLL ArrowErrorCode ArrowArrayReserve(struct ArrowArray* array, int64_t additional_size_elements); -/// \brief Append the contents of an ArrowArrayView to an ArrowArray +/// \brief Append storage from an ArrowArrayView to an ArrowArray /// -/// Appends each logical element of array_view to array. array must have been -/// initialized with the same storage type and layout as array_view and prepared -/// using ArrowArrayStartAppending(). Dictionary values referenced by array_view -/// are not copied. Returns ENOTSUP for unsupported storage types. -NANOARROW_DLL ArrowErrorCode ArrowArrayAppendArrayView( +/// Appends each logical storage element of array_view to array. array must have +/// been initialized with compatible storage and prepared using +/// ArrowArrayStartAppending(). Dictionary values referenced by array_view are +/// not copied; dictionary-encoded inputs require a dictionary-encoded output. +/// Returns EINVAL for incompatible storage and ENOTSUP for unsupported storage. +NANOARROW_DLL ArrowErrorCode ArrowArrayAppendStorageFromArrayView( struct ArrowArray* array, const struct ArrowArrayView* array_view, struct ArrowError* error); From 4974cd7d3e9545cf95043712dda3a38fc9d8e137 Mon Sep 17 00:00:00 2001 From: Rusty Conover Date: Thu, 3 Sep 2026 09:02:38 -0400 Subject: [PATCH 3/7] perf: bulk append matching fixed-width storage --- src/nanoarrow/common/array.c | 101 +++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/src/nanoarrow/common/array.c b/src/nanoarrow/common/array.c index 385eea048..f7a300667 100644 --- a/src/nanoarrow/common/array.c +++ b/src/nanoarrow/common/array.c @@ -169,6 +169,14 @@ static int ArrowArrayCanAppendStorageType(enum ArrowType dst_type, static ArrowErrorCode ArrowArrayCheckCanAppendStorageFromArrayView( struct ArrowArray* dst, const struct ArrowArrayView* src, struct ArrowError* error) { + if (src->offset < 0 || src->length < 0 || src->offset > INT64_MAX - src->length) { + ArrowErrorSet(error, + "Expected source offset and length to describe a valid int64 " + "range but found %" PRId64 " and %" PRId64, + src->offset, src->length); + return EINVAL; + } + if (!ArrowArrayIsInternal(dst)) { ArrowErrorSet(error, "Expected destination to be an internal ArrowArray"); return EINVAL; @@ -404,6 +412,95 @@ static ArrowErrorCode ArrowArrayAppendStorageFromArrayViewElement( } } +static int ArrowArrayCanAppendFixedWidthStorage(struct ArrowArray* dst, + const struct ArrowArrayView* src) { + struct ArrowArrayPrivateData* private_data = + (struct ArrowArrayPrivateData*)dst->private_data; + return private_data->storage_type == src->storage_type && src->n_children == 0 && + src->layout.buffer_type[1] == NANOARROW_BUFFER_TYPE_DATA && + src->layout.element_size_bits[1] > 0 && + src->layout.element_size_bits[1] % 8 == 0; +} + +static ArrowErrorCode ArrowArrayAppendValidityFromArrayView( + struct ArrowArray* dst, const struct ArrowArrayView* src, struct ArrowError* error) { + struct ArrowBitmap* dst_validity = ArrowArrayValidityBitmap(dst); + const uint8_t* src_validity = src->buffer_views[0].data.as_uint8; + if (src_validity == NULL && dst_validity->buffer.data == NULL) { + return NANOARROW_OK; + } + + if (dst_validity->buffer.data == NULL) { + NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowBitmapAppend(dst_validity, 1, dst->length), + error); + } + + if (src_validity == NULL) { + NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowBitmapAppend(dst_validity, 1, src->length), + error); + return NANOARROW_OK; + } + + NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowBitmapReserve(dst_validity, src->length), + error); + int8_t validity[1024]; + for (int64_t offset = 0; offset < src->length;) { + int64_t remaining = src->length - offset; + int64_t chunk_size = + remaining < (int64_t)sizeof(validity) ? remaining : (int64_t)sizeof(validity); + ArrowBitsUnpackInt8(src_validity, src->offset + offset, chunk_size, validity); + ArrowBitmapAppendInt8Unsafe(dst_validity, validity, chunk_size); + offset += chunk_size; + } + + return NANOARROW_OK; +} + +static ArrowErrorCode ArrowArrayAppendFixedWidthStorageFromArrayView( + struct ArrowArray* dst, const struct ArrowArrayView* src, struct ArrowError* error) { + if (src->length == 0) { + return NANOARROW_OK; + } + + int64_t element_size_bytes = src->layout.element_size_bits[1] / 8; + if (src->offset > INT64_MAX / element_size_bytes || + src->length > INT64_MAX / element_size_bytes) { + ArrowErrorSet(error, + "Expected fixed-width append size to fit in int64 but found " + "element size %" PRId64 ", offset %" PRId64 ", and length %" PRId64, + element_size_bytes, src->offset, src->length); + return EOVERFLOW; + } + + int64_t src_offset_bytes = src->offset * element_size_bytes; + int64_t src_size_bytes = src->length * element_size_bytes; + if (src->buffer_views[1].data.as_uint8 == NULL || + src_offset_bytes > src->buffer_views[1].size_bytes || + src_size_bytes > src->buffer_views[1].size_bytes - src_offset_bytes) { + ArrowErrorSet(error, + "Expected fixed-width source buffer to contain %" PRId64 + " bytes at offset %" PRId64 " but its size is %" PRId64, + src_size_bytes, src_offset_bytes, src->buffer_views[1].size_bytes); + return EINVAL; + } + + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + ArrowArrayAppendValidityFromArrayView(dst, src, error), error); + + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + ArrowBufferAppend(ArrowArrayBuffer(dst, 1), + src->buffer_views[1].data.as_uint8 + src_offset_bytes, + src_size_bytes), + error); + + if (src->buffer_views[0].data.as_uint8 != NULL) { + dst->null_count += src->length - ArrowBitCountSet(src->buffer_views[0].data.as_uint8, + src->offset, src->length); + } + dst->length += src->length; + return NANOARROW_OK; +} + static int64_t ArrowArrayViewResolveRun(const struct ArrowArrayView* run_ends, int64_t logical_offset) { if (run_ends->length <= 1) { @@ -438,6 +535,10 @@ ArrowErrorCode ArrowArrayAppendStorageFromArrayView(struct ArrowArray* dst, NANOARROW_RETURN_NOT_OK_WITH_ERROR( ArrowArrayCheckCanAppendStorageFromArrayView(dst, src, error), error); + if (ArrowArrayCanAppendFixedWidthStorage(dst, src)) { + return ArrowArrayAppendFixedWidthStorageFromArrayView(dst, src, error); + } + if (src->storage_type == NANOARROW_TYPE_RUN_END_ENCODED) { if (src->length == 0) { return NANOARROW_OK; From e56f47b6392784c36d96e56d1d25043c2adfc687 Mon Sep 17 00:00:00 2001 From: Rusty Conover Date: Thu, 3 Sep 2026 09:10:24 -0400 Subject: [PATCH 4/7] test: expand array view appender coverage Add Arrow C++ conversion matrices plus sliced validity, overflow, dictionary, and sliced run-end tests. Preserve detailed errors produced by nested append helpers. --- src/nanoarrow/common/array.c | 46 ++-- src/nanoarrow/common/array_test.cc | 399 +++++++++++++++++++++++++++-- 2 files changed, 397 insertions(+), 48 deletions(-) diff --git a/src/nanoarrow/common/array.c b/src/nanoarrow/common/array.c index f7a300667..d57b607d1 100644 --- a/src/nanoarrow/common/array.c +++ b/src/nanoarrow/common/array.c @@ -231,9 +231,8 @@ static ArrowErrorCode ArrowArrayCheckCanAppendStorageFromArrayView( } for (int64_t i = 0; i < src->n_children; i++) { - NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowArrayCheckCanAppendStorageFromArrayView( - dst->children[i], src->children[i], error), - error); + NANOARROW_RETURN_NOT_OK(ArrowArrayCheckCanAppendStorageFromArrayView( + dst->children[i], src->children[i], error)); } return NANOARROW_OK; @@ -318,10 +317,8 @@ static ArrowErrorCode ArrowArrayAppendStorageFromArrayViewElement( } case NANOARROW_TYPE_STRUCT: for (int64_t child_i = 0; child_i < src->n_children; child_i++) { - NANOARROW_RETURN_NOT_OK_WITH_ERROR( - ArrowArrayAppendStorageFromArrayViewElement( - dst->children[child_i], src->children[child_i], src->offset + i, error), - error); + NANOARROW_RETURN_NOT_OK(ArrowArrayAppendStorageFromArrayViewElement( + dst->children[child_i], src->children[child_i], src->offset + i, error)); } NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowArrayFinishElement(dst), error); return NANOARROW_OK; @@ -342,10 +339,8 @@ static ArrowErrorCode ArrowArrayAppendStorageFromArrayViewElement( } for (int64_t child_i = 0; child_i < child_length; child_i++) { - NANOARROW_RETURN_NOT_OK_WITH_ERROR( - ArrowArrayAppendStorageFromArrayViewElement( - dst->children[0], src->children[0], child_offset + child_i, error), - error); + NANOARROW_RETURN_NOT_OK(ArrowArrayAppendStorageFromArrayViewElement( + dst->children[0], src->children[0], child_offset + child_i, error)); } NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowArrayFinishElement(dst), error); return NANOARROW_OK; @@ -353,10 +348,8 @@ static ArrowErrorCode ArrowArrayAppendStorageFromArrayViewElement( case NANOARROW_TYPE_FIXED_SIZE_LIST: { int64_t child_offset = (src->offset + i) * src->layout.child_size_elements; for (int64_t child_i = 0; child_i < src->layout.child_size_elements; child_i++) { - NANOARROW_RETURN_NOT_OK_WITH_ERROR( - ArrowArrayAppendStorageFromArrayViewElement( - dst->children[0], src->children[0], child_offset + child_i, error), - error); + NANOARROW_RETURN_NOT_OK(ArrowArrayAppendStorageFromArrayViewElement( + dst->children[0], src->children[0], child_offset + child_i, error)); } NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowArrayFinishElement(dst), error); return NANOARROW_OK; @@ -366,11 +359,8 @@ static ArrowErrorCode ArrowArrayAppendStorageFromArrayViewElement( int8_t type_id = ArrowArrayViewUnionTypeId(src, i); int8_t child_index = ArrowArrayViewUnionChildIndex(src, i); int64_t child_offset = ArrowArrayViewUnionChildOffset(src, i); - NANOARROW_RETURN_NOT_OK_WITH_ERROR( - ArrowArrayAppendStorageFromArrayViewElement(dst->children[child_index], - src->children[child_index], - child_offset, error), - error); + NANOARROW_RETURN_NOT_OK(ArrowArrayAppendStorageFromArrayViewElement( + dst->children[child_index], src->children[child_index], child_offset, error)); NANOARROW_RETURN_NOT_OK_WITH_ERROR( ArrowBufferAppend(ArrowArrayBuffer(dst, 0), &type_id, sizeof(type_id)), error); if (src->storage_type == NANOARROW_TYPE_DENSE_UNION) { @@ -484,8 +474,7 @@ static ArrowErrorCode ArrowArrayAppendFixedWidthStorageFromArrayView( return EINVAL; } - NANOARROW_RETURN_NOT_OK_WITH_ERROR( - ArrowArrayAppendValidityFromArrayView(dst, src, error), error); + NANOARROW_RETURN_NOT_OK(ArrowArrayAppendValidityFromArrayView(dst, src, error)); NANOARROW_RETURN_NOT_OK_WITH_ERROR( ArrowBufferAppend(ArrowArrayBuffer(dst, 1), @@ -532,8 +521,7 @@ static int64_t ArrowArrayViewResolveRun(const struct ArrowArrayView* run_ends, ArrowErrorCode ArrowArrayAppendStorageFromArrayView(struct ArrowArray* dst, const struct ArrowArrayView* src, struct ArrowError* error) { - NANOARROW_RETURN_NOT_OK_WITH_ERROR( - ArrowArrayCheckCanAppendStorageFromArrayView(dst, src, error), error); + NANOARROW_RETURN_NOT_OK(ArrowArrayCheckCanAppendStorageFromArrayView(dst, src, error)); if (ArrowArrayCanAppendFixedWidthStorage(dst, src)) { return ArrowArrayAppendFixedWidthStorageFromArrayView(dst, src, error); @@ -561,10 +549,8 @@ ArrowErrorCode ArrowArrayAppendStorageFromArrayView(struct ArrowArray* dst, int64_t dst_run_end = run_end_offset + clipped_run_end - src->offset; NANOARROW_RETURN_NOT_OK_WITH_ERROR( ArrowArrayAppendInt(dst->children[0], dst_run_end), error); - NANOARROW_RETURN_NOT_OK_WITH_ERROR( - ArrowArrayAppendStorageFromArrayViewElement(dst->children[1], src->children[1], - i, error), - error); + NANOARROW_RETURN_NOT_OK(ArrowArrayAppendStorageFromArrayViewElement( + dst->children[1], src->children[1], i, error)); if (src_run_end >= slice_end) { break; } @@ -574,8 +560,8 @@ ArrowErrorCode ArrowArrayAppendStorageFromArrayView(struct ArrowArray* dst, } for (int64_t i = 0; i < src->length; i++) { - NANOARROW_RETURN_NOT_OK_WITH_ERROR( - ArrowArrayAppendStorageFromArrayViewElement(dst, src, i, error), error); + NANOARROW_RETURN_NOT_OK( + ArrowArrayAppendStorageFromArrayViewElement(dst, src, i, error)); } return NANOARROW_OK; } diff --git a/src/nanoarrow/common/array_test.cc b/src/nanoarrow/common/array_test.cc index b8337e1a9..29b67891b 100644 --- a/src/nanoarrow/common/array_test.cc +++ b/src/nanoarrow/common/array_test.cc @@ -33,6 +33,7 @@ #include #include #include +#include #include #if defined(NANOARROW_BUILD_TESTS_WITH_ARROW) && defined(ARROW_VERSION_MAJOR) && \ @@ -5088,17 +5089,17 @@ TEST(ArrayMoveSharedTest, ArrayWithNullBuffers) { ArrowArrayRelease(&shared); } -static ArrowErrorCode AppendArrayViewForTest(const struct ArrowArrayView* src, - struct ArrowArray* dst, - struct ArrowError* error) { +static ArrowErrorCode AppendStorageFromArrayViewForTest(const struct ArrowArrayView* src, + struct ArrowArray* dst, + struct ArrowError* error) { NANOARROW_RETURN_NOT_OK(ArrowArrayInitFromArrayView(dst, src, error)); NANOARROW_RETURN_NOT_OK(ArrowArrayStartAppending(dst)); NANOARROW_RETURN_NOT_OK(ArrowArrayAppendStorageFromArrayView(dst, src, error)); return ArrowArrayFinishBuildingDefault(dst, error); } -static void ExpectPrimitiveArrayViewAppendIdentical(struct ArrowArray* src, - enum ArrowType type) { +static void ExpectPrimitiveStorageAppendIdentical(struct ArrowArray* src, + enum ArrowType type) { struct ArrowError error; struct ArrowArrayView src_view; ArrowArrayViewInitFromType(&src_view, type); @@ -5106,7 +5107,7 @@ static void ExpectPrimitiveArrayViewAppendIdentical(struct ArrowArray* src, << error.message; struct ArrowArray dst; - ASSERT_EQ(AppendArrayViewForTest(&src_view, &dst, &error), NANOARROW_OK) + ASSERT_EQ(AppendStorageFromArrayViewForTest(&src_view, &dst, &error), NANOARROW_OK) << error.message; struct ArrowArrayView dst_view; ArrowArrayViewInitFromType(&dst_view, type); @@ -5124,7 +5125,7 @@ static void ExpectPrimitiveArrayViewAppendIdentical(struct ArrowArray* src, ArrowArrayViewReset(&src_view); } -TEST(ArrayTest, ArrayAppendArrayViewPrimitiveTypes) { +TEST(ArrayTest, ArrayAppendStorageFromArrayViewPrimitiveTypes) { struct ArrowArray array; ASSERT_EQ(ArrowArrayInitFromType(&array, NANOARROW_TYPE_INT64), NANOARROW_OK); @@ -5132,28 +5133,28 @@ TEST(ArrayTest, ArrayAppendArrayViewPrimitiveTypes) { ASSERT_EQ(ArrowArrayAppendInt(&array, -42), NANOARROW_OK); ASSERT_EQ(ArrowArrayAppendNull(&array, 1), NANOARROW_OK); ASSERT_EQ(ArrowArrayFinishBuildingDefault(&array, nullptr), NANOARROW_OK); - ExpectPrimitiveArrayViewAppendIdentical(&array, NANOARROW_TYPE_INT64); + ExpectPrimitiveStorageAppendIdentical(&array, NANOARROW_TYPE_INT64); ArrowArrayRelease(&array); ASSERT_EQ(ArrowArrayInitFromType(&array, NANOARROW_TYPE_UINT64), NANOARROW_OK); ASSERT_EQ(ArrowArrayStartAppending(&array), NANOARROW_OK); ASSERT_EQ(ArrowArrayAppendUInt(&array, UINT64_MAX), NANOARROW_OK); ASSERT_EQ(ArrowArrayFinishBuildingDefault(&array, nullptr), NANOARROW_OK); - ExpectPrimitiveArrayViewAppendIdentical(&array, NANOARROW_TYPE_UINT64); + ExpectPrimitiveStorageAppendIdentical(&array, NANOARROW_TYPE_UINT64); ArrowArrayRelease(&array); ASSERT_EQ(ArrowArrayInitFromType(&array, NANOARROW_TYPE_DOUBLE), NANOARROW_OK); ASSERT_EQ(ArrowArrayStartAppending(&array), NANOARROW_OK); ASSERT_EQ(ArrowArrayAppendDouble(&array, 1.25), NANOARROW_OK); ASSERT_EQ(ArrowArrayFinishBuildingDefault(&array, nullptr), NANOARROW_OK); - ExpectPrimitiveArrayViewAppendIdentical(&array, NANOARROW_TYPE_DOUBLE); + ExpectPrimitiveStorageAppendIdentical(&array, NANOARROW_TYPE_DOUBLE); ArrowArrayRelease(&array); ASSERT_EQ(ArrowArrayInitFromType(&array, NANOARROW_TYPE_STRING), NANOARROW_OK); ASSERT_EQ(ArrowArrayStartAppending(&array), NANOARROW_OK); ASSERT_EQ(ArrowArrayAppendString(&array, "some value"_asv), NANOARROW_OK); ASSERT_EQ(ArrowArrayFinishBuildingDefault(&array, nullptr), NANOARROW_OK); - ExpectPrimitiveArrayViewAppendIdentical(&array, NANOARROW_TYPE_STRING); + ExpectPrimitiveStorageAppendIdentical(&array, NANOARROW_TYPE_STRING); ArrowArrayRelease(&array); ASSERT_EQ(ArrowArrayInitFromType(&array, NANOARROW_TYPE_INTERVAL_MONTHS), NANOARROW_OK); @@ -5163,7 +5164,7 @@ TEST(ArrayTest, ArrayAppendArrayViewPrimitiveTypes) { interval.months = 42; ASSERT_EQ(ArrowArrayAppendInterval(&array, &interval), NANOARROW_OK); ASSERT_EQ(ArrowArrayFinishBuildingDefault(&array, nullptr), NANOARROW_OK); - ExpectPrimitiveArrayViewAppendIdentical(&array, NANOARROW_TYPE_INTERVAL_MONTHS); + ExpectPrimitiveStorageAppendIdentical(&array, NANOARROW_TYPE_INTERVAL_MONTHS); ArrowArrayRelease(&array); ASSERT_EQ(ArrowArrayInitFromType(&array, NANOARROW_TYPE_DECIMAL128), NANOARROW_OK); @@ -5173,11 +5174,11 @@ TEST(ArrayTest, ArrayAppendArrayViewPrimitiveTypes) { ArrowDecimalSetInt(&decimal, 1234); ASSERT_EQ(ArrowArrayAppendDecimal(&array, &decimal), NANOARROW_OK); ASSERT_EQ(ArrowArrayFinishBuildingDefault(&array, nullptr), NANOARROW_OK); - ExpectPrimitiveArrayViewAppendIdentical(&array, NANOARROW_TYPE_DECIMAL128); + ExpectPrimitiveStorageAppendIdentical(&array, NANOARROW_TYPE_DECIMAL128); ArrowArrayRelease(&array); } -TEST(ArrayTest, ArrayAppendArrayViewNestedAndSliced) { +TEST(ArrayTest, ArrayAppendStorageFromArrayViewNestedAndSliced) { struct ArrowError error; struct ArrowSchema schema; ArrowSchemaInit(&schema); @@ -5209,7 +5210,7 @@ TEST(ArrayTest, ArrayAppendArrayViewNestedAndSliced) { src_view.null_count = 1; struct ArrowArray dst; - ASSERT_EQ(AppendArrayViewForTest(&src_view, &dst, &error), NANOARROW_OK) + ASSERT_EQ(AppendStorageFromArrayViewForTest(&src_view, &dst, &error), NANOARROW_OK) << error.message; EXPECT_EQ(dst.length, 2); EXPECT_EQ(dst.children[0]->length, 2); @@ -5229,7 +5230,7 @@ TEST(ArrayTest, ArrayAppendArrayViewNestedAndSliced) { ArrowSchemaRelease(&schema); } -TEST(ArrayTest, ArrayAppendArrayViewUnions) { +TEST(ArrayTest, ArrayAppendStorageFromArrayViewUnions) { for (enum ArrowType union_type : {NANOARROW_TYPE_DENSE_UNION, NANOARROW_TYPE_SPARSE_UNION}) { struct ArrowError error; @@ -5253,7 +5254,7 @@ TEST(ArrayTest, ArrayAppendArrayViewUnions) { ASSERT_EQ(ArrowArrayViewInitFromSchema(&src_view, &schema, &error), NANOARROW_OK); ASSERT_EQ(ArrowArrayViewSetArray(&src_view, &src, &error), NANOARROW_OK); struct ArrowArray dst; - ASSERT_EQ(AppendArrayViewForTest(&src_view, &dst, &error), NANOARROW_OK) + ASSERT_EQ(AppendStorageFromArrayViewForTest(&src_view, &dst, &error), NANOARROW_OK) << error.message; EXPECT_EQ(dst.length, 2); @@ -5271,7 +5272,7 @@ TEST(ArrayTest, ArrayAppendArrayViewUnions) { } } -TEST(ArrayTest, ArrayAppendArrayViewRunEndEncoded) { +TEST(ArrayTest, ArrayAppendStorageFromArrayViewRunEndEncoded) { struct ArrowError error; struct ArrowSchema schema; ArrowSchemaInit(&schema); @@ -5304,14 +5305,20 @@ TEST(ArrayTest, ArrayAppendArrayViewRunEndEncoded) { ASSERT_EQ(ArrowArrayViewSetArray(&dst_view, &dst, &error), NANOARROW_OK); EXPECT_EQ(ArrowArrayViewGetIntUnsafe(dst_view.children[0], 2), 5); EXPECT_EQ(ArrowArrayViewGetIntUnsafe(dst_view.children[0], 3), 6); + ArrowArrayViewReset(&dst_view); src_view.offset = 1; src_view.length = 1; ASSERT_EQ(ArrowArrayAppendStorageFromArrayView(&dst, &src_view, &error), NANOARROW_OK) << error.message; + ASSERT_EQ(ArrowArrayFinishBuildingDefault(&dst, &error), NANOARROW_OK) << error.message; EXPECT_EQ(dst.length, 7); EXPECT_EQ(dst.children[0]->length, 5); EXPECT_EQ(dst.children[1]->length, 5); + ASSERT_EQ(ArrowArrayViewInitFromSchema(&dst_view, &schema, &error), NANOARROW_OK); + ASSERT_EQ(ArrowArrayViewSetArray(&dst_view, &dst, &error), NANOARROW_OK); + EXPECT_EQ(ArrowArrayViewGetIntUnsafe(dst_view.children[0], 4), 7); + EXPECT_EQ(ArrowArrayViewGetStringUnsafe(dst_view.children[1], 4), "a"_asv); ArrowArrayViewReset(&dst_view); ArrowArrayRelease(&dst); @@ -5319,3 +5326,359 @@ TEST(ArrayTest, ArrayAppendArrayViewRunEndEncoded) { ArrowArrayRelease(&src); ArrowSchemaRelease(&schema); } + +TEST(ArrayTest, ArrayAppendStorageFromArrayViewFixedWidthSlicedValidity) { + struct ArrowError error; + struct ArrowArray src; + ASSERT_EQ(ArrowArrayInitFromType(&src, NANOARROW_TYPE_INT32), NANOARROW_OK); + ASSERT_EQ(ArrowArrayStartAppending(&src), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendNull(&src, 1), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendInt(&src, 1), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendNull(&src, 1), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendInt(&src, 3), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendNull(&src, 1), NANOARROW_OK); + ASSERT_EQ(ArrowArrayFinishBuildingDefault(&src, &error), NANOARROW_OK); + + struct ArrowArrayView src_view; + ArrowArrayViewInitFromType(&src_view, NANOARROW_TYPE_INT32); + ASSERT_EQ(ArrowArrayViewSetArray(&src_view, &src, &error), NANOARROW_OK); + src_view.offset = 1; + src_view.length = 3; + src_view.null_count = 1; + + struct ArrowArray dst; + ASSERT_EQ(ArrowArrayInitFromType(&dst, NANOARROW_TYPE_INT32), NANOARROW_OK); + ASSERT_EQ(ArrowArrayStartAppending(&dst), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendInt(&dst, 10), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendStorageFromArrayView(&dst, &src_view, &error), NANOARROW_OK) + << error.message; + + struct ArrowArray all_valid; + ASSERT_EQ(ArrowArrayInitFromType(&all_valid, NANOARROW_TYPE_INT32), NANOARROW_OK); + ASSERT_EQ(ArrowArrayStartAppending(&all_valid), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendInt(&all_valid, 4), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendInt(&all_valid, 5), NANOARROW_OK); + ASSERT_EQ(ArrowArrayFinishBuildingDefault(&all_valid, &error), NANOARROW_OK); + struct ArrowArrayView all_valid_view; + ArrowArrayViewInitFromType(&all_valid_view, NANOARROW_TYPE_INT32); + ASSERT_EQ(ArrowArrayViewSetArray(&all_valid_view, &all_valid, &error), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendStorageFromArrayView(&dst, &all_valid_view, &error), + NANOARROW_OK) + << error.message; + ASSERT_EQ(ArrowArrayAppendStorageFromArrayView(&dst, &src_view, &error), NANOARROW_OK) + << error.message; + ASSERT_EQ(ArrowArrayFinishBuildingDefault(&dst, &error), NANOARROW_OK) << error.message; + + struct ArrowArrayView dst_view; + ArrowArrayViewInitFromType(&dst_view, NANOARROW_TYPE_INT32); + ASSERT_EQ(ArrowArrayViewSetArray(&dst_view, &dst, &error), NANOARROW_OK); + EXPECT_EQ(dst.length, 9); + EXPECT_EQ(dst.null_count, 2); + EXPECT_EQ(ArrowArrayViewGetIntUnsafe(&dst_view, 0), 10); + EXPECT_EQ(ArrowArrayViewGetIntUnsafe(&dst_view, 1), 1); + EXPECT_TRUE(ArrowArrayViewIsNull(&dst_view, 2)); + EXPECT_EQ(ArrowArrayViewGetIntUnsafe(&dst_view, 3), 3); + EXPECT_EQ(ArrowArrayViewGetIntUnsafe(&dst_view, 4), 4); + EXPECT_EQ(ArrowArrayViewGetIntUnsafe(&dst_view, 5), 5); + EXPECT_TRUE(ArrowArrayViewIsNull(&dst_view, 7)); + + struct ArrowArray dst_all_valid; + ASSERT_EQ(ArrowArrayInitFromType(&dst_all_valid, NANOARROW_TYPE_INT32), NANOARROW_OK); + ASSERT_EQ(ArrowArrayStartAppending(&dst_all_valid), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendStorageFromArrayView(&dst_all_valid, &all_valid_view, &error), + NANOARROW_OK); + EXPECT_EQ(dst_all_valid.buffers[0], nullptr); + + ArrowArrayRelease(&dst_all_valid); + ArrowArrayViewReset(&dst_view); + ArrowArrayViewReset(&all_valid_view); + ArrowArrayRelease(&all_valid); + ArrowArrayRelease(&dst); + ArrowArrayViewReset(&src_view); + ArrowArrayRelease(&src); +} + +TEST(ArrayTest, ArrayAppendStorageFromArrayViewErrors) { + struct ArrowError error; + struct ArrowArray src; + ASSERT_EQ(ArrowArrayInitFromType(&src, NANOARROW_TYPE_INT64), NANOARROW_OK); + ASSERT_EQ(ArrowArrayStartAppending(&src), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendInt(&src, INT8_MAX + 1), NANOARROW_OK); + ASSERT_EQ(ArrowArrayFinishBuildingDefault(&src, &error), NANOARROW_OK); + struct ArrowArrayView src_view; + ArrowArrayViewInitFromType(&src_view, NANOARROW_TYPE_INT64); + ASSERT_EQ(ArrowArrayViewSetArray(&src_view, &src, &error), NANOARROW_OK); + + struct ArrowArray dst; + ASSERT_EQ(ArrowArrayInitFromType(&dst, NANOARROW_TYPE_INT8), NANOARROW_OK); + ASSERT_EQ(ArrowArrayStartAppending(&dst), NANOARROW_OK); + EXPECT_EQ(ArrowArrayAppendStorageFromArrayView(&dst, &src_view, &error), EINVAL); + EXPECT_NE(error.message[0], '\0'); + EXPECT_EQ(dst.length, 0); + + ArrowArrayRelease(&dst); + ArrowArrayViewReset(&src_view); + ArrowArrayRelease(&src); + + struct ArrowSchema list_schema; + ASSERT_EQ(ArrowSchemaInitFromType(&list_schema, NANOARROW_TYPE_LIST), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetType(list_schema.children[0], NANOARROW_TYPE_NA), NANOARROW_OK); + ASSERT_EQ(ArrowArrayInitFromSchema(&src, &list_schema, &error), NANOARROW_OK); + ASSERT_EQ(ArrowArrayStartAppending(&src), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendNull(src.children[0], 1), NANOARROW_OK); + ASSERT_EQ(ArrowArrayFinishElement(&src), NANOARROW_OK); + ASSERT_EQ(ArrowArrayFinishBuildingDefault(&src, &error), NANOARROW_OK); + ASSERT_EQ(ArrowArrayViewInitFromSchema(&src_view, &list_schema, &error), NANOARROW_OK); + ASSERT_EQ(ArrowArrayViewSetArray(&src_view, &src, &error), NANOARROW_OK); + ASSERT_EQ(ArrowArrayInitFromSchema(&dst, &list_schema, &error), NANOARROW_OK); + ASSERT_EQ(ArrowArrayStartAppending(&dst), NANOARROW_OK); + dst.children[0]->length = INT32_MAX; + EXPECT_EQ(ArrowArrayAppendStorageFromArrayView(&dst, &src_view, &error), EOVERFLOW); + EXPECT_NE(error.message[0], '\0'); + + ArrowArrayRelease(&dst); + ArrowArrayViewReset(&src_view); + ArrowArrayRelease(&src); + ArrowSchemaRelease(&list_schema); +} + +TEST(ArrayTest, ArrayAppendStorageFromArrayViewDictionary) { + struct ArrowError error; + struct ArrowSchema schema; + ASSERT_EQ(ArrowSchemaInitFromType(&schema, NANOARROW_TYPE_INT8), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaAllocateDictionary(&schema), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaInitFromType(schema.dictionary, NANOARROW_TYPE_STRING), + NANOARROW_OK); + + struct ArrowArray src; + ASSERT_EQ(ArrowArrayInitFromSchema(&src, &schema, &error), NANOARROW_OK); + ASSERT_EQ(ArrowArrayStartAppending(&src), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendString(src.dictionary, "a"_asv), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendString(src.dictionary, "b"_asv), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendInt(&src, 0), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendInt(&src, 1), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendInt(&src, 0), NANOARROW_OK); + ASSERT_EQ(ArrowArrayFinishBuildingDefault(&src, &error), NANOARROW_OK); + struct ArrowArrayView src_view; + ASSERT_EQ(ArrowArrayViewInitFromSchema(&src_view, &schema, &error), NANOARROW_OK); + ASSERT_EQ(ArrowArrayViewSetArray(&src_view, &src, &error), NANOARROW_OK); + + struct ArrowArray dst; + ASSERT_EQ(ArrowArrayInitFromSchema(&dst, &schema, &error), NANOARROW_OK); + ASSERT_EQ(ArrowArrayStartAppending(&dst), NANOARROW_OK); + ASSERT_EQ( + ArrowArrayAppendStorageFromArrayView(dst.dictionary, src_view.dictionary, &error), + NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendStorageFromArrayView(&dst, &src_view, &error), NANOARROW_OK); + ASSERT_EQ(ArrowArrayFinishBuilding(&dst, NANOARROW_VALIDATION_LEVEL_FULL, &error), + NANOARROW_OK) + << error.message; + + struct ArrowArrayView dst_view; + ASSERT_EQ(ArrowArrayViewInitFromSchema(&dst_view, &schema, &error), NANOARROW_OK); + ASSERT_EQ(ArrowArrayViewSetArray(&dst_view, &dst, &error), NANOARROW_OK); + int identical = 0; + ASSERT_EQ(ArrowArrayViewCompare(&src_view, &dst_view, NANOARROW_COMPARE_IDENTICAL, + &identical, &error), + NANOARROW_OK); + EXPECT_EQ(identical, 1) << error.message; + + struct ArrowArray plain_dst; + ASSERT_EQ(ArrowArrayInitFromType(&plain_dst, NANOARROW_TYPE_INT8), NANOARROW_OK); + ASSERT_EQ(ArrowArrayStartAppending(&plain_dst), NANOARROW_OK); + EXPECT_EQ(ArrowArrayAppendStorageFromArrayView(&plain_dst, &src_view, &error), EINVAL); + EXPECT_STREQ(error.message, + "Can't append storage when exactly one of source and destination is " + "dictionary-encoded"); + + struct ArrowArrayView plain_view; + ArrowArrayViewInitFromType(&plain_view, NANOARROW_TYPE_INT8); + ASSERT_EQ(ArrowArrayViewSetArray(&plain_view, &plain_dst, &error), NANOARROW_OK); + EXPECT_EQ(ArrowArrayAppendStorageFromArrayView(&dst, &plain_view, &error), EINVAL); + EXPECT_STREQ(error.message, + "Can't append storage when exactly one of source and destination is " + "dictionary-encoded"); + + ArrowArrayViewReset(&plain_view); + ArrowArrayRelease(&plain_dst); + ArrowArrayViewReset(&dst_view); + ArrowArrayRelease(&dst); + ArrowArrayViewReset(&src_view); + ArrowArrayRelease(&src); + ArrowSchemaRelease(&schema); +} + +#if defined(NANOARROW_BUILD_TESTS_WITH_ARROW) + +enum class ArrowCppAppendValueKind { kSigned, kUnsigned, kString }; + +static std::shared_ptr BuildArrowCppAppendArray( + const std::shared_ptr& type, ArrowCppAppendValueKind value_kind) { + auto maybe_builder = MakeBuilder(type); + ARROW_EXPECT_OK(maybe_builder.status()); + auto builder = std::move(maybe_builder).ValueUnsafe(); + + for (int i = 0; i < 3; i++) { + if (i == 1) { + ARROW_EXPECT_OK(builder->AppendNull()); + continue; + } + + std::shared_ptr scalar; + switch (value_kind) { + case ArrowCppAppendValueKind::kSigned: { + auto maybe_scalar = MakeScalar(type, (int64_t)(i == 0 ? -1 : 42)); + ARROW_EXPECT_OK(maybe_scalar.status()); + scalar = maybe_scalar.ValueUnsafe(); + break; + } + case ArrowCppAppendValueKind::kUnsigned: { + auto maybe_scalar = MakeScalar(type, (uint64_t)(i == 0 ? 1 : 42)); + ARROW_EXPECT_OK(maybe_scalar.status()); + scalar = maybe_scalar.ValueUnsafe(); + break; + } + case ArrowCppAppendValueKind::kString: { + auto maybe_scalar = MakeScalar(type, std::string(i == 0 ? "abc" : "def")); + ARROW_EXPECT_OK(maybe_scalar.status()); + scalar = maybe_scalar.ValueUnsafe(); + break; + } + } + + ARROW_EXPECT_OK(builder->AppendScalar(*scalar)); + } + + auto maybe_array = builder->Finish(); + ARROW_EXPECT_OK(maybe_array.status()); + return maybe_array.ValueUnsafe(); +} + +static std::shared_ptr BuildArrowCppAppendListArray( + const std::shared_ptr& type) { + auto child_builder = std::make_shared(); + if (type->id() == Type::LIST) { + ListBuilder builder(default_memory_pool(), child_builder, type); + ARROW_EXPECT_OK(builder.Append()); + ARROW_EXPECT_OK(child_builder->Append(1)); + ARROW_EXPECT_OK(child_builder->Append(2)); + ARROW_EXPECT_OK(builder.AppendNull()); + ARROW_EXPECT_OK(builder.Append()); + ARROW_EXPECT_OK(child_builder->Append(3)); + auto maybe_array = builder.Finish(); + ARROW_EXPECT_OK(maybe_array.status()); + return maybe_array.ValueUnsafe(); + } else { + LargeListBuilder builder(default_memory_pool(), child_builder, type); + ARROW_EXPECT_OK(builder.Append()); + ARROW_EXPECT_OK(child_builder->Append(1)); + ARROW_EXPECT_OK(child_builder->Append(2)); + ARROW_EXPECT_OK(builder.AppendNull()); + ARROW_EXPECT_OK(builder.Append()); + ARROW_EXPECT_OK(child_builder->Append(3)); + auto maybe_array = builder.Finish(); + ARROW_EXPECT_OK(maybe_array.status()); + return maybe_array.ValueUnsafe(); + } +} + +static void ExpectArrowCppStorageAppendEquals(const std::shared_ptr& src, + const std::shared_ptr& expected) { + struct ArrowError error; + struct ArrowArray src_array; + struct ArrowSchema src_schema; + ARROW_EXPECT_OK(ExportArray(*src, &src_array, &src_schema)); + struct ArrowArrayView src_view; + ASSERT_EQ(ArrowArrayViewInitFromSchema(&src_view, &src_schema, &error), NANOARROW_OK) + << error.message; + ASSERT_EQ(ArrowArrayViewSetArray(&src_view, &src_array, &error), NANOARROW_OK) + << error.message; + + struct ArrowSchema dst_schema; + ARROW_EXPECT_OK(ExportType(*expected->type(), &dst_schema)); + struct ArrowArray dst; + ASSERT_EQ(ArrowArrayInitFromSchema(&dst, &dst_schema, &error), NANOARROW_OK) + << error.message; + ASSERT_EQ(ArrowArrayStartAppending(&dst), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendStorageFromArrayView(&dst, &src_view, &error), NANOARROW_OK) + << error.message; + ASSERT_EQ(ArrowArrayFinishBuilding(&dst, NANOARROW_VALIDATION_LEVEL_FULL, &error), + NANOARROW_OK) + << error.message; + + auto maybe_actual = ImportArray(&dst, &dst_schema); + ARROW_EXPECT_OK(maybe_actual.status()); + EXPECT_TRUE(maybe_actual.ValueUnsafe()->Equals(expected)) + << maybe_actual.ValueUnsafe()->ToString() << "\n!=\n" + << expected->ToString(); + + ArrowArrayViewReset(&src_view); + ArrowArrayRelease(&src_array); + ArrowSchemaRelease(&src_schema); +} + +TEST(ArrayTest, ArrayAppendStorageFromArrayViewArrowCppSignedIntegers) { + std::vector> types = {int8(), int16(), int32(), int64()}; + for (const auto& src_type : types) { + auto src = BuildArrowCppAppendArray(src_type, ArrowCppAppendValueKind::kSigned); + for (const auto& dst_type : types) { + SCOPED_TRACE(src_type->ToString() + " -> " + dst_type->ToString()); + auto expected = + BuildArrowCppAppendArray(dst_type, ArrowCppAppendValueKind::kSigned); + ExpectArrowCppStorageAppendEquals(src, expected); + } + } +} + +TEST(ArrayTest, ArrayAppendStorageFromArrayViewArrowCppUnsignedIntegers) { + std::vector> types = {uint8(), uint16(), uint32(), uint64()}; + for (const auto& src_type : types) { + auto src = BuildArrowCppAppendArray(src_type, ArrowCppAppendValueKind::kUnsigned); + for (const auto& dst_type : types) { + SCOPED_TRACE(src_type->ToString() + " -> " + dst_type->ToString()); + auto expected = + BuildArrowCppAppendArray(dst_type, ArrowCppAppendValueKind::kUnsigned); + ExpectArrowCppStorageAppendEquals(src, expected); + } + } +} + +TEST(ArrayTest, ArrayAppendStorageFromArrayViewArrowCppStrings) { + std::vector> types = {utf8(), large_utf8()}; + for (const auto& src_type : types) { + auto src = BuildArrowCppAppendArray(src_type, ArrowCppAppendValueKind::kString); + for (const auto& dst_type : types) { + SCOPED_TRACE(src_type->ToString() + " -> " + dst_type->ToString()); + auto expected = + BuildArrowCppAppendArray(dst_type, ArrowCppAppendValueKind::kString); + ExpectArrowCppStorageAppendEquals(src, expected); + } + } +} + +TEST(ArrayTest, ArrayAppendStorageFromArrayViewArrowCppBinary) { + std::vector> types = {binary(), large_binary()}; + for (const auto& src_type : types) { + auto src = BuildArrowCppAppendArray(src_type, ArrowCppAppendValueKind::kString); + for (const auto& dst_type : types) { + SCOPED_TRACE(src_type->ToString() + " -> " + dst_type->ToString()); + auto expected = + BuildArrowCppAppendArray(dst_type, ArrowCppAppendValueKind::kString); + ExpectArrowCppStorageAppendEquals(src, expected); + } + } +} + +TEST(ArrayTest, ArrayAppendStorageFromArrayViewArrowCppLists) { + std::vector> types = {list(int32()), large_list(int32())}; + for (const auto& src_type : types) { + auto src = BuildArrowCppAppendListArray(src_type); + for (const auto& dst_type : types) { + SCOPED_TRACE(src_type->ToString() + " -> " + dst_type->ToString()); + auto expected = BuildArrowCppAppendListArray(dst_type); + ExpectArrowCppStorageAppendEquals(src, expected); + } + } +} + +#endif From 693718d9f112a7d99134177ed7dc9b2c63d02a6e Mon Sep 17 00:00:00 2001 From: Rusty Conover Date: Thu, 3 Sep 2026 14:59:41 -0400 Subject: [PATCH 5/7] fix: allow fixed-to-variable storage appends --- src/nanoarrow/common/array.c | 3 ++ src/nanoarrow/common/array_test.cc | 86 ++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+) diff --git a/src/nanoarrow/common/array.c b/src/nanoarrow/common/array.c index d57b607d1..21de77353 100644 --- a/src/nanoarrow/common/array.c +++ b/src/nanoarrow/common/array.c @@ -135,6 +135,7 @@ static int ArrowTypeIsListLike(enum ArrowType type) { case NANOARROW_TYPE_MAP: case NANOARROW_TYPE_LIST_VIEW: case NANOARROW_TYPE_LARGE_LIST_VIEW: + case NANOARROW_TYPE_FIXED_SIZE_LIST: return 1; default: return 0; @@ -211,6 +212,7 @@ static ArrowErrorCode ArrowArrayCheckCanAppendStorageFromArrayView( } if (src->storage_type == NANOARROW_TYPE_FIXED_SIZE_LIST && + dst_type == NANOARROW_TYPE_FIXED_SIZE_LIST && src->layout.child_size_elements != private_data->layout.child_size_elements) { ArrowErrorSet(error, "Expected source and destination fixed-size list child sizes to " @@ -221,6 +223,7 @@ static ArrowErrorCode ArrowArrayCheckCanAppendStorageFromArrayView( } if (src->storage_type == NANOARROW_TYPE_FIXED_SIZE_BINARY && + dst_type == NANOARROW_TYPE_FIXED_SIZE_BINARY && src->layout.element_size_bits[1] != private_data->layout.element_size_bits[1]) { ArrowErrorSet(error, "Expected source and destination fixed-size binary widths to " diff --git a/src/nanoarrow/common/array_test.cc b/src/nanoarrow/common/array_test.cc index 29b67891b..038de0c02 100644 --- a/src/nanoarrow/common/array_test.cc +++ b/src/nanoarrow/common/array_test.cc @@ -5398,6 +5398,92 @@ TEST(ArrayTest, ArrayAppendStorageFromArrayViewFixedWidthSlicedValidity) { ArrowArrayRelease(&src); } +TEST(ArrayTest, ArrayAppendStorageFromArrayViewFixedToVariableWidth) { + struct ArrowError error; + + struct ArrowSchema fixed_binary_schema; + ArrowSchemaInit(&fixed_binary_schema); + ASSERT_EQ(ArrowSchemaSetTypeFixedSize(&fixed_binary_schema, + NANOARROW_TYPE_FIXED_SIZE_BINARY, 3), + NANOARROW_OK); + struct ArrowArray fixed_binary; + ASSERT_EQ(ArrowArrayInitFromSchema(&fixed_binary, &fixed_binary_schema, &error), + NANOARROW_OK); + ASSERT_EQ(ArrowArrayStartAppending(&fixed_binary), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendBytes(&fixed_binary, {{"abc"}, 3}), NANOARROW_OK); + ASSERT_EQ(ArrowArrayFinishBuildingDefault(&fixed_binary, &error), NANOARROW_OK); + struct ArrowArrayView fixed_binary_view; + ASSERT_EQ( + ArrowArrayViewInitFromSchema(&fixed_binary_view, &fixed_binary_schema, &error), + NANOARROW_OK); + ASSERT_EQ(ArrowArrayViewSetArray(&fixed_binary_view, &fixed_binary, &error), + NANOARROW_OK); + + struct ArrowArray binary; + ASSERT_EQ(ArrowArrayInitFromType(&binary, NANOARROW_TYPE_BINARY), NANOARROW_OK); + ASSERT_EQ(ArrowArrayStartAppending(&binary), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendStorageFromArrayView(&binary, &fixed_binary_view, &error), + NANOARROW_OK) + << error.message; + ASSERT_EQ(ArrowArrayFinishBuildingDefault(&binary, &error), NANOARROW_OK); + struct ArrowArrayView binary_view; + ArrowArrayViewInitFromType(&binary_view, NANOARROW_TYPE_BINARY); + ASSERT_EQ(ArrowArrayViewSetArray(&binary_view, &binary, &error), NANOARROW_OK); + EXPECT_EQ(ArrowArrayViewGetStringUnsafe(&binary_view, 0), "abc"_asv); + + ArrowArrayViewReset(&binary_view); + ArrowArrayRelease(&binary); + ArrowArrayViewReset(&fixed_binary_view); + ArrowArrayRelease(&fixed_binary); + ArrowSchemaRelease(&fixed_binary_schema); + + struct ArrowSchema fixed_list_schema; + ArrowSchemaInit(&fixed_list_schema); + ASSERT_EQ( + ArrowSchemaSetTypeFixedSize(&fixed_list_schema, NANOARROW_TYPE_FIXED_SIZE_LIST, 2), + NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetType(fixed_list_schema.children[0], NANOARROW_TYPE_INT32), + NANOARROW_OK); + struct ArrowArray fixed_list; + ASSERT_EQ(ArrowArrayInitFromSchema(&fixed_list, &fixed_list_schema, &error), + NANOARROW_OK); + ASSERT_EQ(ArrowArrayStartAppending(&fixed_list), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendInt(fixed_list.children[0], 1), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendInt(fixed_list.children[0], 2), NANOARROW_OK); + ASSERT_EQ(ArrowArrayFinishElement(&fixed_list), NANOARROW_OK); + ASSERT_EQ(ArrowArrayFinishBuildingDefault(&fixed_list, &error), NANOARROW_OK); + struct ArrowArrayView fixed_list_view; + ASSERT_EQ(ArrowArrayViewInitFromSchema(&fixed_list_view, &fixed_list_schema, &error), + NANOARROW_OK); + ASSERT_EQ(ArrowArrayViewSetArray(&fixed_list_view, &fixed_list, &error), NANOARROW_OK); + + struct ArrowSchema list_schema; + ASSERT_EQ(ArrowSchemaInitFromType(&list_schema, NANOARROW_TYPE_LIST), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetType(list_schema.children[0], NANOARROW_TYPE_INT32), + NANOARROW_OK); + struct ArrowArray list; + ASSERT_EQ(ArrowArrayInitFromSchema(&list, &list_schema, &error), NANOARROW_OK); + ASSERT_EQ(ArrowArrayStartAppending(&list), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendStorageFromArrayView(&list, &fixed_list_view, &error), + NANOARROW_OK) + << error.message; + ASSERT_EQ(ArrowArrayFinishBuildingDefault(&list, &error), NANOARROW_OK); + struct ArrowArrayView list_view; + ASSERT_EQ(ArrowArrayViewInitFromSchema(&list_view, &list_schema, &error), NANOARROW_OK); + ASSERT_EQ(ArrowArrayViewSetArray(&list_view, &list, &error), NANOARROW_OK); + EXPECT_EQ(ArrowArrayViewListChildOffset(&list_view, 0), 0); + EXPECT_EQ(ArrowArrayViewListChildOffset(&list_view, 1), 2); + EXPECT_EQ(ArrowArrayViewGetIntUnsafe(list_view.children[0], 0), 1); + EXPECT_EQ(ArrowArrayViewGetIntUnsafe(list_view.children[0], 1), 2); + + ArrowArrayViewReset(&list_view); + ArrowArrayRelease(&list); + ArrowSchemaRelease(&list_schema); + ArrowArrayViewReset(&fixed_list_view); + ArrowArrayRelease(&fixed_list); + ArrowSchemaRelease(&fixed_list_schema); +} + TEST(ArrayTest, ArrayAppendStorageFromArrayViewErrors) { struct ArrowError error; struct ArrowArray src; From b4f4adc00297c052dbeff92313a500fad8dfcdf9 Mon Sep 17 00:00:00 2001 From: Rusty Conover Date: Fri, 4 Sep 2026 07:51:10 -0400 Subject: [PATCH 6/7] fix: bound partial bitmap appends Avoid reading past short int8 or int32 input arrays when an append begins in a partially filled bitmap byte. --- src/nanoarrow/common/buffer_test.cc | 20 ++++++++++++++++++++ src/nanoarrow/common/inline_buffer.h | 8 ++++++++ 2 files changed, 28 insertions(+) diff --git a/src/nanoarrow/common/buffer_test.cc b/src/nanoarrow/common/buffer_test.cc index 3b3cc8af6..51c0964e5 100644 --- a/src/nanoarrow/common/buffer_test.cc +++ b/src/nanoarrow/common/buffer_test.cc @@ -647,6 +647,16 @@ TEST(BitmapTest, BitmapTestAppendInt8Unsafe) { EXPECT_EQ(ArrowBitGet(bitmap.buffer.data, i), test_values[i - 136]); } + // Append fewer values than are needed to complete the current byte + int8_t short_values[] = {1, 0, 1}; + ASSERT_EQ(ArrowBitmapReserve(&bitmap, 3), NANOARROW_OK); + ArrowBitmapAppendInt8Unsafe(&bitmap, short_values, 3); + EXPECT_EQ(bitmap.size_bits, 207); + EXPECT_EQ(bitmap.buffer.size_bytes, 26); + for (int i = 0; i < 3; i++) { + EXPECT_EQ(ArrowBitGet(bitmap.buffer.data, 204 + i), short_values[i]); + } + ArrowBitmapReset(&bitmap); } @@ -702,6 +712,16 @@ TEST(BitmapTest, BitmapTestAppendInt32Unsafe) { EXPECT_EQ(ArrowBitGet(bitmap.buffer.data, i), test_values[i - 136]); } + // Append fewer values than are needed to complete the current byte + int32_t short_values[] = {1, 0, 1}; + ASSERT_EQ(ArrowBitmapReserve(&bitmap, 3), NANOARROW_OK); + ArrowBitmapAppendInt32Unsafe(&bitmap, short_values, 3); + EXPECT_EQ(bitmap.size_bits, 207); + EXPECT_EQ(bitmap.buffer.size_bytes, 26); + for (int i = 0; i < 3; i++) { + EXPECT_EQ(ArrowBitGet(bitmap.buffer.data, 204 + i), short_values[i]); + } + ArrowBitmapReset(&bitmap); } diff --git a/src/nanoarrow/common/inline_buffer.h b/src/nanoarrow/common/inline_buffer.h index c9a78602b..1d4997380 100644 --- a/src/nanoarrow/common/inline_buffer.h +++ b/src/nanoarrow/common/inline_buffer.h @@ -635,6 +635,10 @@ static inline void ArrowBitmapAppendInt8Unsafe(struct ArrowBitmap* bitmap, // First byte if ((out_i_cursor % 8) != 0) { int64_t n_partial_bits = _ArrowRoundUpToMultipleOf8(out_i_cursor) - out_i_cursor; + if (n_partial_bits > n_remaining) { + n_partial_bits = n_remaining; + } + for (int i = 0; i < n_partial_bits; i++) { ArrowBitSetTo(bitmap->buffer.data, out_i_cursor++, values[i]); } @@ -685,6 +689,10 @@ static inline void ArrowBitmapAppendInt32Unsafe(struct ArrowBitmap* bitmap, // First byte if ((out_i_cursor % 8) != 0) { int64_t n_partial_bits = _ArrowRoundUpToMultipleOf8(out_i_cursor) - out_i_cursor; + if (n_partial_bits > n_remaining) { + n_partial_bits = n_remaining; + } + for (int i = 0; i < n_partial_bits; i++) { ArrowBitSetTo(bitmap->buffer.data, out_i_cursor++, (uint8_t)values[i]); } From 0603bc1cb1e75c0199c348740c1dfbd4e13554ef Mon Sep 17 00:00:00 2001 From: Rusty Conover Date: Fri, 4 Sep 2026 09:13:04 -0400 Subject: [PATCH 7/7] fix: initialize validity append scratch buffer Ensure clang-tidy can prove that all values passed through the bitmap packing path are initialized. --- src/nanoarrow/common/array.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nanoarrow/common/array.c b/src/nanoarrow/common/array.c index 21de77353..adf84264b 100644 --- a/src/nanoarrow/common/array.c +++ b/src/nanoarrow/common/array.c @@ -436,7 +436,7 @@ static ArrowErrorCode ArrowArrayAppendValidityFromArrayView( NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowBitmapReserve(dst_validity, src->length), error); - int8_t validity[1024]; + int8_t validity[1024] = {0}; for (int64_t offset = 0; offset < src->length;) { int64_t remaining = src->length - offset; int64_t chunk_size =