From bf22ecae1c935a2e57c9250e0f2c7b151366fd52 Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Tue, 14 Jul 2026 11:39:29 +0530 Subject: [PATCH 01/18] Update Erlang/OTP, Rebar3 and ExDoc --- .github/workflows/docs.yml | 4 ++-- .github/workflows/erlang.yml | 12 ++++++------ rebar.config | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index e149adb..7c1865a 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -45,8 +45,8 @@ jobs: - name: Install Erlang/OTP uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1 with: - otp-version: 25.1.0 - rebar3-version: '3.18.0' + otp-version: 29.0 + rebar3-version: 3.27.0 - name: Cache Hex packages uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 diff --git a/.github/workflows/erlang.yml b/.github/workflows/erlang.yml index 29e3b12..f2b6f56 100644 --- a/.github/workflows/erlang.yml +++ b/.github/workflows/erlang.yml @@ -43,8 +43,8 @@ jobs: - name: Install Erlang/OTP uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1 with: - otp-version: 25.1.0 - rebar3-version: '3.18.0' + otp-version: 29.0 + rebar3-version: 3.27.0 - name: Cache Hex packages uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -79,8 +79,8 @@ jobs: - name: Install Erlang/OTP uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1 with: - otp-version: 25.1.0 - rebar3-version: '3.18.0' + otp-version: 29.0 + rebar3-version: 3.27.0 - name: Cache Hex packages uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -118,8 +118,8 @@ jobs: - name: Install Erlang/OTP uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1 with: - otp-version: 25.1.0 - rebar3-version: '3.18.0' + otp-version: 29.0 + rebar3-version: 3.27.0 - name: Cache Hex packages uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 diff --git a/rebar.config b/rebar.config index 5a757fa..46065cd 100644 --- a/rebar.config +++ b/rebar.config @@ -29,7 +29,7 @@ ]}. {plugins, [rebar3_cargo]}. -{project_plugins, [erlfmt, {rebar3_ex_doc, "0.2.18"}]}. +{project_plugins, [erlfmt, {rebar3_ex_doc, "0.2.30"}]}. % Let erlfmt write files on format as opposed to not doing anything {erlfmt, [write]}. From 876f81a2f59c79e6d3cd6d586f3d7743108e2a1a Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Tue, 14 Jul 2026 18:39:37 +0530 Subject: [PATCH 02/18] Migrate arrow_array --- src/arrow_array.erl | 192 ++++++++++++++++++-------------------------- 1 file changed, 80 insertions(+), 112 deletions(-) diff --git a/src/arrow_array.erl b/src/arrow_array.erl index 9c2045b..194ee80 100644 --- a/src/arrow_array.erl +++ b/src/arrow_array.erl @@ -15,87 +15,51 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides a record, a behaviour, and functions to work with Apache Arrow -%% Arrays. -%% -%% This module as mentioned, provides conveniences for working with Apache Arrow -%% Arrays[1] of different Layouts[2]. Firstly, it provides a record to represent -%% all the Layout. Secondly, it provides a behaviour for different layouts to -%% adhere to common functionality. Lastly, it provides functions to work with -%% Arrays in a common manner. -%% -%% == The Structure of an Array == -%% -%% `arrow''s implementation of an Array has the following fields in its -%% record definition: -%% -%%
    -%%
  1. -%% `layout', of type {@link atom()}, which represents the Layout of the -%% Array. -%%
    2. -%%
  2. -%% `type', of type {@link arrow_type:arrow_type()}, which represents -%% the Logical Type[3] of the Array. -%%
    3. -%%
  3. `len', of type {@link pos_integer()}, which represents the Array's Length[4].
    4. -%%
  4. -%% `element_len', of type {@link pos_integer()} or `undefined', which -%% represents the Length of each element in an Array. Currently it only has -%% an integer value in the Fixed-Size List Layout[5] -%%
    5. -%%
  5. -%% `null_count', of type {@link non_neg_integer()}, which represents the -%% Array's Null Count[4], or the number of undefined values in the Array. -%%
    6. -%%
  6. -%% `validity_bitmap', which is a buffer (`arrow_buffer') or the atom -%% `undefined', which represents the Array's Validity Bitmap[6]. -%%
    7. -%%
  7. -%% `offsets', which is a buffer (`arrow_buffer') which represents the -%% Offsets[7], or the start position of each slot in the data buffer of an -%% Array. -%%
    8. -%%
  8. -%% `data', which is a buffer (`arrow_buffer'), which represents the -%% Array's Value Buffer, whose layout differs based on the Array Layout. -%%
    9. -%%
-%% -%% Certain fields are not required for certain layouts. For example, for the -%% Fixed-Sized Primitive Layout, the offsets field is not required, in which -%% case it is assigned as `undefined'. Similarly, the `validity_bitmap' is not -%% required if there are no null values, in which case it is also assigned as -%% `undefined'. -%% -%% == The Behaviour of an Array == -%% -%% As of right now, a layout needs to implement the `c:from_erlang/2' callback. This is -%% then used in the `from_erlang/3' function to create new arrays. -%% -%% == Functions for working with Arrays == -%% -%% As of right now, only functions to access the various fields, to create -%% new arrays, and to serialize arrays to arrow exist. -%% -%% == References == -%% -%% [1]: [https://arrow.apache.org/docs/format/Glossary.html#term-array] -%% -%% [2]: [https://arrow.apache.org/docs/format/Glossary.html#term-physical-layout] -%% -%% [3]: [https://arrow.apache.org/docs/format/Glossary.html#term-type] -%% -%% [4]: [https://arrow.apache.org/docs/format/Columnar.html#null-count] -%% -%% [5]: [https://arrow.apache.org/docs/format/Columnar.html#fixed-size-list-layout] -%% -%% [6]: [https://arrow.apache.org/docs/format/Columnar.html#validity-bitmaps] -%% -%% [7]: [https://arrow.apache.org/docs/format/Columnar.html#variable-size-binary-layout] -%% @end -module(arrow_array). +-moduledoc """ +Provides a record, a behaviour, and functions to work with Apache Arrow Arrays. + +This module as mentioned, provides conveniences for working with Apache Arrow +[Arrays](https://arrow.apache.org/docs/format/Glossary.html#term-array) of +different +[Layouts](https://arrow.apache.org/docs/format/Glossary.html#term-physical-layout). +Firstly, it provides a record to represent all the Layout. Secondly, it provides +a behaviour for different layouts to adhere to common functionality. Lastly, it +provides functions to work with Arrays in a common manner. + +# The Structure of an Array + +`arrow`'s implementation of an Array has the following fields in its record +definition, each corresponding to a field in the Arrow Columnar Format +specification: + +| Field | Type | Representing | +|-------------------|-----------------------------|-------------------------------------------------------------------------------------------| +| `layout` | `t:atom/0` | Layout | +| `type` | `t:arrow_type:arrow_type/0` | [Logical Type](https://arrow.apache.org/docs/format/Glossary.html#term-type) | +| `len` | `t:pos_integer/0` | [Length](https://arrow.apache.org/docs/format/Columnar.html#array-lengths) | +| `element_len` | `t:pos_integer/0` | Length of each element | +| `null_count` | `t:non_neg_integer/0` | [Null Count](https://arrow.apache.org/docs/format/Columnar.html#null-count) | +| `validity_bitmap` | `arrow_buffer` | [Validity Bitmap](https://arrow.apache.org/docs/format/Columnar.html#validity-bitmaps) | +| `offsets` | `arrow_buffer` | [Offsets](https://arrow.apache.org/docs/format/Columnar.html#variable-size-binary-layout) | +| `data` | `arrow_buffer` | Value Buffer | + + +Certain fields are not required for certain layouts. For example, for the +Fixed-Sized Primitive Layout, the offsets field is not required, in which case +it is assigned as `undefined`. Similarly, the `validity_bitmap` is not required +if there are no null values, in which case it is also assigned as `undefined`. + +# The Behaviour of an Array + +As of right now, a layout needs to implement the `c:from_erlang/2` callback. +This is then used in the `from_erlang/3` function to create new arrays. + +# Functions for working with Arrays + +As of right now, only functions to access the various fields, to create +new arrays, and to serialize arrays to arrow exist. +""". -export([ from_erlang/3, layout/1, @@ -113,23 +77,27 @@ -export_type([array/0, layout/0]). +-doc "Represents an Array.". -type array() :: #array{}. -%% Represents an Array. +-doc "Represents the Layout of an Array.". -type layout() :: fixed_primitive | variable_binary | fixed_list | variable_list. -%% Represents the Layout of an Array. %%%%%%%%%%%%%%%%%%%% %% Array Creation %% %%%%%%%%%%%%%%%%%%%% +-doc """ +Creates a new array of a certain layout, given its value and options from its +erlang representation. +""". -callback from_erlang(Value :: [arrow_type:native_type()], Opts :: map()) -> Array :: #array{}. -%% Creates a new array of a certain layout, given its value and options from its -%% erlang representation. -%% @doc A common way to create a new array, given its layout, value, and options. -%% from its erlang representation. +-doc """ +A common way to create a new array, given its layout, value, and options. +from its erlang representation. +""". -spec from_erlang( Layout :: layout(), Value :: [arrow_type:native_type()], @@ -148,42 +116,42 @@ from_erlang(Layout, Value, Opts) -> %% Array Data and Metadata Access %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% @doc Returns the layout of an array. +-doc "Returns the layout of an array.". -spec layout(Array :: #array{}) -> Layout :: layout(). layout(Array) -> Array#array.layout. -%% @doc Returns the type of an array. +-doc "Returns the type of an array.". -spec type(Array :: #array{}) -> Type :: arrow_type:arrow_type(). type(Array) -> Array#array.type. -%% @doc Returns the length of an array. +-doc "Returns the length of an array.". -spec len(Array :: #array{}) -> Length :: pos_integer(). len(Array) -> Array#array.len. -%% @doc Returns the length of an array. +-doc "Returns the length of an array.". -spec element_len(Array :: #array{}) -> Length :: pos_integer() | undefined. element_len(Array) -> Array#array.element_len. -%% @doc Returns the null count of an array. +-doc "Returns the null count of an array.". -spec null_count(Array :: #array{}) -> NullCount :: non_neg_integer(). null_count(Array) -> Array#array.null_count. -%% @doc Returns the validity bitmap of an array. +-doc "Returns the validity bitmap of an array.". -spec validity_bitmap(Array :: #array{}) -> ValidityBitmap :: #buffer{} | undefined. validity_bitmap(Array) -> Array#array.validity_bitmap. -%% @doc Returns the offsets of an array. +-doc "Returns the offsets of an array.". -spec offsets(Array :: #array{}) -> Offsets :: #buffer{} | undefined. offsets(Array) -> Array#array.offsets. -%% @doc Returns the data of an array. +-doc "Returns the data of an array.". -spec data(Array :: #array{}) -> Data :: #buffer{} | #array{} | undefined. data(Array) -> Array#array.data. @@ -192,24 +160,24 @@ data(Array) -> %% Array Serialization %% %%%%%%%%%%%%%%%%%%%%%%%%% -%% @doc Serializes an array into the Arrow binary form. -%% -%% Serializes the buffers of an Array and concatenates them in the following -%% order: -%% -%%
    -%%
  1. `validity'
    2. -%%
  2. `offsets'
    3. -%%
  3. `data'
    4. -%%
-%% -%% In case an array doesn't have any of the following buffers, it is ommitted. -%% (e.g. validity in arrays with a null count of 0, offsets in fixed primitive -%% arrays). In the case of a nested array, `data' will be serialized form of -%% nested array. -%% -%% Do note that this is just binary form that includes the buffers in an Array, -%% and not IPC. +-doc """ +Serializes an array into the Arrow binary form. + +Serializes the buffers of an Array and concatenates them in the following +order: + +1. `validity` +2. `offsets` +3. `data` + +In case an array doesn't have any of the following buffers, it is ommitted. +(e.g. validity in arrays with a null count of 0, offsets in fixed primitive +arrays). In the case of a nested array, `data` will be serialized form of nested +array. + +Do note that this is just binary form that includes the buffers in an Array, +and not IPC. +""". -spec to_arrow(Array :: #array{}) -> Arrow :: binary(). to_arrow(Array) -> Validity = some(validity_bitmap(Array)), From 8b8cba671836d3c9a19a9fce54d930e1d92708ac Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Tue, 14 Jul 2026 21:08:45 +0530 Subject: [PATCH 03/18] Fix erlfmt errors --- src/arrow_array.erl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/arrow_array.erl b/src/arrow_array.erl index 194ee80..a4eba71 100644 --- a/src/arrow_array.erl +++ b/src/arrow_array.erl @@ -16,6 +16,9 @@ % under the License. -module(arrow_array). + +%% TODO Check formatting with erlfmt +%% erlfmt:ignore -moduledoc """ Provides a record, a behaviour, and functions to work with Apache Arrow Arrays. @@ -160,6 +163,8 @@ data(Array) -> %% Array Serialization %% %%%%%%%%%%%%%%%%%%%%%%%%% +%% TODO Check formatting with erlfmt +%% erlfmt:ignore -doc """ Serializes an array into the Arrow binary form. From 9e39717959b533471e55eb1bb76f853a28e54e92 Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Tue, 14 Jul 2026 21:14:48 +0530 Subject: [PATCH 04/18] Quote OTP and Rebar3 versions --- .github/workflows/docs.yml | 4 ++-- .github/workflows/erlang.yml | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 7c1865a..f719874 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -45,8 +45,8 @@ jobs: - name: Install Erlang/OTP uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1 with: - otp-version: 29.0 - rebar3-version: 3.27.0 + otp-version: '29.0' + rebar3-version: '3.27.0' - name: Cache Hex packages uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 diff --git a/.github/workflows/erlang.yml b/.github/workflows/erlang.yml index f2b6f56..8be4a14 100644 --- a/.github/workflows/erlang.yml +++ b/.github/workflows/erlang.yml @@ -43,8 +43,8 @@ jobs: - name: Install Erlang/OTP uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1 with: - otp-version: 29.0 - rebar3-version: 3.27.0 + otp-version: '29.0' + rebar3-version: '3.27.0' - name: Cache Hex packages uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -79,8 +79,8 @@ jobs: - name: Install Erlang/OTP uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1 with: - otp-version: 29.0 - rebar3-version: 3.27.0 + otp-version: '29.0' + rebar3-version: '3.27.0' - name: Cache Hex packages uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -118,8 +118,8 @@ jobs: - name: Install Erlang/OTP uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1 with: - otp-version: 29.0 - rebar3-version: 3.27.0 + otp-version: '29.0' + rebar3-version: '3.27.0' - name: Cache Hex packages uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 From a0c0bbcc6ff103cf48e59b78425edb6ad9c969e5 Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Tue, 14 Jul 2026 21:38:18 +0530 Subject: [PATCH 05/18] Simplify type signatures --- src/arrow_array.erl | 30 +++++++++++++++--------------- src/arrow_array.hrl | 6 +++--- src/arrow_bitmap.erl | 4 ++-- src/arrow_buffer.erl | 15 ++++++++++----- src/arrow_ipc_record_batch.erl | 2 +- src/arrow_offsets.erl | 4 ++-- 6 files changed, 33 insertions(+), 28 deletions(-) diff --git a/src/arrow_array.erl b/src/arrow_array.erl index a4eba71..e04ae61 100644 --- a/src/arrow_array.erl +++ b/src/arrow_array.erl @@ -43,9 +43,9 @@ specification: | `len` | `t:pos_integer/0` | [Length](https://arrow.apache.org/docs/format/Columnar.html#array-lengths) | | `element_len` | `t:pos_integer/0` | Length of each element | | `null_count` | `t:non_neg_integer/0` | [Null Count](https://arrow.apache.org/docs/format/Columnar.html#null-count) | -| `validity_bitmap` | `arrow_buffer` | [Validity Bitmap](https://arrow.apache.org/docs/format/Columnar.html#validity-bitmaps) | -| `offsets` | `arrow_buffer` | [Offsets](https://arrow.apache.org/docs/format/Columnar.html#variable-size-binary-layout) | -| `data` | `arrow_buffer` | Value Buffer | +| `validity_bitmap` | `t:arrow_buffer:buffer/0` | [Validity Bitmap](https://arrow.apache.org/docs/format/Columnar.html#validity-bitmaps) | +| `offsets` | `t:arrow_buffer:buffer/0` | [Offsets](https://arrow.apache.org/docs/format/Columnar.html#variable-size-binary-layout) | +| `data` | `t:arrow_buffer:buffer/0` | Value Buffer | Certain fields are not required for certain layouts. For example, for the @@ -95,7 +95,7 @@ Creates a new array of a certain layout, given its value and options from its erlang representation. """. -callback from_erlang(Value :: [arrow_type:native_type()], Opts :: map()) -> - Array :: #array{}. + Array :: array(). -doc """ A common way to create a new array, given its layout, value, and options. @@ -106,7 +106,7 @@ from its erlang representation. Value :: [arrow_type:native_type()], Opts :: map() | arrow_type:arrow_type() ) -> - Array :: #array{}. + Array :: array(). from_erlang(Layout, Value, Opts) -> case Layout of fixed_primitive -> arrow_fixed_primitive_array:from_erlang(Value, Opts); @@ -120,42 +120,42 @@ from_erlang(Layout, Value, Opts) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -doc "Returns the layout of an array.". --spec layout(Array :: #array{}) -> Layout :: layout(). +-spec layout(Array :: array()) -> Layout :: layout(). layout(Array) -> Array#array.layout. -doc "Returns the type of an array.". --spec type(Array :: #array{}) -> Type :: arrow_type:arrow_type(). +-spec type(Array :: array()) -> Type :: arrow_type:arrow_type(). type(Array) -> Array#array.type. -doc "Returns the length of an array.". --spec len(Array :: #array{}) -> Length :: pos_integer(). +-spec len(Array :: array()) -> Length :: pos_integer(). len(Array) -> Array#array.len. -doc "Returns the length of an array.". --spec element_len(Array :: #array{}) -> Length :: pos_integer() | undefined. +-spec element_len(Array :: array()) -> Length :: pos_integer() | undefined. element_len(Array) -> Array#array.element_len. -doc "Returns the null count of an array.". --spec null_count(Array :: #array{}) -> NullCount :: non_neg_integer(). +-spec null_count(Array :: array()) -> NullCount :: non_neg_integer(). null_count(Array) -> Array#array.null_count. -doc "Returns the validity bitmap of an array.". --spec validity_bitmap(Array :: #array{}) -> ValidityBitmap :: #buffer{} | undefined. +-spec validity_bitmap(Array :: array()) -> ValidityBitmap :: arrow_buffer:buffer() | undefined. validity_bitmap(Array) -> Array#array.validity_bitmap. -doc "Returns the offsets of an array.". --spec offsets(Array :: #array{}) -> Offsets :: #buffer{} | undefined. +-spec offsets(Array :: array()) -> Offsets :: arrow_buffer:buffer() | undefined. offsets(Array) -> Array#array.offsets. -doc "Returns the data of an array.". --spec data(Array :: #array{}) -> Data :: #buffer{} | #array{} | undefined. +-spec data(Array :: array()) -> Data :: arrow_buffer:buffer() | array() | undefined. data(Array) -> Array#array.data. @@ -183,7 +183,7 @@ array. Do note that this is just binary form that includes the buffers in an Array, and not IPC. """. --spec to_arrow(Array :: #array{}) -> Arrow :: binary(). +-spec to_arrow(Array :: array()) -> Arrow :: binary(). to_arrow(Array) -> Validity = some(validity_bitmap(Array)), Offsets = some(offsets(Array)), @@ -191,7 +191,7 @@ to_arrow(Array) -> <>. --spec some(Value :: #array{} | #buffer{} | undefined) -> Binary :: binary(). +-spec some(Value :: array() | arrow_buffer:buffer() | undefined) -> Binary :: binary(). some(undefined) -> <<>>; some(Buffer) when is_record(Buffer, buffer) -> diff --git a/src/arrow_array.hrl b/src/arrow_array.hrl index dbb5ae9..4446bf1 100644 --- a/src/arrow_array.hrl +++ b/src/arrow_array.hrl @@ -23,7 +23,7 @@ len :: pos_integer(), element_len :: pos_integer() | undefined, null_count :: non_neg_integer(), - validity_bitmap :: #buffer{} | undefined, - offsets :: #buffer{} | undefined, - data :: #buffer{} | #array{} + validity_bitmap :: arrow_buffer:buffer() | undefined, + offsets :: arrow_buffer:buffer() | undefined, + data :: arrow_buffer:buffer() | #array{} }). diff --git a/src/arrow_bitmap.erl b/src/arrow_bitmap.erl index 1747514..973429f 100644 --- a/src/arrow_bitmap.erl +++ b/src/arrow_bitmap.erl @@ -65,7 +65,7 @@ %% @doc Returns the Validity Bitmap along with the Null Count, of %% an Array. -spec validity_bitmap(Value :: [arrow_type:native_type()] | list()) -> - {Bitmap :: #buffer{}, non_neg_integer()}. + {Bitmap :: arrow_buffer:buffer(), non_neg_integer()}. validity_bitmap(Value) -> case (lists:member(undefined, Value)) orelse (lists:member(nil, Value)) of true -> @@ -78,7 +78,7 @@ validity_bitmap(Value) -> Value :: [arrow_type:native_type()], Acc :: binary(), NullCount :: non_neg_integer() -) -> {Bitmap :: #buffer{}, NullCount :: non_neg_integer()}. +) -> {Bitmap :: arrow_buffer:buffer(), NullCount :: non_neg_integer()}. bitmap([X1, X2, X3, X4, X5, X6, X7, X8 | Rest], Acc, NullCount) -> %% By assigning B8 as X1's validity, we are following LSB numbering. B8 = validity(X1), diff --git a/src/arrow_buffer.erl b/src/arrow_buffer.erl index 08ff561..79aa125 100644 --- a/src/arrow_buffer.erl +++ b/src/arrow_buffer.erl @@ -56,6 +56,11 @@ -include("arrow_buffer.hrl"). +-export_type([buffer/0]). + +-doc "Represents an Arrow Buffer.". +-type buffer() :: arrow_buffer:buffer(). + %% @doc Creates a new buffer from a list of Erlang values or binaries, given its %% type %% @end @@ -63,7 +68,7 @@ Value :: [arrow_type:native_type()] | binary(), Type :: arrow_type:arrow_longhand_type() ) -> - Buffer :: #buffer{}. + Buffer :: arrow_buffer:buffer(). from_erlang(Data, Type) -> Len = case Type of @@ -82,7 +87,7 @@ from_erlang(Data, Type) -> Type :: arrow_type:arrow_longhand_type(), DataLen :: pos_integer() | undefined ) -> - Buffer :: #buffer{}. + Buffer :: arrow_buffer:buffer(). from_erlang(Data, Type, DataLen) -> Len = case Type of @@ -97,7 +102,7 @@ from_erlang(Data, Type, DataLen) -> %% @doc Returns an Arrow buffer binary given a buffer. %% @end --spec to_arrow(Buffer :: #buffer{}) -> binary(). +-spec to_arrow(Buffer :: arrow_buffer:buffer()) -> binary(). to_arrow(Buffer) when is_record(Buffer, buffer) -> Type = Buffer#buffer.type, Bin = @@ -114,14 +119,14 @@ to_arrow(_Buffer) -> %% @doc Returns a list of Erlang values or binaries from a buffer. %% @end --spec to_erlang(Buffer :: #buffer{}) -> [arrow_type:native_type()]. +-spec to_erlang(Buffer :: arrow_buffer:buffer()) -> [arrow_type:native_type()]. to_erlang(Buffer) when is_record(Buffer, buffer) -> Buffer#buffer.data; to_erlang(_Buffer) -> erlang:error(badarg). %% @doc Returns the size of the buffer inclusive of padding in bytes. --spec size(Buffer :: #buffer{}) -> pos_integer(). +-spec size(Buffer :: arrow_buffer:buffer()) -> pos_integer(). size(Buffer) -> Len = Buffer#buffer.length * 8, round((Len + arrow_utils:pad_len(Len)) / 8). diff --git a/src/arrow_ipc_record_batch.erl b/src/arrow_ipc_record_batch.erl index 8e7fac8..c69b3d7 100644 --- a/src/arrow_ipc_record_batch.erl +++ b/src/arrow_ipc_record_batch.erl @@ -115,7 +115,7 @@ array_data(Array, CurOffset) -> {Buffers ++ [DataBuffer], NewOffset} end. --spec buffer_data(Buffer :: #buffer{}, CurOffset :: non_neg_integer()) -> +-spec buffer_data(Buffer :: arrow_buffer:buffer(), CurOffset :: non_neg_integer()) -> {BufferData :: buffer(), NewOffset :: pos_integer()}. buffer_data(Buffer, CurOffset) -> { diff --git a/src/arrow_offsets.erl b/src/arrow_offsets.erl index 73461c7..31f4b72 100644 --- a/src/arrow_offsets.erl +++ b/src/arrow_offsets.erl @@ -66,7 +66,7 @@ Value :: [arrow_type:native_type()], Type :: arrow_type:arrow_longhand_type() ) -> - Buffer :: #buffer{}. + Buffer :: arrow_buffer:buffer(). new(Values, Type) -> new(Values, Type, length(Values)). @@ -76,7 +76,7 @@ new(Values, Type) -> Type :: arrow_type:arrow_longhand_type(), Length :: pos_integer() ) -> - Buffer :: #buffer{}. + Buffer :: arrow_buffer:buffer(). new(Values, Type, Len) -> Offsets = offsets(Values, [0], 0, Type), arrow_buffer:from_erlang(Offsets, {s, 32}, Len + 1). From d8266b2fca21434a9b996e319e97ebacf1f40436 Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Wed, 15 Jul 2026 10:53:43 +0530 Subject: [PATCH 06/18] Migrate arrow_buffer --- src/arrow_buffer.erl | 87 ++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 48 deletions(-) diff --git a/src/arrow_buffer.erl b/src/arrow_buffer.erl index 79aa125..0c6cff8 100644 --- a/src/arrow_buffer.erl +++ b/src/arrow_buffer.erl @@ -15,43 +15,35 @@ % specific language governing permissions and limitations % under the License. -%% @doc Buffer implementation for `arrow'. -%% This module adds suppport for buffers, or Contiguous Memory Regions. -%% -%% There are multiple things to know about buffers[1]: -%% -%%
    -%%
  1. -%% Each value it stores is called an element or a slot[2]. -%%
    2. -%%
  2. -%% Each slot's length (in bytes) is a positive integer. As a result when we -%% say that a slot has a length of 1, we mean that each slot has a length of -%% 1 byte. -%%
    3. -%%
  3. -%% The buffer's length in the metadata refers to the unpadded binary's size in bytes. -%%
    4. -%%
  4. -%% All buffers have a size that is a multiple of 64. If their data's length -%% is not a multiple of 64, it must be padded (in this implementation, by -%% zeros). -%%
    5. -%%
  5. -%% Null values are represented in this implementation by zeros. -%%
    6. -%%
  6. -%% In this implementation buffers can be initialized from raw bytes as data -%% apart datatypes supported by Arrow. This is so that the Validity Bitmap -%% Buffer can be initialized. -%%
    7. -%%
-%% -%% [1]: [https://arrow.apache.org/docs/format/Glossary.html#term-buffer] -%% -%% [2]: [https://arrow.apache.org/docs/format/Glossary.html#term-slot] -%% @end -module(arrow_buffer). + +%% TODO Check formatting with erlfmt +%% erlfmt:ignore +-moduledoc """ +Buffer implementation for `arrow`. + +This module adds suppport for buffers, or Contiguous Memory Regions. There are +multiple things to know about +[buffers](https://arrow.apache.org/docs/format/Glossary.html#term-buffer): + +1. Each value it stores is called an element or a + [slot](https://arrow.apache.org/docs/format/Glossary.html#term-slot). + +2. Each slot's length (in bytes) is a positive integer. As a result when we say + that a slot has a length of 1, we mean that each slot has a length of 1 byte. + +3. The buffer's length in the metadata refers to the unpadded binary's size in + bytes. + +4. All buffers have a size that is a multiple of 64. If their data's length is + not a multiple of 64, it must be padded (in this implementation, by zeros). + +5. Null values are represented in this implementation by zeros. + +6. In this implementation buffers can be initialized from raw bytes as data + apart datatypes supported by Arrow. This is so that the Validity Bitmap + Buffer can be initialized. +""". -export([from_erlang/2, from_erlang/3, to_arrow/1, to_erlang/1, size/1]). -include("arrow_buffer.hrl"). @@ -59,11 +51,11 @@ -export_type([buffer/0]). -doc "Represents an Arrow Buffer.". --type buffer() :: arrow_buffer:buffer(). +-type buffer() :: #buffer{}. -%% @doc Creates a new buffer from a list of Erlang values or binaries, given its -%% type -%% @end +-doc """ +Creates a new buffer from a list of Erlang values or binaries, given its type. +""". -spec from_erlang( Value :: [arrow_type:native_type()] | binary(), Type :: arrow_type:arrow_longhand_type() @@ -79,9 +71,10 @@ from_erlang(Data, Type) -> end, from_erlang(Data, Type, Len). -%% @doc Creates a new buffer from a list of Erlang values or binaries, given its -%% type and length -%% @end +-doc """ +Creates a new buffer from a list of Erlang values or binaries, given its +and length +""". -spec from_erlang( Data :: [arrow_type:native_type()] | binary(), Type :: arrow_type:arrow_longhand_type(), @@ -100,8 +93,7 @@ from_erlang(Data, Type, DataLen) -> end, #buffer{type = Type, length = Len, data = Data}. -%% @doc Returns an Arrow buffer binary given a buffer. -%% @end +-doc "Returns an Arrow buffer binary given a buffer.". -spec to_arrow(Buffer :: arrow_buffer:buffer()) -> binary(). to_arrow(Buffer) when is_record(Buffer, buffer) -> Type = Buffer#buffer.type, @@ -117,15 +109,14 @@ to_arrow(Buffer) when is_record(Buffer, buffer) -> to_arrow(_Buffer) -> erlang:error(badarg). -%% @doc Returns a list of Erlang values or binaries from a buffer. -%% @end +-doc "Returns a list of Erlang values or binaries from a buffer.". -spec to_erlang(Buffer :: arrow_buffer:buffer()) -> [arrow_type:native_type()]. to_erlang(Buffer) when is_record(Buffer, buffer) -> Buffer#buffer.data; to_erlang(_Buffer) -> erlang:error(badarg). -%% @doc Returns the size of the buffer inclusive of padding in bytes. +-doc "Returns the size of the buffer inclusive of padding in bytes.". -spec size(Buffer :: arrow_buffer:buffer()) -> pos_integer(). size(Buffer) -> Len = Buffer#buffer.length * 8, From 0cfa915b043e4182e2db76569132ef5d4be59ce1 Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Sun, 19 Jul 2026 15:24:26 +0530 Subject: [PATCH 07/18] Migrate arrow_offsets --- src/arrow_offsets.erl | 88 +++++++++++++++++++++---------------------- 1 file changed, 43 insertions(+), 45 deletions(-) diff --git a/src/arrow_offsets.erl b/src/arrow_offsets.erl index 31f4b72..4e1d49b 100644 --- a/src/arrow_offsets.erl +++ b/src/arrow_offsets.erl @@ -15,53 +15,49 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides support for Apache Arrow's Offsets -%% -%% Arrow has a concept of offsets[1] in order to tell the length of a slot[2] or -%% a single element in an array of variable-size elements. This module provides -%% support for generating offsets. -%% -%% There are couple of things to remember about offsets: -%% -%%
    -%%
  1. -%% Each element in the offsets coresponds to the distance in bytes of the -%% coresponding element in the values from the beginning of the buffer. -%% -%% I.E., distance of values[j] from beginning of the buffer = offsets[j]. -%%
    2. -%%
  2. -%% The very last element in the offsets is the length of the buffer, or -%% distance of the end of the last slot from the beginning of the buffer. -%%
    3. -%%
  3. -%% Thus, the offsets is one element longer than the values, or: -%% -%% length(offsets) == length(values) + 1 -%%
    4. -%%
  4. -%% Therefore, in order to find the length of a slot, we subtract the offset -%% the current element from the offset of the next element, or: -%% -%% slot[j] = offsets[j + 1] - offsets[j] -%%
    5. -%%
  5. -%% Null values have an offset of 0 as they take no memory in the buffer. -%% Thus, the previous offset and the current offset are equivalent if the -%% current element is a null. -%%
    6. -%%
-%% -%% [1]: [https://arrow.apache.org/docs/format/Columnar.html#variable-size-binary-layout] -%% -%% [2]: [https://arrow.apache.org/docs/format/Columnar.html#terminology] -%% @end -module(arrow_offsets). --export([new/2, new/3]). +-moduledoc """ +Provides support for Apache Arrow's Offsets + +Arrow has a concept of +[offsets](https://arrow.apache.org/docs/format/Columnar.html#variable-size-binary-layout) +in order to tell the length of a +[slot](https://arrow.apache.org/docs/format/Columnar.html#terminology) or a +single element in an array of variable-size elements. This module provides +support for generating offsets. + +There are couple of things to remember about offsets: + +Firstly, each element in the offsets coresponds to the distance in bytes of the +coresponding element in the values from the beginning of the buffer. + +I.E., distance of `values[j]` from beginning of the buffer = `offsets[j]`. + +Secondly, the very last element in the offsets is the length of the buffer, or +distance of the end of the last slot from the beginning of the buffer. --include("arrow_buffer.hrl"). +Thus, the offsets is one element longer than the values, or: + +``` +length(offsets) == length(values) + 1 +``` + +Therefore, in order to find the length of a slot, we subtract the offset the +current element from the offset of the next element, or: + +``` +slot[j] = offsets[j + 1] - offsets[j] +``` + +Finally, null values have an offset of 0 as they take no memory in the buffer. +Thus, the previous offset and the current offset are equivalent if the current +element is a null. +""". +-export([new/2, new/3]). -%% @doc Returns the offsets array given some values and their type as a buffer. +-doc """ +Returns the offsets array given some values and their type as a buffer. +""". -spec new( Value :: [arrow_type:native_type()], Type :: arrow_type:arrow_longhand_type() @@ -70,7 +66,9 @@ new(Values, Type) -> new(Values, Type, length(Values)). -%% @doc Returns the offsets array given some values, their type and length as a buffer. +-doc """ +Returns the offsets array given some values, their type and length as a buffer. +""". -spec new( Value :: [arrow_type:native_type()], Type :: arrow_type:arrow_longhand_type(), From 553f351f0b86566832067e8b7ececcd7c61c6b45 Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Sun, 19 Jul 2026 15:25:55 +0530 Subject: [PATCH 08/18] Migrate arrow_utils --- src/arrow_utils.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/arrow_utils.erl b/src/arrow_utils.erl index ff45a22..a32caea 100644 --- a/src/arrow_utils.erl +++ b/src/arrow_utils.erl @@ -15,8 +15,8 @@ % specific language governing permissions and limitations % under the License. -%% @private Utils module -module(arrow_utils). +-moduledoc false. -export([nesting/1, flatten/1, flatten/3, pad_len/1, pad_len/2]). %% Finds the nesting level of deep list From 4097e8dc4f38e7b85c2296ab6194dc72e1ee9187 Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Sun, 19 Jul 2026 15:35:27 +0530 Subject: [PATCH 09/18] Migrate arrow_bitmap --- src/arrow_bitmap.erl | 76 ++++++++++++++++++++------------------------ 1 file changed, 34 insertions(+), 42 deletions(-) diff --git a/src/arrow_bitmap.erl b/src/arrow_bitmap.erl index 973429f..449a9df 100644 --- a/src/arrow_bitmap.erl +++ b/src/arrow_bitmap.erl @@ -15,55 +15,47 @@ % specific language governing permissions and limitations % under the License. -%% @doc Validity Bitmap implementation for `arrow'. -%% -%% Defines a function `validity_bitmap/1' to return the Validity Bitmap[1] along -%% with the Null Count[2], of an Array. -%% -%% An important thing to consider about our implementation of the Null Count is -%% that we need to support both `undefined' and `nil' as null values as they are -%% the conventions for null values in Erlang, and Elixir respectively. -%% -%% There are 5 important characteristics to remember about the validity bitmap: -%% -%%
    -%%
  1. -%% A null value is represented by a 0 bit, and a non null value by a 1 bit. -%%
    2. -%%
  2. -%% Every 8 elements's validities are batched into a byte, which are then -%% reversed as Arrow uses least-significant bit (LSB) numbering (more in -%% attached reference). -%%
    3. -%%
  3. -%% If a "batch" consists of less than 8 elements, its validity needs to be -%% padded by 0 bits so that it can make a byte. -%%
    4. -%%
  4. -%% Each byte is stored in a slot of a Buffer (see docs for -%% `arrow_buffer'). This buffer with the validities of each batch of 8 -%% elements make up what is called the Validity Bitmap. -%%
    5. -%%
  5. -%% If the Null Count is 0, we can allocate the Validity Bitmap as a NULL -%% pointer (which in Erlang's case is `undefined'). -%%
    6. -%%
-%% [1]: [https://arrow.apache.org/docs/format/Columnar.html#validity-bitmaps] -%% -%% [2]: [https://arrow.apache.org/docs/format/Columnar.html#null-count] -%% @end -module(arrow_bitmap). --export([validity_bitmap/1]). +-moduledoc """ +Validity Bitmap implementation for `arrow`. + +Defines a function `validity_bitmap/1` to return the [Validity +Bitmap](https://arrow.apache.org/docs/format/Columnar.html#validity-bitmaps) +along with the [Null +Count](https://arrow.apache.org/docs/format/Columnar.html#null-count), of an +Array. + +An important thing to consider about our implementation of the Null Count is +that we need to support both `undefined` and `nil` as null values as they are +the conventions for null values in Erlang, and Elixir respectively. + +There are 5 important characteristics to remember about the validity bitmap: --include("arrow_buffer.hrl"). +1. A null value is represented by a 0 bit, and a non null value by a 1 bit. + +2. Every 8 elements's validities are batched into a byte, which are then + reversed as Arrow uses least-significant bit (LSB) numbering (more in + attached reference). + +3. If a "batch" consists of less than 8 elements, its validity needs to be + padded by 0 bits so that it can make a byte. + +4. Each byte is stored in a slot of a Buffer (see docs for `arrow_buffer`). This + buffer with the validities of each batch of 8 elements make up what is called + the Validity Bitmap. + +5. If the Null Count is 0, we can allocate the Validity Bitmap as a NULL pointer + (which in Erlang's case is `undefined`). +""". +-export([validity_bitmap/1]). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Validity Bitmap & Null Count %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% @doc Returns the Validity Bitmap along with the Null Count, of -%% an Array. +-doc """ +Returns the Validity Bitmap along with the Null Count, of an Array. +""". -spec validity_bitmap(Value :: [arrow_type:native_type()] | list()) -> {Bitmap :: arrow_buffer:buffer(), non_neg_integer()}. validity_bitmap(Value) -> From 5ece60b8c9fb9552450d9f6dd0ea18d69e7c9c81 Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Sun, 19 Jul 2026 18:22:30 +0530 Subject: [PATCH 10/18] Migrate arrow_type --- src/arrow_type.erl | 211 ++++++++++++++++++++------------------------- 1 file changed, 93 insertions(+), 118 deletions(-) diff --git a/src/arrow_type.erl b/src/arrow_type.erl index f7a92b0..a239b01 100644 --- a/src/arrow_type.erl +++ b/src/arrow_type.erl @@ -15,123 +15,94 @@ % specific language governing permissions and limitations % under the License. -%%%----------------------------------------------------------------------------- -%% @doc Provides representation for as well as functions for working with Arrow -%% specific datatypes. -%% -%% This module provides a longhand (a 2 tuple) as well as a shorthand (an atom) -%% represenations for representing a primitive types. Additionally it represents -%% Nested Types (i.e. Lists) as well as Native Types (i.e. Erlang Native Types -%% that are supported by this package). -%% -%% Generally, when we say an array A has type X, we mean that each element in A -%% has type X. Thus, an array of type of `Int8' would look like `[1, 2, 3]', but -%% an array of type `List' would look like `[[1, 2], [3]]'. -%% -%% == Primitive Types == -%% -%% A Primitive Type is any type that refers to a single value (as opposed to a -%% list of values). Currently it includes Booleans, Signed Integers, Unsigned -%% Integers, Floats and Binaries. Both a longhand and shorthand syntax had been -%% provided to initiate a type. -%% -%% The following is a comprehensive list of all supported primitive types: -%% -%%
    -%%
  1. Booleans
    2. -%%
  2. Signed Integers: Includes `Int 8', `Int 16', `Int 32' and `Int 64'
    3. -%%
  3. -%% Unsigned Integers: Includes `UInt 8', `UInt 16', `UInt 32' and `UInt 64' -%%
    4. -%%
  4. Floats: Includes `Float 16', `Float 32' and `Float 64'
    5. -%%
  5. Binaries
    6. -%%
-%% -%% === Shorthand Syntax === -%% -%% In the shorthand syntax, you can just use a single atom to refer to a type, -%% such that the atom is `'. The name is a single letter that refers -%% to the type, and the size is that type's size in bits. Take the example of -%% `Int 8'. Its name is `s', and size is `8'. Thus in shorthand, it is `s8'. In -%% the case of datatypes without variations, such as Booleans and Binaries, they -%% are represented by just their datatype. Do note that internally the longhand -%% syntax is used. -%% -%% Here you can find the name of each type: -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%%
TypeNameExample Atom
BooleansN/A`bool'
Signed Integer`s'`s64'
Unsigned Integer`u'`u16'
Float`f'`f32'
BinariesN/A`bin'
-%% -%% Types that are shorthand are under the `t:arrow_shorthand_type()' and -%% `arrow_short_*()' types. -%% -%% === Longhand Syntax === -%% -%% In longhand syntax, you use a 2 tuple to to refer to the type, such that the -%% tuple is `{Name, Size}'. Thus, `Int 8' would be represented as `{s, 8}'. Do -%% note that in the case or Booleans and Binaries their `Size' is `undefined', as -%% the have variable or undefined size. -%% -%% Types that are longhand are under the `t:arrow_longhand_type()' and -%% `arrow_long_*()' types. -%% -%% == Nested Type == -%% -%% A Nested Type is any data structure that supports nesting. This can include -%% Lists, FixedLists, Maps, Structs and more, but is curently limited to -%% FixedLists. Nested Types are represented using a 3 tuple: `{NestedType, Type, -%% Length}'. `NestedType' refers to the data structure that is nesting your -%% primitive type. `Type' refers to the type that is being nested in the data -%% structure. Do note that `Type' can be a Primitive Type, or another Nested -%% Type. Both the shorthand and the longhand syntax can be used or Primitive -%% Types.`Length' refers to the length of each element, and will only be -%% required to represent FixedLists. -%% -%% It is important to remember while nesting that the type refers to the type of -%% each element in an array. A list of `Int8' is 2 dimensional, and a list of -%% `List' is 3 dimensional. -%% -%% == Native Type == -%% -%% A Native Type is any Erlang Datatype that can be serialized by `arrow', -%% and is represented by the type `t:native_type()'. It currently only includes -%% values that are primitive in nature such as booleans, integers, floats, -%% binaries, and additionally the atoms `undefined' and `nil' to represent a -%% `NULL' value. -%% -%% @end -%%%----------------------------------------------------------------------------- -module(arrow_type). +-moduledoc """ +Provides representation for as well as functions for working with Arrow +specific datatypes. + +This module provides a longhand (a 2 tuple) as well as a shorthand (an atom) +represenations for representing a primitive types. Additionally it represents +Nested Types (i.e. Lists) as well as Native Types (i.e. Erlang Native Types +that are supported by this package). + +Generally, when we say an array A has type X, we mean that each element in A +has type X. Thus, an array of type of `Int8` would look like `[1, 2, 3]`, but +an array of type `List` would look like `[[1, 2], [3]]`. + +## Primitive Types + +A Primitive Type is any type that refers to a single value (as opposed to a +list of values). Currently it includes Booleans, Signed Integers, Unsigned +Integers, Floats and Binaries. Both a longhand and shorthand syntax had been +provided to initiate a type. + +The following is a comprehensive list of all supported primitive types: + +1. Booleans +2. Signed Integers: Includes `Int 8`, `Int 16`, `Int 32` and + `Int 64` +3. Unsigned Integers: Includes `UInt 8`, `UInt 16`, `UInt 32` and + `UInt 64` +4. Floats: Includes `Float 16`, `Float 32` and `Float 64` +5. Binaries + +### Shorthand Syntax + +In the shorthand syntax, you can just use a single atom to refer to a type, +such that the atom is ``. The name is a single letter that refers +to the type, and the size is that type's size in bits. Take the example of +`Int 8`. Its name is `s`, and size is `8`. Thus in shorthand, it is `s8`. In +the case of datatypes without variations, such as Booleans and Binaries, they +are represented by just their datatype. Do note that internally the longhand +syntax is used. + +Here you can find the name of each table: + +| Type | Name | Example Atom | +|------------------|-------|--------------| +| Booleans | N/A | `bool` | +| Signed Integer | `s` | `s64` | +| Unsigned Integer | `u ` | `u16 ` | +| Float | `f ` | `f32 ` | +| Binaries | N/A | `bin ` | + +Types that are shorthand are under the `t:arrow_shorthand_type()` and +`arrow_short_*()` types. + +### Longhand Syntax + +In longhand syntax, you use a 2 tuple to to refer to the type, such that the +tuple is `{Name, Size}`. Thus, `Int 8` would be represented as `{s, 8}`. Do +note that in the case or Booleans and Binaries their `Size` is `undefined`, as +the have variable or undefined size. + +Types that are longhand are under the `t:arrow_longhand_type()` and +`arrow_long_*()` types. + +### Type + +A Nested Type is any data structure that supports nesting. This can include +Lists, FixedLists, Maps, Structs and more, but is curently limited to +FixedLists. Nested Types are represented using a 3 tuple: `{NestedType, Type, +Length}`. `NestedType` refers to the data structure that is nesting your +primitive type. `Type` refers to the type that is being nested in the data +structure. Do note that `Type` can be a Primitive Type, or another Nested +Type. Both the shorthand and the longhand syntax can be used or Primitive +Types.`Length` refers to the length of each element, and will only be +required to represent FixedLists. + +It is important to remember while nesting that the type refers to the type of +each element in an array. A list of `Int8` is 2 dimensional, and a list of +`List` is 3 dimensional. + +## Native Type + +A Native Type is any Erlang Datatype that can be serialized by `arrow`, +and is represented by the type `t:native_type()`. It currently only includes +values that are primitive in nature such as booleans, integers, floats, +binaries, and additionally the atoms `undefined` and `nil` to represent a +`NULL` value. +""". -export([normalize/1, bit_length/1, byte_length/1, serialize/2]). -export_type([ arrow_type/0, @@ -354,7 +325,9 @@ normalize(_Type) -> erlang:error(badarg). -spec bit_length(Type :: arrow_type()) -> Length :: pos_integer() | undefined. -%% @doc Returns the size of the type in bits. +-doc """ +Returns the size of the type in bits. +""". bit_length(Type) when is_atom(Type) -> bit_length(normalize(Type)); bit_length({Type, Size}) when (Type =:= s) orelse (Type =:= u) orelse (Type =:= f) -> @@ -367,7 +340,9 @@ bit_length(Type) when tuple_size(Type) =:= 3 -> erlang:error(badarg). -spec byte_length(Type :: arrow_type()) -> Length :: pos_integer() | undefined. -%% @doc Returns the size of the type in bytes. +-doc """ +Returns the size of the type in bytes. +""". byte_length(Type) when is_atom(Type) -> byte_length(normalize(Type)); byte_length(Type) when tuple_size(Type) =:= 3 -> From 9b8fb5e3a816d5dabccf3a4fb7193162d51cef22 Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Sun, 19 Jul 2026 19:13:15 +0530 Subject: [PATCH 11/18] Migrate arrow_ipc_* --- src/arrow_ipc_field.erl | 76 ++++++++++----------- src/arrow_ipc_file.erl | 25 ++++--- src/arrow_ipc_message.erl | 119 +++++++++++++++++---------------- src/arrow_ipc_record_batch.erl | 69 ++++++++----------- src/arrow_ipc_schema.erl | 71 ++++++++------------ src/arrow_ipc_type.erl | 27 ++++---- 6 files changed, 182 insertions(+), 205 deletions(-) diff --git a/src/arrow_ipc_field.erl b/src/arrow_ipc_field.erl index 4ed6145..f9eb2f0 100644 --- a/src/arrow_ipc_field.erl +++ b/src/arrow_ipc_field.erl @@ -15,61 +15,55 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides a record and functions to deal with individual fields in -%% a Schema -%% -%% A Field[1] represents a single column in a table (which is represented by a -%% Schema[2]). This module provides a record and a function to manage all the -%% metadata required to represent a column. Metadata such as: -%% -%%
    -%%
  1. -%% `name': The Name of the column. This can either be a string or null. -%%
    2. -%%
  2. -%% `nullable': A boolean representing whether a column can have null values. -%% Generally true. -%%
    3. -%%
  3. -%% `type': The Type of the column, and is of type -%% `arrow_ipc_type:ipc_type()' -%%
    4. -%%
  4. -%% `dictionary': A boolean representing if the column is dictionary encoded -%%
    5. -%%
  5. -%% `children': The child fields of a nested datatype -%%
    6. -%%
  6. -%% `custom_metadata': A list of custom metadata in key-value format -%%
    7. -%%
-%% -%% Currently, dictionary encoding and custom metadata are not supported, but -%% they have been added for forwards comapatibility. -%% -%% [1]: [https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L469-L492] -%% -%% [2]: [https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L514-L530] -%% @end -module(arrow_ipc_field). +-moduledoc """ +Provides a record and functions to deal with individual fields in +a Schema + +A +[Field](https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L469-L492) +represents a single column in a table (which is represented by a +[Schema](https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L514-L530). +This module provides a record and a function to manage all the metadata required +to represent a column. Metadata such as: + +1. `name`: The Name of the column. This can either be a string or + null. +2. `nullable`: A boolean representing whether a column can have null + values. Generally true. +3. `type`: The Type of the column, and is of type + `t:arrow_ipc_type:ipc_type/0` +4. `dictionary`: A boolean representing if the column is dictionary + encoded +5. `children`: The child fields of a nested datatype +6. `custom_metadata`: A list of custom metadata in key-value format + +Currently, dictionary encoding and custom metadata are not supported, but +they have been added for forwards comapatibility. +""". -export([from_erlang/1, from_erlang/2, from_erlang/3]). -include("arrow_ipc_schema.hrl"). -%% @doc Creates a field given the type of the column. Assigns the name as -%% `undefined'. +-doc """ +Creates a field given the type of the column. Assigns the name as +`undefined`. +""". -spec from_erlang(Type :: arrow_ipc_type:ipc_type()) -> Field :: #field{}. from_erlang(Type) -> from_erlang(Type, undefined, []). -%% @doc Creates a field given the type and name of the column. +-doc """ +Creates a field given the type and name of the column. +""". -spec from_erlang(Type :: arrow_ipc_type:ipc_type(), Name :: string() | undefined) -> Field :: #field{}. from_erlang(Type, Name) -> from_erlang(Type, Name, []). -%% @doc Creates a field given the type, name and children of the column. +-doc """ +Creates a field given the type, name and children of the column. +""". -spec from_erlang( Type :: arrow_ipc_type:ipc_type(), Name :: string() | undefined, Children :: [#field{}] ) -> diff --git a/src/arrow_ipc_file.erl b/src/arrow_ipc_file.erl index ee45e28..a75f3dc 100644 --- a/src/arrow_ipc_file.erl +++ b/src/arrow_ipc_file.erl @@ -15,16 +15,15 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides records and functions to deal with the IPC File. -%% -%% The IPC File[1] is an extension of the IPC Stream[2] that supports random -%% access with the help of a footer which contains the offsets of all the -%% messages. -%% -%% [1]: https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format -%% [2]: https://arrow.apache.org/docs/format/Columnar.html#ipc-streaming-format -%% @end -module(arrow_ipc_file). +-moduledoc """ +Provides records and functions to deal with the IPC File. The [IPC +File](https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format) is an +extension of the [IPC Stream]( +https://arrow.apache.org/docs/format/Columnar.html#ipc-streaming-format) that +supports random access with the help of a footer which contains the offsets of +all the messages. +""". -export([from_erlang/2, to_ipc/1]). -include("arrow_ipc_message.hrl"). @@ -34,7 +33,9 @@ %% from_erlang/2 %% %%%%%%%%%%%%%%%%%%% -%% @doc Creates a file given a schema message and a list of record batch messages. +-doc """ +Creates a file given a schema message and a list of record batch messages. +""". -spec from_erlang(Schema :: #message{}, RecordBatches :: [#message{}]) -> #file{}. from_erlang(SchemaMsg, RecordBatches) -> SchemaEMF = arrow_ipc_message:to_ipc(SchemaMsg), @@ -68,7 +69,9 @@ blocks(Offset, [H | T], Blocks, EMFs) -> %% to_ipc/1 %% %%%%%%%%%%%%%% -%% @doc Serializes a file into the IPC File Format +-doc """ +Serializes a file into the IPC File Format. +""". -spec to_ipc(File :: #file{}) -> SerializedFile :: binary(). to_ipc(File) -> Footer = arrow_format_nif:serialize_footer(File#file.footer), diff --git a/src/arrow_ipc_message.erl b/src/arrow_ipc_message.erl index 58c3f14..9d5344c 100644 --- a/src/arrow_ipc_message.erl +++ b/src/arrow_ipc_message.erl @@ -15,72 +15,72 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides a record and functions to deal with the Encapsulated Message -%% Format. -%% -%% A Message is a serialized form of a Schema[1] or a RecordBatch[2] (which may -%% be required to read a serialized array) along with some metadata. This module -%% provides a record and a function to manage all the metadata required to -%% represent a message. Metadata such as: -%% -%%
    -%%
  1. -%% `version': The Apache Arrow Format Version. One of v1..v5. Defaults to v5. -%%
    2. -%%
  2. -%% `header': The metadata of the Schema or RecordBatch -%%
    3. -%%
  3. -%% `body_length': The length of the body in bytes -%%
    4. -%%
  4. -%% `custom_metadata': A list of custom metadata in key-value format -%%
    5. -%%
  5. -%% `body': The actual body. Can be undefined (in the case of Schema) or a -%% binary (in the case of Record Batch). -%%
    6. -%%
-%% -%% Currently, changing the version and custom metadata are not supported, but -%% they have been added for forwards comapatibility. -%% -%% This module also provides the `to_ipc/1' function which serializes the -%% message into the Encapsulated Message Format[3]. However, this function gives -%% incomplete output with invalid metadata, because of an unsatisfied -%% dependency on flatbuffers, which is required for serializing the metadata. -%% -%% [1]: [https://arrow.apache.org/docs/format/Columnar.html#schema-message] -%% -%% [2]: [https://arrow.apache.org/docs/format/Columnar.html#recordbatch-message] -%% -%% [3]: [https://arrow.apache.org/docs/format/Columnar.html#encapsulated-message-format] -%% @end -module(arrow_ipc_message). +-moduledoc """ +Provides a record and functions to deal with the Encapsulated Message +Format. + +A Message is a serialized form of a +[Schema](https://arrow.apache.org/docs/format/Columnar.html#schema-message) or a +[RecordBatch](https://arrow.apache.org/docs/format/Columnar.html#recordbatch-message) +(which may be required to read a serialized array) along with some metadata. +This module provides a record and a function to manage all the metadata required +to represent a message. Metadata such as: + +1. `version`: The Apache Arrow Format Version. One of `v1..v5`. + Defaults to v5. +2. `header`: The metadata of the Schema or RecordBatch +3. `body_length`: The length of the body in bytes +4. `custom_metadata`: A list of custom metadata in key-value format +5. `body`: The actual body. Can be undefined (in the case of Schema) + or a binary (in the case of Record Batch). + +Currently, changing the version and custom metadata are not supported, but they +have been added for forwards comapatibility. + +This module also provides the `to_ipc/1` function which serializes the message +into the [Encapsulated Message +Format](https://arrow.apache.org/docs/format/Columnar.html#encapsulated-message-format). +However, this function gives *incomplete output* with invalid metadata, because +of an unsatisfied dependency on flatbuffers, which is required for serializing +the metadata. +""". -export([from_erlang/1, from_erlang/2, to_ipc/1, to_stream/1, metadata_len/1, body_from_erlang/1]). -export_type([metadata_version/0, key_value/0]). -include("arrow_ipc_message.hrl"). +-doc """ +The Arrow version. See the +[definition](https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L28-L49) +for more info: +""". -type metadata_version() :: v1 | v2 | v3 | v4 | v5. -%% The Arrow version. See the definition for more info: -%% [https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L28-L49] +-doc """ +Key-Value structure for custom metadata. See the +[definition](https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L432-L439) +for more info: +""". -type key_value() :: #{key => string(), value => string()}. -%% Key-Value structure for custom metadata. See the definition for more info: -%% [https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L432-L439] -%% @doc Creates a message given a data header. +-doc """ +Creates a message given a data header. +""". -spec from_erlang(Header :: #schema{} | #record_batch{}) -> Message :: #message{}. from_erlang(Header) -> #message{header = Header, body_length = 0}. -%% @doc Creates a message given a data header and a body. +-doc """ +Creates a message given a data header and a body. +""". -spec from_erlang(Header :: #schema{} | #record_batch{}, Body :: binary()) -> Message :: #message{}. from_erlang(Header, Body) -> #message{header = Header, body = Body, body_length = byte_size(Body)}. -%% @doc Serializes a message into the Encapsulated Message Format. +-doc """ +Serializes a message into the Encapsulated Message Format. +""". -spec to_ipc(Message :: #message{}) -> EMF :: binary(). to_ipc(Message) -> %% 0xFFFFFFFF in int32 @@ -98,7 +98,9 @@ to_ipc(Message) -> <>. -%% @doc Serializes a list of messages or EMFs into a Stream. +-doc """ +Serializes a list of messages or EMFs into a Stream. +""". -spec to_stream(Messages :: [#message{}] | [binary()]) -> Stream :: binary(). to_stream([H | _] = Messages) -> Msgs = @@ -117,7 +119,9 @@ to_stream([H | _] = Messages) -> %% metadata_len/1 %% %%%%%%%%%%%%%%%%%%%% -%% @doc Returns the Metadata Length of an EMF. +-doc """ +Returns the Metadata Length of an EMF. +""". -spec metadata_len(EMF :: binary()) -> MetadataLen :: non_neg_integer(). metadata_len(EMF) -> <<_:32, MetadataLen:32, _Rest/binary>> = EMF, @@ -127,13 +131,14 @@ metadata_len(EMF) -> %% body_from_erlang/1 %% %%%%%%%%%%%%%%%%%%%%%%%% -%% @doc Returns the body of message from a list of arrays. -%% -%% Shorthand for: -%% ``` -%% <<<<(arrow_array:to_arrow(Array))/binary>> || Array <- Columns>> -%% ''' -%% @end +-doc """ +Returns the body of message from a list of arrays. + +Shorthand for: +``` +<<<<(arrow_array:to_arrow(Array))/binary>> || Array <- Columns>> +``` +""". -spec body_from_erlang(Columns :: [#array{}]) -> Body :: binary(). body_from_erlang(Columns) -> <<<<(arrow_array:to_arrow(Array))/binary>> || Array <- Columns>>. diff --git a/src/arrow_ipc_record_batch.erl b/src/arrow_ipc_record_batch.erl index c69b3d7..0d3f3dc 100644 --- a/src/arrow_ipc_record_batch.erl +++ b/src/arrow_ipc_record_batch.erl @@ -15,46 +15,33 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides a record and functions to deal with RecordBatches -%% -%% A RecordBatch[1] represents a list of equal length arrays and their -%% coresponding buffers. This module provides a record and a function to manage -%% all the metadata required to represent a RecordBatch. Metadata such as: -%% -%%
    -%%
  1. -%% `length': The number of rows or records. In other words, the length of -%% an array. -%%
    2. -%%
  2. -%% `nodes': A list of maps, where each map has the length and null count -%% of an array -%%
    3. -%%
  3. -%% `buffers': A list of maps, where each map has the length and the offset -%% (from the beginning of the message body) of a buffer of an array. -%%
    4. -%%
  4. -%% `compression': The compression applied on the body of the Record Batch. -%% Can either by `undefined' (i.e. no compression), `zstd' for Zstandard[2], -%% or `lz4_frame' for LZ4 Frame[3]. Defaults to `undefined'. -%%
    5. -%%
-%% -%% Currently, compression is not supported, but it has been added for forwards -%% comapatibility. -%% -%% You can find RecordBatches in the Arrow spec here[4]. -%% -%% [1]: [https://github.com/apache/arrow/blob/16328f0ccc73b7df665b4a18feb6adf26b7aa0e2/format/Message.fbs#L81-L102] -%% -%% [2]: [https://facebook.github.io/zstd/] -%% -%% [3]: [https://android.googlesource.com/platform/external/lz4/+/HEAD/doc/lz4_Frame_format.md] -%% -%% [4]: [https://arrow.apache.org/docs/format/Columnar.html#recordbatch-message] -%% @end -module(arrow_ipc_record_batch). +-moduledoc """ +Provides a record and functions to deal with RecordBatches +A [RecordBatch](https://github.com/apache/arrow/blob/16328f0ccc73b7df665b4a18feb6adf26b7aa0e2/format/Message.fbs#L81-L102) represents a list of equal length arrays and their +coresponding buffers. This module provides a record and a function to manage +all the metadata required to represent a RecordBatch. Metadata such as: + +1. `length`: The number of rows or records. In other words, the + length of an array. +2. `nodes`: A list of maps, where each map has the length and null + count of an array +3. `buffers`: A list of maps, where each map has the length and the + offset (from the beginning of the message body) of a buffer of an + array. + +4. `compression`: The compression applied on the body of the Record + Batch. Can either by `undefined` (i.e. no compression), `zstd` for + [Zstandard](https://facebook.github.io/zstd/), or `lz4_frame` for [LZ4 + Frame](https://android.googlesource.com/platform/external/lz4/+/HEAD/doc/lz4_Frame_format.md). + Defaults to `undefined`. + +Currently, compression is not supported, but it has been added for forwards +comapatibility. + +You can find RecordBatches in the Arrow spec +[here](https://arrow.apache.org/docs/format/Columnar.html#recordbatch-message). +""". -export([from_erlang/1]). -export_type([field_node/0, buffer/0]). @@ -64,7 +51,9 @@ -type field_node() :: #{length => pos_integer(), null_count => non_neg_integer()}. -type buffer() :: #{offset => non_neg_integer(), length => pos_integer()}. -%% @doc Creates a RecordBatch given a list of arrays +-doc """ +Creates a RecordBatch given a list of arrays +""". -spec from_erlang(Arrays :: [#array{}]) -> RecordBatch :: #record_batch{}. from_erlang(Arrays) -> FieldNodes = lists:map( diff --git a/src/arrow_ipc_schema.erl b/src/arrow_ipc_schema.erl index 22632fd..a26d766 100644 --- a/src/arrow_ipc_schema.erl +++ b/src/arrow_ipc_schema.erl @@ -15,58 +15,45 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides a record and functions to deal with Schemas -%% -%% A Schema[1] represents a table, or a list of arrays of equal length. This -%% module provides a record and a function to manage all the metadata required -%% to represent a schema. Metadata such as: -%% -%%
    -%%
  1. -%% `endianness': The Endianness of the table. One of `little' or `big'. -%% Defaults to `little'. -%%
    2. -%%
  2. -%% `fields': The list of fields[2] in a table. -%%
    3. -%%
  3. -%% `type': The Layout of the column -%%
    4. -%%
  4. -%% `custom_metadata': A list of custom metadata in key-value format -%%
    5. -%%
  5. -%% `features': Any features used by the table which may not be present in -%% other implementations of Arrow. -%%
    6. -%%
-%% -%% Currently, big endianness, custom metadata and features are not supported, -%% but they have been added for forwards comapatibility. -%% -%% You can find Schemas in the Arrow spec here[3]. -%% -%% [1]: [https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L514-L530] -%% -%% [2]: [https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L469-L492] -%% -%% [3]: [https://arrow.apache.org/docs/format/Columnar.html#schema-message] -%% @end -module(arrow_ipc_schema). +-moduledoc """ +Provides a record and functions to deal with Schemas A +[Schema](https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L514-L530)[1] +represents a table, or a list of arrays of equal length. This module provides a +record and a function to manage all the metadata required to represent a schema. +Metadata such as: + +1. `endianness`: The Endianness of the table. One of `little` or + `big`. Defaults to `little`. +2. `fields`: The list of [fields](https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L469-L492) in a table. +3. `type`: The Layout of the column +4. `custom_metadata`: A list of custom metadata in key-value format +5. `features`: Any features used by the table which may not be + present in other implementations of Arrow. + +Currently, big endianness, custom metadata and features are not supported, but +they have been added for forwards comapatibility. + +You can find Schemas in the Arrow spec +[here](https://arrow.apache.org/docs/format/Columnar.html#schema-message). +""". -export([from_erlang/1]). -export_type([endianness/0, feature/0]). -include("arrow_ipc_schema.hrl"). +-doc "Endianness of the data. Either `little` or `big`.". -type endianness() :: little | big. -%% Endianness of the data. Either `little' or `big'. +-doc """ +Features used in the data which may not be present in other implementations. +See the [definition](https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L51-L78). +""". -type feature() :: unused | dictionary_replacement | compressed_body. -%% Features used in the data which may not be present in other implementations. -%% See the definition: -%% [https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L51-L78] -%% @doc Creates a Schema given an ordered list of fields. +-doc """ +Creates a Schema given an ordered list of fields. +""". -spec from_erlang(Fields :: [#field{}]) -> Schema :: #schema{}. from_erlang(Fields) -> #schema{fields = Fields}. diff --git a/src/arrow_ipc_type.erl b/src/arrow_ipc_type.erl index f4b1e7c..5e4e172 100644 --- a/src/arrow_ipc_type.erl +++ b/src/arrow_ipc_type.erl @@ -15,20 +15,17 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides types and functions to work with IPC types. -%% -%% This module provides functions and types to produce the types in IPC Schema -%% definitions[1]. These types are generated according to these[2] definitions. -%% The types have been represented in the form `{TypeName, Metadata}', where -%% `TypeName' is the name of the type and is an atom, and `Metadata' is a map of -%% all the metadata associated with it. In case a type has no metadata -%% associated with it, it is represented as just `TypeName' -%% -%% [1]:https://github.com/apache/arrow/blob/main/format/Schema.fbs -%% -%% [2]: https://github.com/apache/arrow/blob/main/format/Schema.fbs#L82-L430 -%% @end -module(arrow_ipc_type). +-moduledoc """ +Provides types and functions to work with IPC types. This module provides +functions and types to produce the types in IPC Schema +[definitions](https://github.com/apache/arrow/blob/main/format/Schema.fbs). +These types are generated according to `Type` union in the schema. The types +have been represented in the form `{TypeName, Metadata}`, where `TypeName` is +the name of the type and is an atom, and `Metadata` is a map of all the metadata +associated with it. In case a type has no metadata associated with it, it is +represented as just `TypeName` +""". -export([from_erlang/1]). -export_type([ ipc_type/0, @@ -90,7 +87,9 @@ %% from_erlang/1 %% %%%%%%%%%%%%%%%%%%% -%% @doc Returns the IPC Type for an `#array{}'. +-doc """ +Returns the IPC Type for an `t:arrow_array:array()`. +""". -spec from_erlang(Array :: arrow_array:array()) -> Type :: ipc_type(). from_erlang(Array) -> case Array#array.layout of From 3f62fbda8af09eb1681c7a4d84318ae9680953d8 Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Sun, 19 Jul 2026 19:37:14 +0530 Subject: [PATCH 12/18] Simplify arrow_ipc_* type signatures --- src/arrow_ipc_field.erl | 14 ++++++++++---- src/arrow_ipc_file.erl | 10 +++++++--- src/arrow_ipc_message.erl | 18 ++++++++++++------ src/arrow_ipc_record_batch.erl | 7 ++++--- src/arrow_ipc_schema.erl | 7 +++++-- 5 files changed, 38 insertions(+), 18 deletions(-) diff --git a/src/arrow_ipc_field.erl b/src/arrow_ipc_field.erl index f9eb2f0..1b6938a 100644 --- a/src/arrow_ipc_field.erl +++ b/src/arrow_ipc_field.erl @@ -42,14 +42,20 @@ Currently, dictionary encoding and custom metadata are not supported, but they have been added for forwards comapatibility. """. -export([from_erlang/1, from_erlang/2, from_erlang/3]). +-export_type([field/0]). -include("arrow_ipc_schema.hrl"). +-doc """ +Represents an IPC Field. +""". +-type field() :: #field{}. + -doc """ Creates a field given the type of the column. Assigns the name as `undefined`. """. --spec from_erlang(Type :: arrow_ipc_type:ipc_type()) -> Field :: #field{}. +-spec from_erlang(Type :: arrow_ipc_type:ipc_type()) -> Field :: field(). from_erlang(Type) -> from_erlang(Type, undefined, []). @@ -57,7 +63,7 @@ from_erlang(Type) -> Creates a field given the type and name of the column. """. -spec from_erlang(Type :: arrow_ipc_type:ipc_type(), Name :: string() | undefined) -> - Field :: #field{}. + Field :: field(). from_erlang(Type, Name) -> from_erlang(Type, Name, []). @@ -65,8 +71,8 @@ from_erlang(Type, Name) -> Creates a field given the type, name and children of the column. """. -spec from_erlang( - Type :: arrow_ipc_type:ipc_type(), Name :: string() | undefined, Children :: [#field{}] + Type :: arrow_ipc_type:ipc_type(), Name :: string() | undefined, Children :: [field()] ) -> - Field :: #field{}. + Field :: field(). from_erlang(Type, Name, Children) -> #field{name = Name, type = Type, children = Children}. diff --git a/src/arrow_ipc_file.erl b/src/arrow_ipc_file.erl index a75f3dc..f3eb82c 100644 --- a/src/arrow_ipc_file.erl +++ b/src/arrow_ipc_file.erl @@ -29,6 +29,8 @@ all the messages. -include("arrow_ipc_message.hrl"). -include("arrow_ipc_file.hrl"). +-type file() :: #file{}. + %%%%%%%%%%%%%%%%%%% %% from_erlang/2 %% %%%%%%%%%%%%%%%%%%% @@ -36,7 +38,9 @@ all the messages. -doc """ Creates a file given a schema message and a list of record batch messages. """. --spec from_erlang(Schema :: #message{}, RecordBatches :: [#message{}]) -> #file{}. +-spec from_erlang( + Schema :: arrow_ipc_message:message(), RecordBatches :: [arrow_ipc_message:message()] +) -> file(). from_erlang(SchemaMsg, RecordBatches) -> SchemaEMF = arrow_ipc_message:to_ipc(SchemaMsg), {RecordBatchBlocks, EMFs} = blocks(byte_size(SchemaEMF), RecordBatches, [], []), @@ -50,7 +54,7 @@ from_erlang(SchemaMsg, RecordBatches) -> %% Returns the blocks as wells as the intermediate EMFs used to generate each block. -spec blocks( Offset :: non_neg_integer(), - Messages :: [#message{}], + Messages :: [arrow_ipc_message:message()], Blocks :: [#block{}], EMFs :: [binary()] ) -> {Blocks :: [#block{}], EMFs :: [binary()]}. @@ -72,7 +76,7 @@ blocks(Offset, [H | T], Blocks, EMFs) -> -doc """ Serializes a file into the IPC File Format. """. --spec to_ipc(File :: #file{}) -> SerializedFile :: binary(). +-spec to_ipc(File :: file()) -> SerializedFile :: binary(). to_ipc(File) -> Footer = arrow_format_nif:serialize_footer(File#file.footer), Sz = byte_size(Footer), diff --git a/src/arrow_ipc_message.erl b/src/arrow_ipc_message.erl index 9d5344c..18265e1 100644 --- a/src/arrow_ipc_message.erl +++ b/src/arrow_ipc_message.erl @@ -46,10 +46,13 @@ of an unsatisfied dependency on flatbuffers, which is required for serializing the metadata. """. -export([from_erlang/1, from_erlang/2, to_ipc/1, to_stream/1, metadata_len/1, body_from_erlang/1]). --export_type([metadata_version/0, key_value/0]). +-export_type([message/0, metadata_version/0, key_value/0]). -include("arrow_ipc_message.hrl"). +-doc "Represents a Message.". +-type message() :: #message{}. + -doc """ The Arrow version. See the [definition](https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L28-L49) @@ -67,21 +70,24 @@ for more info: -doc """ Creates a message given a data header. """. --spec from_erlang(Header :: #schema{} | #record_batch{}) -> Message :: #message{}. +-spec from_erlang(Header :: arrow_ipc_schema:schema() | arrow_ipc_record_batch:record_batch()) -> + Message :: message(). from_erlang(Header) -> #message{header = Header, body_length = 0}. -doc """ Creates a message given a data header and a body. """. --spec from_erlang(Header :: #schema{} | #record_batch{}, Body :: binary()) -> Message :: #message{}. +-spec from_erlang( + Header :: arrow_ipc_schema:schema() | arrow_ipc_record_batch:record_batch(), Body :: binary() +) -> Message :: message(). from_erlang(Header, Body) -> #message{header = Header, body = Body, body_length = byte_size(Body)}. -doc """ Serializes a message into the Encapsulated Message Format. """. --spec to_ipc(Message :: #message{}) -> EMF :: binary(). +-spec to_ipc(Message :: message()) -> EMF :: binary(). to_ipc(Message) -> %% 0xFFFFFFFF in int32 Continuation = <<-1:32>>, @@ -101,7 +107,7 @@ to_ipc(Message) -> -doc """ Serializes a list of messages or EMFs into a Stream. """. --spec to_stream(Messages :: [#message{}] | [binary()]) -> Stream :: binary(). +-spec to_stream(Messages :: [message()] | [binary()]) -> Stream :: binary(). to_stream([H | _] = Messages) -> Msgs = if @@ -139,6 +145,6 @@ Shorthand for: <<<<(arrow_array:to_arrow(Array))/binary>> || Array <- Columns>> ``` """. --spec body_from_erlang(Columns :: [#array{}]) -> Body :: binary(). +-spec body_from_erlang(Columns :: [arrow_array:array()]) -> Body :: binary(). body_from_erlang(Columns) -> <<<<(arrow_array:to_arrow(Array))/binary>> || Array <- Columns>>. diff --git a/src/arrow_ipc_record_batch.erl b/src/arrow_ipc_record_batch.erl index 0d3f3dc..0a3d517 100644 --- a/src/arrow_ipc_record_batch.erl +++ b/src/arrow_ipc_record_batch.erl @@ -43,18 +43,19 @@ You can find RecordBatches in the Arrow spec [here](https://arrow.apache.org/docs/format/Columnar.html#recordbatch-message). """. -export([from_erlang/1]). --export_type([field_node/0, buffer/0]). +-export_type([field_node/0, buffer/0, record_batch/0]). -include("arrow_ipc_record_batch.hrl"). -include("arrow_array.hrl"). -type field_node() :: #{length => pos_integer(), null_count => non_neg_integer()}. -type buffer() :: #{offset => non_neg_integer(), length => pos_integer()}. +-type record_batch() :: #record_batch{}. -doc """ Creates a RecordBatch given a list of arrays """. --spec from_erlang(Arrays :: [#array{}]) -> RecordBatch :: #record_batch{}. +-spec from_erlang(Arrays :: [arrow_array:array()]) -> RecordBatch :: record_batch(). from_erlang(Arrays) -> FieldNodes = lists:map( fun(X) -> #{length => X#array.len, null_count => X#array.null_count} end, Arrays @@ -72,7 +73,7 @@ from_erlang(Arrays) -> #record_batch{length = Length, nodes = FieldNodes, buffers = lists:reverse(Buffers)}. %% Returns the buffer data for all the buffers and nested arrays in an array. --spec array_data(Array :: #array{}, CurOffset :: non_neg_integer()) -> +-spec array_data(Array :: arrow_array:array(), CurOffset :: non_neg_integer()) -> {[buffer()], NewOffset :: non_neg_integer()}. array_data(Array, CurOffset) -> Bitmap = Array#array.validity_bitmap, diff --git a/src/arrow_ipc_schema.erl b/src/arrow_ipc_schema.erl index a26d766..3e56f39 100644 --- a/src/arrow_ipc_schema.erl +++ b/src/arrow_ipc_schema.erl @@ -38,7 +38,7 @@ You can find Schemas in the Arrow spec [here](https://arrow.apache.org/docs/format/Columnar.html#schema-message). """. -export([from_erlang/1]). --export_type([endianness/0, feature/0]). +-export_type([endianness/0, feature/0, schema/0]). -include("arrow_ipc_schema.hrl"). @@ -51,9 +51,12 @@ See the [definition](https://github.com/apache/arrow/blob/3456131ab7350bee5d9569 """. -type feature() :: unused | dictionary_replacement | compressed_body. +-doc "Represents a schema". +-type schema() :: #schema{}. + -doc """ Creates a Schema given an ordered list of fields. """. --spec from_erlang(Fields :: [#field{}]) -> Schema :: #schema{}. +-spec from_erlang(Fields :: [arrow_ipc_field:field()]) -> Schema :: schema(). from_erlang(Fields) -> #schema{fields = Fields}. From 0fb5d57b651d9c936a8f4f3b019203ef5daba389 Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Sun, 19 Jul 2026 20:00:27 +0530 Subject: [PATCH 13/18] Migrate arrow array modules --- src/arrow_fixed_list_array.erl | 68 +++++++++++++--------------- src/arrow_fixed_primitive_array.erl | 69 +++++++++++++---------------- src/arrow_variable_binary_array.erl | 28 +++++++----- src/arrow_variable_list_array.erl | 47 ++++++++++---------- 4 files changed, 103 insertions(+), 109 deletions(-) diff --git a/src/arrow_fixed_list_array.erl b/src/arrow_fixed_list_array.erl index 53a0d49..bf2f238 100644 --- a/src/arrow_fixed_list_array.erl +++ b/src/arrow_fixed_list_array.erl @@ -15,50 +15,46 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides support for Arrow's Fixed-Size List Layout. -%% -%% This module provides support for the Fixed-Size List Layout[1], which is an -%% layout that supports storing a list of lists of a specific length and -%% nesting. -%% -%% == Invalid Input == -%% -%% It is important that care is taken when passing input values to this module. -%% For performance reasons, the input is not validated. The function crashes on -%% nesting that is inconsitent: a. with the type, b. between elements. The lists -%% are checked to have the same length as each other at the top level, but not -%% for deeper levels. The behaviour of the module on invalid input -%% CANNOT BE GUARANTEED. Therefore, one must be careful to not -%% to CRASH THE PROCESS or worse still, PRODUCE INVALID -%% OUTPUT. -%% -%% Any input must follow the following rules: -%% -%%
    -%%
  1. The length of each element must be consistent with the type
    2. -%%
  2. The nesting of each element must be consistent with the type
    3. -%%
  3. The length of each element must be consistent with each other
    4. -%%
  4. The nesting of each element must be consistent with each other
    5. -%%
  5. -%% The nested type is a `fixed_list' (as only then can fixed size be -%% guaranteed) -%%
    6. -%%
-%% -%% [1]: https://arrow.apache.org/docs/format/Columnar.html#fixed-size-list-layout -module(arrow_fixed_list_array). +-moduledoc """ +Provides support for Arrow's Fixed-Size List Layout. + +This module provides support for the [Fixed-Size List +Layout](https://arrow.apache.org/docs/format/Columnar.html#fixed-size-list-layout), +which is an layout that supports storing a list of lists of a specific length +and nesting. + +## Invalid Input + +It is important that care is taken when passing input values to this module. For +performance reasons, the input is not validated. The function crashes on nesting +that is inconsitent: a. with the type, b. between elements. The lists are +checked to have the same length as each other at the top level, but not for +deeper levels. The behaviour of the module on invalid input **CANNOT BE +GUARANTEED**. Therefore, one must be careful to not to **CRASH THE PROCESS** or +worse still, **PRODUCE INVALID OUTPUT**. + +Any input must follow the following rules: + +1. The length of each element must be consistent with the type +2. The nesting of each element must be consistent with the type +3. The length of each element must be consistent with each other +4. The nesting of each element must be consistent with each other +5. The nested type is a `fixed_list` (as only then can fixed size be + guaranteed) +""". -behaviour(arrow_array). -export([from_erlang/2]). -include("arrow_array.hrl"). -%% @doc Creates a Fixed-Size List Array given the values and type. -%% -%% Accepts a map with the type, or the type directly. -%% @end +-doc """ +Creates a Fixed-Size List Array given the values and type. +Accepts a map with the type, or the type directly. +""". -spec from_erlang(Values :: list(), Type :: map() | arrow_type:arrow_type()) -> - Array :: #array{}. + Array :: arrow_array:array(). from_erlang(Values, Opts) when is_map(Opts) -> case maps:get(type, Opts, undefined) of undefined -> diff --git a/src/arrow_fixed_primitive_array.erl b/src/arrow_fixed_primitive_array.erl index 1f5bc5b..d37d31b 100644 --- a/src/arrow_fixed_primitive_array.erl +++ b/src/arrow_fixed_primitive_array.erl @@ -15,40 +15,30 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides support for Arrow's Fixed-Size Primitive Layout. -%% -%% The Primitive Layout[1] uses the following metadata in the `arrow_array': -%% -%%
    -%%
  1. -%% `layout', of type {@link atom()}, and a constant value of `fixed_primitive'. -%%
    2. -%%
  2. -%% `type', of type `t:arrow_type:arrow_primitive_type()', which -%% represents the Logical Type of the Array. -%%
    3. -%%
  3. `len', of type {@link pos_integer()}, which represents the Array's Length.
    4. -%%
  4. -%% `null_count', of type {@link non_neg_integer()}, which represents the Array's -%% Null Count, or the number of undefined values in the Array. -%%
    5. -%%
  5. -%% `validity_bitmap', which is a buffer (`arrow_buffer') or the atom -%% `undefined', which represents the Array's Validity Bitmap[2]. -%%
    6. -%%
  6. -%% `data', which is a buffer (`arrow_buffer'), which represents the Array's Value Buffer. -%%
    7. -%%
-%% -%% The `offsets' field will be allocated as `undefined'. Same goes for the -%% `validity_bitmap' field in the case of a 0 `null_count'. -%% -%% [1]: [https://arrow.apache.org/docs/format/Columnar.html#fixed-size-primitive-layout] -%% -%% [2]: [https://arrow.apache.org/docs/format/Columnar.html#validity-bitmaps] -%% @end -module(arrow_fixed_primitive_array). +-moduledoc """ +Provides support for Arrow's Fixed-Size Primitive Layout. The [Primitive +Layout](https://arrow.apache.org/docs/format/Columnar.html#fixed-size-primitive-layout) +uses the following metadata in the `arrow_array`: + +1. `layout`, of type `t:atom/0`, and a constant value of + `fixed_primitive`. +2. `type`, of type `t:arrow_type:arrow_primitive_type/0`, which + represents the Logical Type of the Array. +3. `len`, of type `t:pos_integer/0`, which represents the Array's + Length. +4. `null_count`, of type `t:non_neg_integer/0`, which represents + the Array's Null Count, or the number of undefined values in the + Array. +5. `validity_bitmap`, which is a buffer (`arrow_buffer`) or the + atom `undefined`, which represents the Array's [Validity + Bitmap](https://arrow.apache.org/docs/format/Columnar.html#validity-bitmaps). +6. `data`, which is a buffer (`arrow_buffer`), which represents the + Array's Value Buffer. + +The `offsets` field will be allocated as `undefined`. Same goes for the +`validity_bitmap` field in the case of a 0 `null_count`. +""". -behaviour(arrow_array). -export([from_erlang/2]). @@ -59,16 +49,17 @@ %% Array Creation %% %%%%%%%%%%%%%%%%%%%% -%% @doc Creates a new primitive array, given its value and type from its erlang -%% representation. -%% -%% Accepts a map with the type, or the type directly. -%% @end +-doc """ +Creates a new primitive array, given its value and type from its erlang +representation. + +Accepts a map with the type, or the type directly. +""". -spec from_erlang( Value :: [arrow_type:native_type()], Type :: map() | arrow_type:arrow_primitive_type() ) -> - Array :: #array{}. + Array :: arrow_array:array(). from_erlang(Value, Opts) when is_map(Opts) -> case maps:get(type, Opts, undefined) of undefined -> diff --git a/src/arrow_variable_binary_array.erl b/src/arrow_variable_binary_array.erl index 574b150..537d76a 100644 --- a/src/arrow_variable_binary_array.erl +++ b/src/arrow_variable_binary_array.erl @@ -15,26 +15,34 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides support for Arrow's Variable-Sized Binary Layout. -%% -%% The Variable-Sized Binary Layout[1] provides support for storing binaries of -%% varying length in a way similar to the primitive layout, i.e. in a 1 -%% Dimensional Array. -module(arrow_variable_binary_array). +-moduledoc """ +Provides support for Arrow's Variable-Sized Binary Layout. + +The [Variable-Sized Binary +Layout](https://arrow.apache.org/docs/format/Columnar.html#variable-size-binary-layout) +provides support for storing binaries of varying length in a way similar to the +primitive layout, i.e. in a 1 Dimensional Array. +""". -behaviour(arrow_array). -export([from_erlang/1, from_erlang/2]). -include("arrow_array.hrl"). -%% @doc Creates a Variable-Sized Binary Array given the values and options in the form of -%% a map, from its erlang representation. --spec from_erlang(Values :: [arrow_type:native_type()], Opts :: map()) -> Array :: #array{}. +-doc """ +Creates a Variable-Sized Binary Array given the values and options in the form of +a map, from its erlang representation. +""". +-spec from_erlang(Values :: [arrow_type:native_type()], Opts :: map()) -> + Array :: arrow_array:array(). from_erlang(Values, _Opts) -> from_erlang(Values). -%% @doc Creates a Variable-Sized Binary Array given the values --spec from_erlang(Values :: [arrow_type:native_type()]) -> Array :: #array{}. +-doc """ +Creates a Variable-Sized Binary Array given the values +""". +-spec from_erlang(Values :: [arrow_type:native_type()]) -> Array :: arrow_array:array(). from_erlang(Values) -> Len = length(Values), {Bitmap, NullCount} = arrow_bitmap:validity_bitmap(Values), diff --git a/src/arrow_variable_list_array.erl b/src/arrow_variable_list_array.erl index 79fc6af..46cdc97 100644 --- a/src/arrow_variable_list_array.erl +++ b/src/arrow_variable_list_array.erl @@ -15,30 +15,29 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides support for Arrow's Fixed-Size List Layout. -%% -%% This module provides support for the Fixed-Size List Layout[1], which is an -%% layout that supports storing a list of lists of a specific length and -%% nesting. -%% -%% == Invalid Input == -%% -%% It is important that care is taken when passing input values to this module. -%% For performance reasons, the input is not validated. The function crashes on -%% nesting that is inconsitent: a. with the type, b. between elements. The -%% behaviour on invalid input CANNOT BE GUARANTEED. Therefore, -%% one must be careful to not to CRASH THE PROCESS or worse -%% still, PRODUCE INVALID OUTPUT. -%% -%% Any input must follow the following rules: -%% -%%
    -%%
  1. The nesting of each element must be consistent with the type
    2. -%%
  2. The nesting of each element must be consistent with each other
    3. -%%
-%% -%% [1]: https://arrow.apache.org/docs/format/Columnar.html#variable-size-list-layout -module(arrow_variable_list_array). +-moduledoc """ +Provides support for Arrow's Fixed-Size List Layout. + +This module provides support for the [Fixed-Size List +Layout](https://arrow.apache.org/docs/format/Columnar.html#variable-size-list-layout), +which is an layout that supports storing a list of lists of a specific length +and nesting. + +## Invalid Input + +It is important that care is taken when passing input values to this +module. For performance reasons, the input is not validated. The +function crashes on nesting that is inconsitent: a. with the type, b. +between elements. The behaviour on invalid input **CANNOT BE +GUARANTEED**. Therefore, one must be careful to not to **CRASH THE +PROCESS** or worse still, **PRODUCE INVALID OUTPUT**. + +Any input must follow the following rules: + +1. The nesting of each element must be consistent with the type +2. The nesting of each element must be consistent with each other +""". -behaviour(arrow_array). @@ -47,7 +46,7 @@ -include("arrow_array.hrl"). -spec from_erlang(Values :: list(), Type :: map() | arrow_type:arrow_type()) -> - Array :: #array{}. + Array :: arrow_array:array(). from_erlang(Values, Opts) when is_map(Opts) -> case maps:get(type, Opts, undefined) of undefined -> From 617c63c949bfbc40680058b27cd87d9abd7cb53a Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Sun, 19 Jul 2026 20:14:54 +0530 Subject: [PATCH 14/18] Fix element_len docs --- src/arrow_array.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/arrow_array.erl b/src/arrow_array.erl index e04ae61..422c5a1 100644 --- a/src/arrow_array.erl +++ b/src/arrow_array.erl @@ -134,7 +134,7 @@ type(Array) -> len(Array) -> Array#array.len. --doc "Returns the length of an array.". +-doc "Returns the length of an element in an array.". -spec element_len(Array :: array()) -> Length :: pos_integer() | undefined. element_len(Array) -> Array#array.element_len. From 4cd56d21bebfff14cad13c2f860224b79010ece0 Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Mon, 20 Jul 2026 17:59:39 +0530 Subject: [PATCH 15/18] Fix variable size list layout link --- src/arrow_variable_list_array.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/arrow_variable_list_array.erl b/src/arrow_variable_list_array.erl index 46cdc97..c8b6d8e 100644 --- a/src/arrow_variable_list_array.erl +++ b/src/arrow_variable_list_array.erl @@ -17,9 +17,9 @@ -module(arrow_variable_list_array). -moduledoc """ -Provides support for Arrow's Fixed-Size List Layout. +Provides support for Arrow's Variable-Size List Layout. -This module provides support for the [Fixed-Size List +This module provides support for the [Variable-Size List Layout](https://arrow.apache.org/docs/format/Columnar.html#variable-size-list-layout), which is an layout that supports storing a list of lists of a specific length and nesting. From a7339630f0df31409dd120242853224a5c377443 Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Mon, 20 Jul 2026 18:06:11 +0530 Subject: [PATCH 16/18] Fix issues in arrow type --- src/arrow_type.erl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/arrow_type.erl b/src/arrow_type.erl index a239b01..1820461 100644 --- a/src/arrow_type.erl +++ b/src/arrow_type.erl @@ -62,11 +62,11 @@ Here you can find the name of each table: |------------------|-------|--------------| | Booleans | N/A | `bool` | | Signed Integer | `s` | `s64` | -| Unsigned Integer | `u ` | `u16 ` | -| Float | `f ` | `f32 ` | -| Binaries | N/A | `bin ` | +| Unsigned Integer | `u` | `u16` | +| Float | `f` | `f32` | +| Binaries | N/A | `bin` | -Types that are shorthand are under the `t:arrow_shorthand_type()` and +Types that are shorthand are under the `t:arrow_shorthand_type/0` and `arrow_short_*()` types. ### Longhand Syntax @@ -76,10 +76,10 @@ tuple is `{Name, Size}`. Thus, `Int 8` would be represented as `{s, 8}`. Do note that in the case or Booleans and Binaries their `Size` is `undefined`, as the have variable or undefined size. -Types that are longhand are under the `t:arrow_longhand_type()` and +Types that are longhand are under the `t:arrow_longhand_type/0` and `arrow_long_*()` types. -### Type +### Nested Types A Nested Type is any data structure that supports nesting. This can include Lists, FixedLists, Maps, Structs and more, but is curently limited to @@ -98,7 +98,7 @@ each element in an array. A list of `Int8` is 2 dimensional, and a list of ## Native Type A Native Type is any Erlang Datatype that can be serialized by `arrow`, -and is represented by the type `t:native_type()`. It currently only includes +and is represented by the type `t:native_type/0`. It currently only includes values that are primitive in nature such as booleans, integers, floats, binaries, and additionally the atoms `undefined` and `nil` to represent a `NULL` value. From 4ee28839c52a47e8f3e3ea6fd9ca3dfe77479572 Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Mon, 20 Jul 2026 18:17:18 +0530 Subject: [PATCH 17/18] Add bitmap TODO --- src/arrow_bitmap.erl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/arrow_bitmap.erl b/src/arrow_bitmap.erl index 449a9df..28b743c 100644 --- a/src/arrow_bitmap.erl +++ b/src/arrow_bitmap.erl @@ -56,6 +56,9 @@ There are 5 important characteristics to remember about the validity bitmap: -doc """ Returns the Validity Bitmap along with the Null Count, of an Array. """. +%% TODO: Bitmap should be arrow_buffer:buffer() | undefined +%% Investigate why Dialyzer isn't raising, and potentially +%% run eqWAlizer via ELP on the CI. -spec validity_bitmap(Value :: [arrow_type:native_type()] | list()) -> {Bitmap :: arrow_buffer:buffer(), non_neg_integer()}. validity_bitmap(Value) -> From 373634ca5ac7c8b367d7fccb44353c95478be5ca Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Mon, 20 Jul 2026 18:35:30 +0530 Subject: [PATCH 18/18] Fix spelling, grammar and formatting errors --- src/arrow_array.erl | 2 +- src/arrow_buffer.erl | 4 ++-- src/arrow_fixed_list_array.erl | 6 +++--- src/arrow_ipc_message.erl | 2 +- src/arrow_ipc_record_batch.erl | 8 +++++--- src/arrow_ipc_schema.erl | 7 ++++--- src/arrow_ipc_type.erl | 9 +++++---- src/arrow_offsets.erl | 4 ++-- src/arrow_type.erl | 8 ++++---- src/arrow_variable_list_array.erl | 2 +- 10 files changed, 28 insertions(+), 24 deletions(-) diff --git a/src/arrow_array.erl b/src/arrow_array.erl index 422c5a1..56a89a4 100644 --- a/src/arrow_array.erl +++ b/src/arrow_array.erl @@ -175,7 +175,7 @@ order: 2. `offsets` 3. `data` -In case an array doesn't have any of the following buffers, it is ommitted. +In case an array doesn't have any of the following buffers, it is omitted. (e.g. validity in arrays with a null count of 0, offsets in fixed primitive arrays). In the case of a nested array, `data` will be serialized form of nested array. diff --git a/src/arrow_buffer.erl b/src/arrow_buffer.erl index 0c6cff8..ecec6d5 100644 --- a/src/arrow_buffer.erl +++ b/src/arrow_buffer.erl @@ -22,7 +22,7 @@ -moduledoc """ Buffer implementation for `arrow`. -This module adds suppport for buffers, or Contiguous Memory Regions. There are +This module adds support for buffers, or Contiguous Memory Regions. There are multiple things to know about [buffers](https://arrow.apache.org/docs/format/Glossary.html#term-buffer): @@ -73,7 +73,7 @@ from_erlang(Data, Type) -> -doc """ Creates a new buffer from a list of Erlang values or binaries, given its -and length +type and length """. -spec from_erlang( Data :: [arrow_type:native_type()] | binary(), diff --git a/src/arrow_fixed_list_array.erl b/src/arrow_fixed_list_array.erl index bf2f238..7308d7c 100644 --- a/src/arrow_fixed_list_array.erl +++ b/src/arrow_fixed_list_array.erl @@ -21,14 +21,14 @@ Provides support for Arrow's Fixed-Size List Layout. This module provides support for the [Fixed-Size List Layout](https://arrow.apache.org/docs/format/Columnar.html#fixed-size-list-layout), -which is an layout that supports storing a list of lists of a specific length -and nesting. +which is a layout that supports storing a list of lists of a specific length and +nesting. ## Invalid Input It is important that care is taken when passing input values to this module. For performance reasons, the input is not validated. The function crashes on nesting -that is inconsitent: a. with the type, b. between elements. The lists are +that is inconsistent: either with the type, or between elements. The lists are checked to have the same length as each other at the top level, but not for deeper levels. The behaviour of the module on invalid input **CANNOT BE GUARANTEED**. Therefore, one must be careful to not to **CRASH THE PROCESS** or diff --git a/src/arrow_ipc_message.erl b/src/arrow_ipc_message.erl index 18265e1..c27eb14 100644 --- a/src/arrow_ipc_message.erl +++ b/src/arrow_ipc_message.erl @@ -36,7 +36,7 @@ to represent a message. Metadata such as: or a binary (in the case of Record Batch). Currently, changing the version and custom metadata are not supported, but they -have been added for forwards comapatibility. +have been added for forwards compatibility. This module also provides the `to_ipc/1` function which serializes the message into the [Encapsulated Message diff --git a/src/arrow_ipc_record_batch.erl b/src/arrow_ipc_record_batch.erl index 0a3d517..6d1cd77 100644 --- a/src/arrow_ipc_record_batch.erl +++ b/src/arrow_ipc_record_batch.erl @@ -17,9 +17,11 @@ -module(arrow_ipc_record_batch). -moduledoc """ -Provides a record and functions to deal with RecordBatches +Provides a record and functions to deal with RecordBatches. + + A [RecordBatch](https://github.com/apache/arrow/blob/16328f0ccc73b7df665b4a18feb6adf26b7aa0e2/format/Message.fbs#L81-L102) represents a list of equal length arrays and their -coresponding buffers. This module provides a record and a function to manage +corresponding buffers. This module provides a record and a function to manage all the metadata required to represent a RecordBatch. Metadata such as: 1. `length`: The number of rows or records. In other words, the @@ -31,7 +33,7 @@ all the metadata required to represent a RecordBatch. Metadata such as: array. 4. `compression`: The compression applied on the body of the Record - Batch. Can either by `undefined` (i.e. no compression), `zstd` for + Batch. Can either be `undefined` (i.e. no compression), `zstd` for [Zstandard](https://facebook.github.io/zstd/), or `lz4_frame` for [LZ4 Frame](https://android.googlesource.com/platform/external/lz4/+/HEAD/doc/lz4_Frame_format.md). Defaults to `undefined`. diff --git a/src/arrow_ipc_schema.erl b/src/arrow_ipc_schema.erl index 3e56f39..072a02a 100644 --- a/src/arrow_ipc_schema.erl +++ b/src/arrow_ipc_schema.erl @@ -17,8 +17,9 @@ -module(arrow_ipc_schema). -moduledoc """ -Provides a record and functions to deal with Schemas A -[Schema](https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L514-L530)[1] +Provides a record and functions to deal with Schemas. + +A [Schema](https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L514-L530) represents a table, or a list of arrays of equal length. This module provides a record and a function to manage all the metadata required to represent a schema. Metadata such as: @@ -32,7 +33,7 @@ Metadata such as: present in other implementations of Arrow. Currently, big endianness, custom metadata and features are not supported, but -they have been added for forwards comapatibility. +they have been added for forwards compatibility. You can find Schemas in the Arrow spec [here](https://arrow.apache.org/docs/format/Columnar.html#schema-message). diff --git a/src/arrow_ipc_type.erl b/src/arrow_ipc_type.erl index 5e4e172..def4957 100644 --- a/src/arrow_ipc_type.erl +++ b/src/arrow_ipc_type.erl @@ -17,14 +17,15 @@ -module(arrow_ipc_type). -moduledoc """ -Provides types and functions to work with IPC types. This module provides -functions and types to produce the types in IPC Schema +Provides types and functions to work with IPC types. + +This module provides functions and types to produce the types in IPC Schema [definitions](https://github.com/apache/arrow/blob/main/format/Schema.fbs). These types are generated according to `Type` union in the schema. The types have been represented in the form `{TypeName, Metadata}`, where `TypeName` is the name of the type and is an atom, and `Metadata` is a map of all the metadata associated with it. In case a type has no metadata associated with it, it is -represented as just `TypeName` +represented as just `TypeName`. """. -export([from_erlang/1]). -export_type([ @@ -88,7 +89,7 @@ represented as just `TypeName` %%%%%%%%%%%%%%%%%%% -doc """ -Returns the IPC Type for an `t:arrow_array:array()`. +Returns the IPC Type for an `t:arrow_array:array/0`. """. -spec from_erlang(Array :: arrow_array:array()) -> Type :: ipc_type(). from_erlang(Array) -> diff --git a/src/arrow_offsets.erl b/src/arrow_offsets.erl index 4e1d49b..9fef0b0 100644 --- a/src/arrow_offsets.erl +++ b/src/arrow_offsets.erl @@ -28,8 +28,8 @@ support for generating offsets. There are couple of things to remember about offsets: -Firstly, each element in the offsets coresponds to the distance in bytes of the -coresponding element in the values from the beginning of the buffer. +Firstly, each element in the offsets corresponds to the distance in bytes of the +corresponding element in the values from the beginning of the buffer. I.E., distance of `values[j]` from beginning of the buffer = `offsets[j]`. diff --git a/src/arrow_type.erl b/src/arrow_type.erl index 1820461..899e96c 100644 --- a/src/arrow_type.erl +++ b/src/arrow_type.erl @@ -20,10 +20,10 @@ Provides representation for as well as functions for working with Arrow specific datatypes. -This module provides a longhand (a 2 tuple) as well as a shorthand (an atom) -represenations for representing a primitive types. Additionally it represents -Nested Types (i.e. Lists) as well as Native Types (i.e. Erlang Native Types -that are supported by this package). +This module provides longhand (a 2 tuple) as well as shorthand (an atom) +representations for representing primitive types. Additionally it represents +Nested Types (i.e. Lists) as well as Native Types (i.e. Erlang Native Types that +are supported by this package). Generally, when we say an array A has type X, we mean that each element in A has type X. Thus, an array of type of `Int8` would look like `[1, 2, 3]`, but diff --git a/src/arrow_variable_list_array.erl b/src/arrow_variable_list_array.erl index c8b6d8e..630954d 100644 --- a/src/arrow_variable_list_array.erl +++ b/src/arrow_variable_list_array.erl @@ -28,7 +28,7 @@ and nesting. It is important that care is taken when passing input values to this module. For performance reasons, the input is not validated. The -function crashes on nesting that is inconsitent: a. with the type, b. +function crashes on nesting that is inconsistent: a. with the type, b. between elements. The behaviour on invalid input **CANNOT BE GUARANTEED**. Therefore, one must be careful to not to **CRASH THE PROCESS** or worse still, **PRODUCE INVALID OUTPUT**.