Improve compile time behavior to support multiversion deployments

.github/workflows/ci.yml

@@ -39,7 +39,7 @@ jobs:
           otp-version: ${{matrix.otp}}
           rebar3-version: "3.24.0"
       - name: Run rebar3 fmt check for Erlang files
-        run: ERLFDB_ASSERT_FDBCLI=0 rebar3 fmt --check
+        run: ERLFDB_ASSERT_API_VERSION=0 rebar3 fmt --check
         if: ${{ matrix.fmt }}
       - name: Run clang-format check for C files
         uses: jidicula/clang-format-action@v4.11.0
@@ -76,6 +76,32 @@ jobs:
           pip3 show foundationdb
           ./test/bindingtester/loop.sh
         if: ${{ matrix.bindingtester }}
+  multiversion:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - otp: "27.0.1"
+            fdb_old: "7.2.2"
+            fdb_new: "7.3.62"
+    name: "linux / OTP ${{ matrix.otp }} / FDB ${{ matrix.fdb_old }} -> ${{ matrix.fdb_new }} (multiversion)"
+    env:
+      FDB_OLD: ${{ matrix.fdb_old }}
+      FDB_NEW: ${{ matrix.fdb_new }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: erlef/setup-beam@v1
+        with:
+          otp-version: ${{matrix.otp}}
+          rebar3-version: "3.24.0"
+      - name: Download FDB packages
+        run: |
+          wget https://github.com/apple/foundationdb/releases/download/${FDB_OLD}/foundationdb-clients_${FDB_OLD}-1_amd64.deb
+          wget https://github.com/apple/foundationdb/releases/download/${FDB_OLD}/foundationdb-server_${FDB_OLD}-1_amd64.deb
+          wget https://github.com/apple/foundationdb/releases/download/${FDB_NEW}/foundationdb-clients_${FDB_NEW}-1_amd64.deb
+          wget https://github.com/apple/foundationdb/releases/download/${FDB_NEW}/foundationdb-server_${FDB_NEW}-1_amd64.deb
+      - name: Run multiversion upgrade test
+        run: bash test/multiversion/run_upgrade_test.sh
   macos:
     runs-on: macos-latest
     strategy:

CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## v1.1.0 (2026-03-07)
+
+### Enhancements
+
+ * Compile-time configuration: `rebar.config.script` reworked to support `ERLFDB_COMPILE_API_VERSION`, `ERLFDB_INCLUDE_DIR`, and `ERLFDB_FDBCLI` env vars, with better fallback behavior when fdbcli isn't available
+ * Multiversion CI job: new GitHub Action that compiles erlfdb against FDB 7.2.2, runs a single fdbserver, upgrades to 7.3.62, and verifies the multi-version client connects to both
+
+### Documentation
+
+ * New `notes/configuration.md` covering all compile-time and runtime options
+
 ## v1.0.0 (2026-02-28)
 
 No changes.

README.md

@@ -109,12 +109,12 @@ against the most recent supported version of FoundationDB.
 
 ### Bypassing dependency checks
 
-When you execute a rebar command, erlfdb attempts to detect the version of the fdbcli
-installed on your system. If it cannot be detected, the rebar command is aborted.
+When you execute a rebar command, erlfdb attempts to detect the FDB API version
+via `fdbcli`. If it cannot be determined, the rebar command is aborted.
 
-To disable the abort, use `ERLFDB_ASSERT_FDBCLI=0`. For example, you can safely use
+To disable the abort, use `ERLFDB_ASSERT_API_VERSION=0`. For example, you can safely use
 this for the `fmt` command, which does not do a compile action.
 
 ```bash
-ERLFDB_ASSERT_FDBCLI=0 rebar3 fmt
+ERLFDB_ASSERT_API_VERSION=0 rebar3 fmt
 ```

notes/configuration.md

@@ -0,0 +1,157 @@
+# Configuration
+
+erlfdb supports compile-time and runtime configuration. Compile-time options are resolved in `rebar.config.script` during the build. Runtime options are set via application environment variables, typically in a `sys.config` file.
+
+## Compile-Time Configuration
+
+The following OS environment variables are read by `rebar.config.script` via `os:getenv/1` during compilation. These must be environment variables (rather than Erlang compile-time defines) because the script runs before Erlang compilation begins — it is responsible for producing the `erl_opts` and `port_env` that drive the build. The purpose of these options is to provide the developer with control over which libfdb_c function calls exist in the NIF library's symbol table, so that the intended library can be loaded at runtime.
+
+### `ERLFDB_INCLUDE_DIR`
+
+Path to the directory containing the FoundationDB C API header files. Defaults to `/usr/local/include`.
+
+```shell
+export ERLFDB_INCLUDE_DIR=/opt/foundationdb/include
+```
+
+### `ERLFDB_COMPILE_API_VERSION`
+
+When set, the build uses this value directly as the compile-time FDB API version, bypassing `fdbcli` checks. This is useful in environments where `fdbcli` is not installed on the build host but the FoundationDB client library and headers are available (e.g. multi-version support, cross-compilation or CI). erlfdb will compile only with features that are supported in this version, and will throw a `badarg` error when calling code executes an erlfdb function that is not supported. For example, calling `erlfdb:get_mapped_range/4` when compiled with version 710.
+
+When unset, the build attempts to find the `fdbcli` binary to determine the compile-time API version (see below).
+
+```shell
+export ERLFDB_COMPILE_API_VERSION=730
+```
+
+### `ERLFDB_FDBCLI`
+
+Path to the `fdbcli` executable. If not set, erlfdb searches `PATH` and `/usr/local/bin`. The build script uses `fdbcli --version` to detect the FDB protocol version, which determines the C API version used during compilation. Ignored when `ERLFDB_COMPILE_API_VERSION` is set.
+
+```shell
+export ERLFDB_FDBCLI=/opt/foundationdb/bin/fdbcli
+```
+
+### `ERLFDB_ASSERT_API_VERSION`
+
+When set to `"0"`, the build will continue with a warning if the FDB API version cannot be determined, instead of aborting. By default (any other value, or unset), an undetectable API version is a fatal error.
+
+```shell
+# Allow the build to proceed without a detected API version
+export ERLFDB_ASSERT_API_VERSION=0
+```
+
+## Runtime Configuration
+
+Runtime configuration is set via the `erlfdb` application environment, typically in a `sys.config` file or equivalent. Unless specified otherwise, all runtime configuration is evaluated when the NIF is loaded, and made permanent for the lifetime of the VM process.
+
+### `api_version`
+
+The FoundationDB C API version to use at runtime. Defaults to the value of the `erlfdb_compile_time_api_version` compile-time macro, which is auto-detected from `fdbcli` during the build, or from `ERLFDB_COMPILE_API_VERSION`.
+
+```erlang
+[
+    {erlfdb, [
+        {api_version, 730}
+    ]}
+].
+```
+
+### `network_options`
+
+A proplist of options passed to the FoundationDB client network layer during NIF initialization. Your options are merged with erlfdb's defaults, with your values taking precedence. Setting any option to `false` causes it to be removed from the resolved list, making it a no-op when options are applied to FDB. This can be used to disable a default.
+
+The resolved options are stored in the `network_options_resolved` application env var and can be inspected at runtime:
+
+```erlang
+application:get_env(erlfdb, network_options_resolved).
+```
+
+#### Defaults
+
+```erlang
+[
+    {callbacks_on_external_threads, true},
+    {external_client_library, {erlfdb_network_options, compile_time_external_client_library, []}},
+    {client_threads_per_version, 1}
+].
+```
+
+#### Key Options
+
+##### `callbacks_on_external_threads`
+
+When `true`, allows libfdb_c to execute future callbacks on external threads. This increases throughput of future resolution and is recommended for erlfdb. Default: `true`.
+
+##### `external_client_library`
+
+Path (as a binary string) to the `libfdb_c` dynamic library. libfdb_c copies this library N times to temporary files so they can be individually loaded by external client threads.
+
+The value may also be an `{M, F, A}` tuple. The function is called when the NIF is loaded and must return a binary string. Default: `{erlfdb_network_options, compile_time_external_client_library, []}`, which returns the library path detected at compile time.
+
+##### `client_threads_per_version`
+
+The number of client threads to create per dynamic library. This is the primary knob for scaling FDB client throughput horizontally.
+
+- When set to `1`, only the local client thread is used.
+- When set to `N > 1`, N external client threads are created in addition to the main thread.
+
+The value may also be an `{M, F, A}` tuple. The function is called when the NIF is loaded and must return a positive integer.
+
+> #### Note {: .info}
+> For each thread, libfdb_c creates a TCP connection for each coordinator in the cluster file. Choose a value sufficient for your workload but no larger.
+
+Default: `1`.
+
+Example scaling to 2 threads:
+
+```erlang
+[
+    {erlfdb, [
+        {network_options, [
+            {client_threads_per_version, 2}
+        ]}
+    ]}
+].
+```
+
+##### `external_client_directory`
+
+Path (as a binary string) to a directory containing additional `libfdb_c` dynamic libraries for multi-version client support. When set, libfdb_c loads all client libraries found in this directory, enabling connections to clusters running different FoundationDB versions. Each library version is loaded alongside the primary client.
+
+```erlang
+[
+    {erlfdb, [
+        {network_options, [
+            {external_client_directory, <<"/usr/lib/foundationdb/multiversion">>}
+        ]}
+    ]}
+].
+```
+
+##### `trace_enable`
+
+A binary string path to a directory where client trace files will be written. The directory must exist and be writable.
+
+##### `trace_format`
+
+The format of trace files. Supported values: `<<"xml">>` (default) and `<<"json">>`.
+
+```erlang
+[
+    {erlfdb, [
+        {network_options, [
+            {trace_enable, <<"/var/log/fdb_traces">>},
+            {trace_format, <<"json">>}
+        ]}
+    ]}
+].
+```
+
+#### All Supported Network Options
+
+The full set of network options corresponds to the [FoundationDB C API network options](https://apple.github.io/foundationdb/api-c.html#c.fdb_network_set_option).
+
+Any option value may be set to `false` to disable it, even if it was set in the defaults. When `false`, the option key and value do not appear in the resolved options.
+
+Any option value may be set to `{M, F, A}` tuple to acquire the value at runtime. The option key and value will appear in the resolved options.

rebar.config

@@ -24,6 +24,7 @@
         <<"README.md">>,
         <<"LICENSE">>,
         <<"CHANGELOG.md">>,
+        <<"notes/configuration.md">>,
         <<"notes/thread-design.md">>,
         <<"notebooks/kv_queue.livemd">>,
         <<"notebooks/tutorial-elixir.livemd">>

rebar.config.script

@@ -1,21 +1,12 @@
 FDBReleasesBaseUrl = "https://api.github.com/repos/apple/foundationdb/releases".
-IncludeDir = "/usr/local/include".
+IncludeDir = case os:getenv("ERLFDB_INCLUDE_DIR") of
+    false -> "/usr/local/include";
+    Dir -> Dir
+end.
 
 % Detect rebar3 or mix
 RebarApi = {error, nofile} =/= code:ensure_loaded(rebar_api).
 
-% Look for fdbcli on PATH, but also check /usr/local/bin explicitly since that is the common location
-FdbCli =
-    case os:find_executable("fdbcli") of
-        false ->
-            case os:find_executable("fdbcli", "/usr/local/bin") of
-                false -> "fdbcli";
-                Exec -> Exec
-            end;
-        Exec ->
-            Exec
-    end.
-
 % Logs to stdout/stderr from either rebar3 or mix
 Log = fun(Level, Fmt, Args) ->
     if
@@ -38,24 +29,48 @@ Log = fun(Level, Fmt, Args) ->
     end
 end.
 
+% Look for fdbcli on PATH, but also check /usr/local/bin explicitly since that is the common location.
+% Override with ERLFDB_FDBCLI env var to point at a specific version's fdbcli.
 % Check architecture against regex
 IsArch = fun(ArchRe) ->
     match =:= re:run(erlang:system_info(system_architecture), ArchRe, [{capture, none}])
 end.
 
-% Hacky means to extract API version from fdbcli protocol version output
-% See https://github.com/apple/foundationdb/blob/master/flow/ProtocolVersion.h
-MaxAPIVersion =
-    begin
-        VsnInfo = os:cmd(FdbCli ++ " --version"),
-        case re:run(VsnInfo, "protocol ([a-f0-9]*)", [{capture, [1], list}]) of
-            {match, [ProtocolStr]} ->
-                ProtocolVsn = list_to_integer(ProtocolStr, 16),
-                APIVersionBytes = (ProtocolVsn band 16#0000000FFF00000) bsr 20,
-                list_to_integer(integer_to_list(APIVersionBytes, 16));
-            nomatch ->
-                undefined
-        end
+CompileAPIVersion =
+    case os:getenv("ERLFDB_COMPILE_API_VERSION") of
+        false ->
+            % Look for fdbcli on PATH, but also check /usr/local/bin explicitly since that is the common location.
+            % Override with ERLFDB_FDBCLI env var to point at a specific version's fdbcli.
+            FdbCli =
+                case os:getenv("ERLFDB_FDBCLI") of
+                    false ->
+                        case os:find_executable("fdbcli") of
+                            false ->
+                                case os:find_executable("fdbcli", "/usr/local/bin") of
+                                    false -> "fdbcli";
+                                    Exec -> Exec
+                                end;
+                            Exec ->
+                                Exec
+                        end;
+                    Path ->
+                        Path
+                end,
+
+            % Hacky means to extract API version from fdbcli protocol version output
+            % See https://github.com/apple/foundationdb/blob/master/flow/ProtocolVersion.h
+            VsnInfo = os:cmd(FdbCli ++ " --version"),
+            case re:run(VsnInfo, "protocol ([a-f0-9]*)", [{capture, [1], list}]) of
+                {match, [ProtocolStr]} ->
+                    ProtocolVsn = list_to_integer(ProtocolStr, 16),
+                    APIVersionBytes = (ProtocolVsn band 16#0000000FFF00000) bsr 20,
+                    list_to_integer(integer_to_list(APIVersionBytes, 16));
+                nomatch ->
+                    undefined
+            end;
+        ApiVsnStr ->
+            Log(info, "Using ERLFDB_COMPILE_API_VERSION=~s", [ApiVsnStr]),
+            list_to_integer(ApiVsnStr)
     end.
 
 % Make an HTTP GET request, expect JSON body
@@ -181,8 +196,8 @@ LogAssetsMap = fun
         )
 end.
 
-% If MaxAPIVersion undetected, print helpful informatiom about where to download the latest FDB Release
-case MaxAPIVersion of
+% If CompileAPIVersion undetected, print helpful information about where to download the latest FDB Release
+case CompileAPIVersion of
     undefined ->
         Log(info, "Checking for latest FoundationDB release...", []),
         {ok, _} = application:ensure_all_started(inets),
@@ -200,14 +215,15 @@ case MaxAPIVersion of
                     "The foundationdb-clients package is required to compile erlfdb. Please visit~n~n        https://github.com/apple/foundationdb/releases~n"
             end,
         [Log(info, Message, []) || Message =/= undefined],
-        AssertFdbCli = "0" =/= os:getenv("ERLFDB_ASSERT_FDBCLI"),
+        AssertApiVersion = "0" =/= os:getenv("ERLFDB_ASSERT_API_VERSION"),
         if
-            AssertFdbCli ->
-                Log(abort, "Error: fdbcli not found on PATH.", []);
+            AssertApiVersion ->
+                Log(abort, "Error: FDB API version could not be determined. Set ERLFDB_COMPILE_API_VERSION or install fdbcli.", []);
             true ->
-                Log(error, "fdbcli not found on PATH. Continuing anyway...", [])
+                Log(error, "FDB API version could not be determined. Continuing anyway...", [])
         end;
     _ ->
+        Log(info, "Compiling with FDB_API_VERSION=~p", [CompileAPIVersion]),
         ok
 end.
 
@@ -224,7 +240,7 @@ DynamicLibraryDir = filename:dirname(DynamicLibrary).
 
 [
     {erl_opts, ErlOpts ++ [
-        {d, erlfdb_api_version, MaxAPIVersion},
+        {d, erlfdb_compile_time_api_version, CompileAPIVersion},
         {d, erlfdb_compile_time_external_client_library, DynamicLibrary}
     ]},
     {port_env, [
@@ -233,21 +249,21 @@ DynamicLibraryDir = filename:dirname(DynamicLibrary).
             "CFLAGS",
             "$CFLAGS -I"++IncludeDir++" -Ic_src/ -g -Wall -Werror " ++
                 (if
-                    MaxAPIVersion < 730 ->
-                        "-DFDB_API_VERSION=" ++ integer_to_list(MaxAPIVersion);
+                    is_integer(CompileAPIVersion) ->
+                        "-DFDB_API_VERSION=" ++ integer_to_list(CompileAPIVersion);
                     true ->
                         "-DFDB_USE_LATEST_API_VERSION=1"
                 end)
         },
         {
             "(linux|solaris|freebsd|netbsd|openbsd|dragonfly|gnu)",
             "LDFLAGS",
-            "$LDFLAGS -L"++DynamicLibraryDir++" -lfdb_c"
+            "$LDFLAGS -Wl,-rpath,\\$ORIGIN -Wl,-rpath,"++DynamicLibraryDir++" -L"++DynamicLibraryDir++" -lfdb_c"
         },
         {
             "(darwin)",
             "LDFLAGS",
-            "$LDFLAGS -rpath "++DynamicLibraryDir++" -L"++DynamicLibraryDir++" -lfdb_c"
+            "$LDFLAGS -rpath @loader_path -rpath "++DynamicLibraryDir++" -L"++DynamicLibraryDir++" -lfdb_c"
         }
     ]}
 ] ++ CONFIG1.